\documentclass{article}

\usepackage[margin=1.2in]{geometry}
\usepackage{graphicx}
\usepackage{amsmath,amssymb,amsthm,bm}
\usepackage{latexsym,color,minipage-marginpar,caption,multirow,verbatim}
\usepackage{enumerate}
\usepackage[round]{natbib}
\usepackage{times}

\newcommand{\cP}{\mathcal{P}}
\newcommand{\cX}{\mathcal{X}}
\newcommand{\cR}{\mathcal{R}}
\newcommand{\cT}{\mathcal{T}}
\newcommand{\cY}{\mathcal{Y}}
\newcommand{\cQ}{\mathcal{Q}}

\newcommand{\EE}{\mathbb{E}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\RR}{\mathbb{R}}
\newcommand{\ZZ}{\mathbb{Z}}

\newcommand{\Var}{\textnormal{Var}}
\newcommand{\MSE}{\textnormal{MSE}}

\DeclareMathOperator*{\argmax}{argmax}


\newcommand{\simiid}{\overset{\text{i.i.d.}}{\sim}}
\newcommand{\simind}{\overset{\text{ind.}}{\sim}}
\newcommand{\ep}{\varepsilon}

\definecolor{darkblue}{rgb}{0.2, 0.2, 0.5}

\newenvironment{solution}{~\\\color{darkblue}{\bf Solution:~\\}}{}
\newenvironment{ifsol}{\color{darkblue}}{}

\newcommand{\optional}{{\bf Optional:} (Not graded, no extra points) }

\theoremstyle{definition}
\newtheorem{problem}{Problem}


\begin{document}

\title{Stats 210A, Fall 2025\\
  Homework 5 \\
  {\large {\bf Due on}: Wednesday, Oct. 8}}
\date{}

\maketitle

\paragraph{Instructions:} You may disregard measure-theoretic niceties about conditioning on measure-zero sets, almost-sure equality vs. actual equality, ``all functions'' vs. ``all measurable functions,'' etc. (unless the problem is explicitly asking about such issues).


\begin{problem}[Ridge regression]
\label{prob:ridge-bayes}

Consider the {\em Gaussian linear model} where
\[
y_i = x_i' \beta + \ep_i, \quad \text{ with } \ep_i \simiid N(0,\sigma^2) \;\text{ for } i =1, \ldots n,
\]
where $\beta \in \RR^d$ is unknown, and the covariate vectors $x_i \in \RR^d$ are fixed and known. Assume the error variance $\sigma^2>0$ is also known. We observe the response vector $y \in \RR^n$.
\begin{enumerate}[(a)]
\item Assume that $d \leq n$, and the design matrix $\mathbf{X}$ (the $n \times d$ matrix whose $i$th row is $x_i'$) has full column rank. Show that the OLS estimator $\hat\beta = (\mathbf{X}'\mathbf{X})^{-1}\mathbf{X}'y$ is the UMVU estimator of $\beta$. 

{\bf Note:} Remember that the design matrix $\mathbf{X}$ is not data in the same sense $y$ is; it is more like a known parameter.




\item Now consider Bayesian estimation with the prior $\beta \sim N(\mu, \tau^2 I_d)$.  Under the same prior as in part (b), find the posterior distribution of $\beta$. Does it matter whether $d > n$, or whether $\mathbf{X}$ has full column rank?




% \item Fix $\beta$ and show that, for sufficiently large $\tau^2$, the MSE of the Bayes estimator is strictly smaller than the MSE of the OLS estimator.

% {\bf Note:} How large we need to take $\tau^2$ depends on the unknown parameter $\beta$, so this doesn't give us a feasible strategy for improving over OLS in practice.

\item Suppose that $\mathbf{X}\gamma = 0$ for some nonzero $\gamma \in \RR^d$ whose entries are all nonzero. Show that no unbiased estimator exists for any $\beta_j$. In your opinion, should this give us any reason for concern about the Bayes estimator? (The subjective part of the question will be graded leniently).



\end{enumerate}

\paragraph{Moral:} While the humble ridge regression method was initially introduced just to deal with poorly conditioned regression problems, it has an appealing Bayesian interpretation. Moreover, using Bayesian methods often helps us sidestep concerns about identifiability --- but we should think hard about the implications of doing so.


\end{problem}


\begin{problem}[Admissibility and Bayes estimators]
\label{prob:admissibility-bayes}

One of the frequentist motivations for Bayes estimators is their connection to admissibility.

\begin{enumerate}[(a)]
\item Suppose that the Bayes estimator $\delta_{\Lambda}$ for the prior $\Lambda$ is unique up to $\cP$-almost-sure equality. That is, for any other Bayes estimator $\tilde\delta_{\Lambda}$, we have $\delta_\Lambda(X) = \tilde\delta_\Lambda(X)$ almost surely, for every $P_\theta\in\cP$. Show that $\delta_\Lambda$ is admissible.



\item Now suppose that $\Theta$ is discrete (possibly countably infinite) and $\Lambda$ has a strictly positive probability mass function $\lambda$, i.e. $\lambda(\theta) > 0$ for all $\theta\in \Theta$. Show that any Bayes estimator with finite Bayes risk is admissible.



\item \optional If $\Theta \subseteq \RR^d$ and $\Lambda$ has a strictly positive density, can we likewise say that all Bayes estimators with finite Bayes risk are admissible? Prove or give a counterexample.



\item A {\em randomized estimator} is an estimator that is a random function of the data. We can formalize it generically as $\delta(X,W)$ where $X\sim P_\theta$ as usual and $W$ is some auxiliary random variable generated by the analyst. For this part, ``admissible'' and ``Bayes'' are defined with respect to all estimators including randomized ones.

Now consider a model with a finite parameter space, $|\Theta|=n<\infty$ and assume we are estimating some real-valued $g(\theta)$ using a bounded non-negative loss $L:\Theta \times \RR \to [0,\infty)$. Show that every admissible estimator is a (possibly randomized) Bayes estimator for some prior.

{\bf Hint:} consider the set $\mathcal{A}$ of all achievable risk functions, and the set $\mathcal{D}_\delta$ of all (possibly unachievable) risk functions that would dominate a given estimator $\delta$. Recall the {\em hyperplane separation theorem}: for any two disjoint non-empty convex subsets $A,B\subseteq\RR^n$ there exist $c\in\RR$ and nonzero $\lambda\in\RR^n$ such that $\lambda'a \geq c \geq \lambda'b$ for all $a\in A, b\in B$. It might help to draw pictures for $n=2$.



\end{enumerate}

\paragraph{Moral:} Minimizing average-case risk is closely related to admissibility, though the general relationship is not quite as tight as what we've shown in this problem for finite parameter spaces. In more general parameter spaces, there is a more general result which roughly states that all admissible estimators are limits of Bayes estimators, under relatively mild conditions.
\end{problem}






\begin{problem}[Other loss functions]
\label{prob:other-loss}

Assume for each problem below that there exists an estimator with finite Bayes risk.
\begin{enumerate}[(a)]
\item Consider a Bayesian model with a discrete parameter $\theta$. What is the Bayes estimator for the loss $L(\theta, d) = 1\{\theta \neq d\}$?




\item Next consider a Bayesian model with a single real parameter $\theta$, and assume that the posterior distribution of $\theta$ given $X=x$ is absolutely continuous (with respect to the Lebesgue measure) for all $x$. What is the Bayes estimator for the {\em absolute error loss} $L(\theta, d) = |\theta-d|$? 




\item Under the same assumptions as part (b), what loss function $L_\gamma(\theta, d)$ would give the posterior $\gamma$ quantile as its Bayes estimator; that is, the estimator $\delta_\gamma(X)$ has $\PP(\theta < \delta_\gamma(X) \mid X) = \gamma$.





\end{enumerate}

\paragraph{Moral:} Using different loss functions often gives other appealing summaries of the posterior distribution besides the posterior mean. Using these loss functions as the prediction error in regression problems can also lead to appealing methods such as quantile regression.

\end{problem}


\begin{problem}[Motivation for the Jeffreys prior]
\label{prob:motivate-jeffreys}

Recall that the {\em Kullback--Leibler (KL) Divergence} from a density $p$ to an approximating density $q$ (on a sample space $\cX$ with respect to base measure $\mu$) is
\[
D_{\text{KL}}(p \,\|\, q) = \EE_p\left[\log\frac{p(X)}{q(X)}\right] = \int_\cX p(x)\log\frac{p(x)}{q(x)}\,d\mu(x),
\]
where the integrand is treated as zero wherever $p(x) = 0$. For a density $p$ and a family $\cP$, define the KL ball of radius $\ep$ around $p$ as
\[
B_\ep(p) = \{q\in\cP:\; D_{\text{KL}}(p\,\|\,q) < \ep\}
\]

Now, assume we have a family $\cP$ of positive densities $p_\theta$ parameterized by $\theta \in \Theta \subseteq \RR^d$, with enough regularity for us to freely take derivatives under the integral, and for those derivatives to be continuous. In particular, assume the Fisher information $J(\theta)$ is continuous and (strictly) positive definite, so $|J(\theta)|>0$.

In this problem you will show that the Jeffreys prior $\Lambda$ spreads its prior mass evenly on $\cP$, in the sense that for distinct values $\theta_1,\theta_2\in\Theta^{\circ}$ (the interior of $\Theta$), we have
\[
\lim_{\ep \to 0} \frac{\Lambda(B_\ep(p_{\theta_1}))}{\Lambda(B_\ep(p_{\theta_2}))} = 1.
\]
Here $\Lambda$ could refer either to the probability distribution with density proportional to $|J(\theta)|^{1/2}$ (if the Jeffreys prior is proper) or an infinite measure with density equal to $|J(\theta)|^{1/2}$. Note the slight abuse of notation where we use $\Lambda$ to refer both to the Jeffreys prior on $\Theta$ and to the corresponding prior on $\cP = \{p_{\theta}:\;\theta\in \Theta\}$.

To rule out perversities, assume also that for each $\theta$ there is some radius $r_\theta$ such that $d_\theta(\zeta) = D_{\text{KL}}(p_{\theta}\,\|\,p_{\zeta})$ is convex for $\|\zeta-\theta\| \leq r$, and $\inf_{\|\zeta - \theta\| > r} d_\theta(\zeta) > 0$ (in particular, this implies identifiability). That is, small KL balls in $\cP$ are not going to include $p_\zeta$ with faraway values of $\zeta$. You don't need to explicitly use this condition; you can just not pay attention to what's happening away from infinitesimal neighborhoods around $\theta_1$ and $\theta_2$ and assume everything is fine.

\begin{enumerate}[(a)]

\item For $\theta\in\Theta^{\circ}$, and any vector $a\in\RR^d$, show that 
  \begin{equation}
    \label{eq:lim-ratio}
\lim_{\ep\to 0} \frac{1}{\ep^2} D_{\text{KL}}(p_{\theta} \,\|\, p_{\theta + \ep a}) = \frac{1}{2}a'J(\theta)a,
\end{equation}
or equivalently $D_{\text{KL}}(p_{\theta} \,\|\, p_{\theta + a}) = \frac{1}{2}a'J(\theta)a + o(\|a\|^2)$.

{\bf Hint 1:} Try differentiating the KL divergence and recall that, for $f:\;\RR^d\to\RR$, where $x,a\in \RR^d$ and $\ep \in \RR$, we have
\begin{align*}
  \frac{d}{d\ep}f(x+\ep a) &= a'\nabla f(x+\ep a)\\
  \frac{d^2}{d\ep^2}f(x+\ep a) &= a'\nabla^2 f(x+\ep a)a
\end{align*}

{\bf Hint 2:} You are welcome to jump to the general case $d\geq 1$, but it may help to warm up with the case $d=1$, in which case the above boils down to
\[
\lim_{\ep\to 0} \frac{1}{\ep^2} D_{\text{KL}}(p_{\theta} \,\|\, p_{\theta + \ep a}) = \frac{a^2}{2}J(\theta)
\]
or more simply
\[
\lim_{\ep\to 0} \frac{1}{\ep^2} D_{\text{KL}}(p_{\theta} \,\|\, p_{\theta + \ep}) = \frac{1}{2}J(\theta)
\]





\item Next, show that for any $\theta\in\Theta^{\circ}$, we have

\[
\lim_{\ep\to 0}\frac{1}{\ep^{d/2}} \Lambda\left(\{\zeta:\; (\zeta - \theta)'J(\theta)(\zeta - \theta) < \ep\}\right) = \begin{cases} V_d/C_\Lambda &\text{if } \Lambda \text{ is proper and } \lambda(\theta) \propto_\theta |J(\theta)|^{1/2}\\ V_d &\text{if } \Lambda \text{ is improper and } \lambda(\theta) = |J(\theta)|^{1/2}\end{cases}.
\]
where $V_d = \int_{\RR^d}1\{\|u\|<1\} du$ is the volume of a unit ball in $\RR^d$ and $C_\Lambda = \int_{\Theta} |J(\theta)|^{1/2} \,d\theta$ is the normalizing constant for the Jeffreys prior, when it is finite.

{\bf Hint:} Try the change of variables $\zeta(a) = \theta + J(\theta)^{-1/2}a$.



\item Now, finish the argument by showing that we have
  \[
  \lim_{\ep\to 0} \frac{1}{\ep^{d/2}} \Lambda(B_{\ep/2}(p_\theta)) = \begin{cases} V_d/C_\Lambda &\text{if } \Lambda \text{ is proper and } \lambda(\theta) \propto_\theta |J(\theta)|^{1/2}\\ V_d &\text{if } \Lambda \text{ is improper and } \lambda(\theta) = |J(\theta)|^{1/2}\end{cases}.
  \]

  {\bf Hint:} Try to relate the sublevel sets of $f_\theta(\zeta) = D_{\text{KL}}(p_{\theta}\,\|\,p_\zeta)$ to sublevel sets of $\frac{1}{2}(\zeta-\theta)'J(\theta)(\zeta-\theta)$.



\end{enumerate}

\paragraph{Moral:} Whereas the flat prior spreads mass evenly over the parameter space, the Jeffreys prior spreads mass evenly over the statistical model itself, by assigning equal probability to small neighborhoods of equal ``KL radius.'' Since the KL divergence exists prior to the parametrization, the Jeffreys prior therefore implements the ``principle of indifference'' without reference to any parametrization of the model.

\end{problem}



\bibliography{biblio}
\bibliographystyle{plainnat}



\end{document}