\documentclass{article}

\usepackage[margin=1.2in]{geometry}
\usepackage{graphicx}
\usepackage{amsmath,amssymb,amsthm,bm}
\usepackage{latexsym,color,minipage-marginpar,caption,multirow,verbatim}
\usepackage{enumerate}
\usepackage[round]{natbib}
\usepackage{times}
\usepackage{booktabs}

\newcommand{\cP}{\mathcal{P}}
\newcommand{\cX}{\mathcal{X}}
\newcommand{\cR}{\mathcal{R}}
\newcommand{\cT}{\mathcal{T}}
\newcommand{\cY}{\mathcal{Y}}
\newcommand{\cQ}{\mathcal{Q}}

\newcommand{\EE}{\mathbb{E}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\RR}{\mathbb{R}}
\newcommand{\ZZ}{\mathbb{Z}}

\newcommand{\Var}{\textnormal{Var}}
\newcommand{\MSE}{\textnormal{MSE}}
\newcommand{\toProb}{\overset{p}{\to}}
\newcommand{\toPtheta}{\overset{{P_\theta}}{\to}}


\DeclareMathOperator*{\argmax}{argmax}


\newcommand{\simiid}{\overset{\text{i.i.d.}}{\sim}}
\newcommand{\simind}{\overset{\text{ind.}}{\sim}}
\newcommand{\ep}{\varepsilon}

\definecolor{darkblue}{rgb}{0.2, 0.2, 0.5}

\newenvironment{solution}{~\\\color{darkblue}{\bf Solution:~\\}}{}
\newenvironment{ifsol}{\color{darkblue}}{}

\newcommand{\optional}{{\bf Optional:} (Not graded, no extra points) }

\theoremstyle{definition}
\newtheorem{problem}{Problem}


\begin{document}

\title{Stats 210A, Fall 2025\\
  Homework 11 \\
  {\large {\bf Due on}: Wednesday, Nov. 19}}
\date{}

\maketitle

\paragraph{Instructions:} See the standing homework instructions on the course web page


\begin{problem}[Precision-weighted average]
\label{prob:prec-weights}

 Suppose that we observe two independent samples $X_1,\ldots,X_n\simiid (\mu, \sigma^2)$ and $Y_1,\ldots,Y_m \simiid (\mu, \tau^2)$, with $n,m> 1$. The notation means that the expectation of a single $X_i$ or $Y_i$ is $\mu\in \RR$, and the variance is $\sigma^2>0$ for a single $X_i$ and $\tau^2>0$ for a single $Y_i$. All three parameters are unknown, but we are primarily interested in estimating the common expectation $\mu$.

A natural estimator is to take a convex combination of the sample averages:
\[
\delta_\gamma(X,Y) = \gamma \overline X + (1-\gamma) \overline Y,
\]
for $\gamma \in [0,1]$.

\begin{enumerate}[(a)]

\item Show that the optimal (variance-minimizing) choice of $\gamma$ is
\[
\gamma^* = \frac{n\sigma^{-2}}{n\sigma^{-2}+m\tau^{-2}} = \frac{1}{1+\rho m/n},
\]
where $\rho = \sigma^2/\tau^2$. $\delta_{\gamma^*}$ is called the {\em precision-weighted average} because $n\sigma^{-2}$ and $m\tau^{-2}$ are the precisions (inverse variances) of $\overline{X}$ and $\overline{Y}$, respectively. Give the variance of $\delta_{\gamma^*}(X,Y)$.



\item Since $\sigma^2$ and $\tau^2$ are unknown, we must estimate them. Let $S_X^2$ and $S_Y^2$ denote the usual sample variances for the two samples. Show that $\hat\rho = S_X^2/S_Y^2$ is a consistent estimator for $\rho$ as $m,n \to \infty$.

{\bf Hint:} It may help to recall the identity $(n-1)S_X^2 = \sum_i X_i^2 - n\overline{X}^2$.

{\bf Note:} If you are wondering what it means for both $m$ and $n$ to go to $\infty$, you may assume that we have a sequence of problems indexed by $k=1,2,\ldots$ and $\min \{m_k,n_k\} \to \infty$ as $k\to\infty$. You should feel free to work more informally than this.



\item Let $\hat\gamma = 1/(1+\hat\rho m/n)$ and assume that $m,n\to\infty$ with $m/n \to c \in (0,\infty)$. Show that the adaptive estimator
\[
\delta_{\hat\gamma}(X,Y) = \hat\gamma \overline X + (1-\hat\gamma) \overline Y
\]
has an asymptotic normal distribution as $n,m\to \infty$, and give its asymptotic distribution after appropriately centering and scaling it. Compare the asymptotic distribution of the adaptive estimator $\delta_{\hat\gamma}(X,Y)$ to the asymptotic distribution of the oracle estimator $\delta_{\gamma^*}(X,Y)$.

{\bf Hint:} Start by considering the asymptotic distribution of $(\overline X, \overline Y)$. You may use without proof the result that if $Z_n \Rightarrow P$ and $W_n \Rightarrow Q$, and $Z_n$ and $W_n$ are independent for each $n$, then $(Z_n,W_n) \to P \times Q$ (meaning the product measure between the distributions $P$ and $Q$).

{\bf Note:} Again, if we want to set up a formal sequence of problems in which the distribution converges, we could assume the ratio $c_k = m_k/n_k$ is converging to $c \in (0,\infty)$, in addition to our previous assumption that $\min \{m_k,n_k\}\to\infty$. As before, you can also work more informally.



\end{enumerate}

\paragraph{Moral:} This is an example of an estimator that's adaptive to an unknown problem parameter, in this case the ratio $\rho = \sigma^2/\tau^2$ which tells us how to choose the optimal value of some tuning parameter $\gamma^*(\rho,m,n)$. A tried and true method in statistics is to get a consistent estimator for the unknown parameter and just plug it in. Asymptotically, this very often works just as well as knowing the value of the nuisance parameter (it's not necessarily that we don't pay a price for not knowing $\rho$, it's just that the price we pay might be lower order than the source of error that matters).
\end{problem}


\begin{problem}[Some Maximum Likelihood Estimators]
\label{prob:mle-examples}

Find the MLE for each model below, and find its asymptotic distribution. You do not need to check the conditions for convergence theorems; just calculate assuming they are in force.

\begin{enumerate}[(a)]
\item Laplace: $X_1,\ldots,X_n \simiid \frac{1}{2}e^{-|x-\theta|}$.

  {\bf Note:} although the log-likelihood is non-differentiable at one point, we can still use the Fisher information as defined by $J_1(\theta) = \Var_\theta[\dot{\ell}_1(\theta;X_i)]$ to get the asymptotic distribution; you may assume this without proof. You may assume $n$ is odd.



\item Binomial: $X_1,\ldots,X_n \simiid \text{Binom}(m,\theta)$. Find the MLE for $\theta$ and for the canonical parameter $\eta = \log\frac{\theta}{1-\theta}$.



\item Gaussian: $X_1,\ldots,X_n \simiid N(\theta,\sigma^2)$. Find (i) the MLE for $\theta$ if $\sigma^2$ is known, (ii) the MLE for $\sigma^2$ if $\theta$ is known, and (iii) the MLE for $(\theta,\sigma^2)$ if neither is known.




\end{enumerate}

\paragraph{Moral:} Our general formula for the asymptotic distribution of the MLE allows us to quickly get good asymptotic approximations to their distributions.

\end{problem}



\begin{problem}[Limiting distribution of $U$-statistics]
\label{prob:u-statistics}

Suppose $X_{1}, \ldots, X_{n} \simiid P$ in some sample space $\cX$. $U_{n} = U_{n}(X_{1}, \ldots, X_{n})$ is called a rank-2 $U$-statistic if
\[U_{n} = \frac{1}{n(n - 1)}\sum_{i=1}^{n}\sum_{j\neq i}h(X_{i}, X_{j})\]
where $h$ is a symmetric function, i.e. $h(x_{1}, x_{2}) = h(x_{2}, x_{1})$ for any $x_{1}, x_{2}\in\cX$.

In this problem, we denote $\theta = \EE h(X_{1}, X_{2})$ and assume that $\EE h(X_{1}, X_{2})^{2} < \infty$. Note that $U_n$ is the nonparametric UMVU estimator of $\theta$.

Perhaps surprisingly, we can derive the asymptotic distribution of $U_n$ in a relatively small number of steps using a technique called {\em H\'{a}jek projection} where we approximate it by an additive function of the independent $X_i$ variables. We walk through the proof below.

\begin{enumerate}[(a)]
\item Define $g(x) = \EE h(x, X_2) - \theta = \int h(x,u)\,d P(u) - \theta$. Show that, for all $i$,
\[
\EE g(X_i) = 0, \quad \text{ and } \;\;\Var(g(X_i)) < \infty.
\]
({\bf Note:} $g$ is a specific function from $\cX$ to $\RR$. It is not a rule for naively substituting symbols into expressions. In particular, note that $g(X_i)$, a random variable, is not the same as the deterministic expression $\EE h(X_i, X_2)-\theta$.)




\item Define $\widehat{U}_{n} = \theta + \frac{2}{n}\sum_{i=1}^{n}g(X_{i})$. Show that $\EE[(U_n-\widehat{U}_n)f(X_i)]=0$ for any $i$ and any measurable function $f(X_i)$ with $\EE[f(X_i)^2] < \infty$.

({\bf Hint:} Condition on $X_i$)



\item Show that ${\sqrt{n}(U_{n} - \widehat{U}_{n})\toProb 0}$ as $n\to\infty$. (Hint: show that $U_n$ and $\widehat{U}_n$ have the same asymptotic variance, and then apply part (b)).

  



\item Conclude that $\sqrt{n}(U_{n} - \theta)\Rightarrow N(0, 4\zeta_{1})$, where $\zeta_{1} = \Var(g(X_{1}))$.




\item Assume that $\cX = \RR$ with $\EE X_i^4 <\infty$. Express the sample variance $S_{n}^{2} = \frac{1}{n-1}\sum_{i=1}^{n}(X_{i} - \overline{X})^{2}$ as a rank-2 U-statistic and use the above results to derive its asymptotic distribution.


\end{enumerate}

({\bf Note:} a similar result holds in general for rank-$r$ $U$-statistics if we set $\widehat{U}_n= \theta + \frac{r}{n}\sum_i g(X_i)$ where ${g(x) = \EE[h(x,X_2,\ldots,X_r)]-\theta}$. )

\paragraph{Moral:} If $P^n$ is the distribution of $(X_1,\ldots,X_n)$ then it is easy to check that the set of all square-integrable random variables of the form $f(X_1,\ldots,X_n)$ (where $f:\; \cX^n \to \RR$ is measurable) forms a vector space over $\RR$, which we call $L^2(P^n)$, where we can define an inner product as
\[
\langle f(X), g(X) \rangle_{L^2} = \EE[ f(X)g(X)] \leq \sqrt{\EE[f(X)^2] \EE[g(X)^2]} < \infty.
\]

Moreover, the subset of those random variables that can be written as $\sum_i f_i(X_i)$, where each $f_i$ is measurable, forms a subspace. Part (b) establishes that the simpler random variable $\widehat{U}_n$ is the {\em projection} of $U_n$ onto this subspace, and part (c) establishes that $U_n$ is asymptotically very close to its projection.
\end{problem}


\begin{problem}[Super-Efficient Estimator]
\label{prob:super-efficiency}

Let $X_1,\ldots,X_n \simiid N(\theta,1)$ and consider estimating $\theta$ via:
\[
\delta_n(X) = \overline{X}_n 1\{|\overline{X}_n| > a_n\},
\]
where $a_n \to 0$ but $a_n\sqrt{n} \to \infty$ as $n \to \infty$ (for example, $a_n = n^{-1/4}$).

\begin{enumerate}[(a)]
\item Show that $\delta_n$ has the same asymptotic distribution as $\overline X_n$ when $\theta \neq 0$, but that $\sqrt{n}(\delta_n-0)\toProb 0$ if $\theta = 0$.



\item Show that, pointwise in $\theta$, as $n\to\infty$,
\[
n\,\text{MSE}(\delta_n;\theta) \to 1\{\theta\neq 0\},
\]
but that the convergence is not uniform in $\theta$; in fact,
\[
\sup_{\theta\in\RR}\;\; n\,\text{MSE}(\delta_n;\theta) \rightarrow \infty.
\]
({\bf Note}: this is an example of a situation where it is incorrect to exchange a limit with a supremum.)



\end{enumerate}

\paragraph{Moral:} The sense in which asymptotically efficient estimators are ``optimal'' is not easy to define, and it isn't obvious how we should compare the asymptotic behavior of different estimators. In this example it would appear initially that the super-efficient estimator renders the sample mean inadmissible. But this is only true if we look at the pointwise limit for fixed $\theta$; at any $n$ there are some values of $\theta$ for which the estimator is performing very badly, and this gets worse and worse as $n$ gets larger.

\end{problem}



\bibliography{biblio}
\bibliographystyle{plainnat}



\end{document}