\documentclass{article}

\usepackage[margin=1.2in]{geometry}
\usepackage{graphicx}
\usepackage{amsmath,amssymb,amsthm,bm}
\usepackage{latexsym,color,minipage-marginpar,caption,multirow,verbatim}
\usepackage{enumerate}
\usepackage{times}

\newcommand{\cP}{\mathcal{P}}
\newcommand{\cX}{\mathcal{X}}
\newcommand{\cR}{\mathcal{R}}

\newcommand{\EE}{\mathbb{E}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\RR}{\mathbb{R}}
\newcommand{\ZZ}{\mathbb{Z}}

\newcommand{\Var}{\textnormal{Var}}
\newcommand{\MSE}{\textnormal{MSE}}


\newcommand{\simiid}{\overset{\text{i.i.d.}}{\sim}}
\newcommand{\ep}{\varepsilon}

\definecolor{darkblue}{rgb}{0.2, 0.2, 0.5}

\newenvironment{solution}{~\\\color{darkblue}{\bf Solution:~\\}}{}
\newenvironment{ifsol}{\color{darkblue}}{}

\theoremstyle{definition}
\newtheorem{problem}{Problem}


\begin{document}

\title{Stats 210A, Fall 2025\\
  Homework 2 \\
  {\large {\bf Due on}: Wednesday, Sep. 17}}
\date{}

\maketitle

\paragraph{Instructions:} You may disregard measure-theoretic niceties about conditioning on measure-zero sets, almost-sure equality vs. actual equality, ``all functions'' vs. ``all measurable functions,'' etc. (unless the problem is explicitly asking about such issues).


\begin{problem}[Uniform location-scale family]
\label{prob:uniflocscale}

Let $X_1, \ldots, X_n\simiid \text{Unif}[\mu-\sigma, \mu + \sigma]$, with $\mu\in\RR$ and $\sigma > 0$ unknown.
%
\begin{enumerate}[(a)]
\item Show that $T(X) = (X_{(1)}, X_{(n)})$ is minimal sufficient.



\item Suppose that we wish to estimate $\mu$ under the squared error loss.  The sample mean $\overline{X}$ may appear to be a reasonable estimator of $\mu$, but we might worry about the fact that it is not a function of $T(X)$.

Guided by the sufficiency principle, we could instead consider the estimator 
\[\delta^*(X) = \frac{X_{(1)} + X_{(n)}}{2}.\] 
Use Monte Carlo simulation to simulate the distribution of both estimators for $\mu = 0, \sigma = 1, n = 1000$. Report the root mean squared error (RMSE), defined as $\sqrt{\MSE}$ (as calculated by Monte Carlo).

For each estimator, plot a histogram of simulated estimates. You can either put both histograms on the same plot, or in side-by-side plots. If you do the latter, it might be more enlightening to use the same range for the horizontal and vertical axes in both plots, so the differences between the histograms are more evident. Include your code.




\item In the previous part, we only compared the two estimators for one setting of the parameters $\mu,\sigma$. Show that, for both estimators, the MSE for generic $(\mu,\sigma)$ does not depend on $\mu$, and is proportional to $\sigma^2$. That is, show that
\[
\MSE(\mu, \sigma; \delta) = \sigma^2 \MSE(0,1; \delta), \quad \text{for } \delta = \overline{X}, \delta^*.
\]
Complete the argument to conclude that $\overline{X}$ is inadmissible whenever $\MSE(0,1;\delta^*) < \MSE(0,1;\overline{X})$.

\item ({\bf Optional:} Not graded, no extra points) In this and the next optional parts, you will calculate the MSE analytically using some nice manipulations. 

If $B \sim \text{Beta}(\alpha,\beta)$ then its density is proportional to $x^{\alpha-1}(1-x)^{\beta-1}$ on $x\in [0,1]$.

If $U_1,\ldots,U_n \simiid U[0,1]$, show that
\[
U_{(n)} \sim \text{Beta}(n,1), \quad\text{ which has density } p(x) = n x^{n-1},
\]
and
\[
U_{(1)}/U_{(n)} \sim \text{Beta}(1,n-1) \quad\text{ which has density } p(x) = (n-1)(1-x)^{n-2},
\]
independently of $U_{(n)}$.

{\bf Hint:} For the first part, start by writing down the CDF of $U_{(n)}$.

{\bf Hint:} For the second part, you may use without proof the fact that, conditional on $U_{(n)} = u$, the remaining $n-1$ values are i.i.d. $\text{Unif}[0,u]$, then proceed similarly to what you did for the first part.





\item ({\bf Optional:} Not graded, no extra points) Compute the MSE of each estimator as a function of $n, \mu,$ and $\sigma$, and show that $\delta^*$ strictly dominates $\overline{X}$ for $n > 2$ (the estimators coincide for $n=2$). What happens to the ratio of their MSE's as $n\to\infty$?

{\bf Hint:} The results from the previous part should be useful. You may use without proof that $\text{Beta}(\alpha,\beta)$ has mean $\frac{\alpha}{\alpha+\beta}$ and variance $\frac{\alpha\beta}{(\alpha+\beta)^2(\alpha + \beta + 1)}$.





\end{enumerate}

\paragraph{Moral:} Understanding and respecting the statistical structure of a model sometimes helps us to come up with estimators that perform dramatically better than the estimator we would have na\"{i}vely thought of. Here is a case where applying the sufficiency principle helped us get a much better estimator than the sample mean.

\end{problem}





\begin{problem}[Convexity of $A(\eta)$ and $\Xi_1$]
\label{prob:convexity-expfam}
Let $\cP=\{p_\eta:\; \eta \in \Xi_1\}$ denote an $s$-parameter exponential family in canonical form
\[
p_\eta(x) = e^{\eta'T(x) - A(\eta)}h(x), \qquad A(\eta) = \log\int_{\cX} e^{\eta'T(x)}h(x)\,d \mu(x),
\]
where $\Xi_1=\{\eta:\; A(\eta) < \infty\}$ is the natural parameter space. 

Recall H\"{o}lder's inequality: if $q_1,q_2\geq 1$ with $q_1^{-1} + q_2^{-1} = 1$, and $f_1$ and $f_2$ are ($\mu$-measurable) functions from $\cX$ to $\RR$, then
\[
\|f_1f_2\|_{L^1(\mu)} \leq \|f_1\|_{L^{q_1}(\mu)}\|f_2\|_{L^{q_2}(\mu)}, \quad \text{ where } \|f\|_{L^{q}(\mu)} = \left(\int_{\cX} |f(x)|^q\,d \mu(x)\right)^{1/q}.
\]
{\bf Note:} $q_1=q_2=2$ reduces to Cauchy-Schwarz.


\begin{enumerate}[(a)]
\item Show that $A(\eta):\;\RR^s \to [0,\infty]$ is a convex function: that is, for {\em any} $\eta_1,\eta_2\in \RR^s$ (not just in $\Xi_1$), and $c\in [0,1]$ then
  \begin{equation}\label{eq:ineq}
  A(c\eta_1 + (1-c)\eta_2) \leq c A(\eta_1) + (1-c) A(\eta_2)
  \end{equation}
  {\bf Hint}: try $q_1=c^{-1}$, $f_1(x)^{1/c}=e^{\eta_1'T(x)}h(x)$.

  


\item Use the previous part to show that $\Xi_1\subseteq \RR^s$ is convex. 

  
  


\end{enumerate}

\paragraph{Moral:} The natural parameter space for any exponential family (meaning the set of all parameters $\eta$ that give normalizable densities) is a convex subset of $\RR^s$.


\end{problem}

\begin{problem}[Expectation of an increasing function]
\label{prob:expectation-increasing}
\begin{enumerate}[(a)]
\item Assume $X\sim P$ is a real-valued random variable. Show that if $f(x)$ and $g(x)$ are non-decreasing functions of $x$, then 
\[
\text{Cov}(f(X),g(X)) \geq 0
\]
{\bf Hint}: first show $\EE\left[(f(X_1)-f(X_2))(g(X_1)-g(X_2))\right] = 2\text{Cov}(f(X_1),g(X_1))$, where $X_1,X_2\simiid P$.



\item Let $p_\eta(x)$ be a one-parameter canonical exponential family with non-decreasing sufficient statistic $T(x)$, where $x\in\cX\subseteq \RR$:
\[
p_\eta(x) = e^{\eta T(x) - A(\eta)}h(x).
\]
Let $\psi(x)$ be any non-decreasing bounded function. Show that, for $\eta\in\Xi_1^\circ$, the interior of the natural parameter space, $\frac{d}{d \eta}\EE_{\eta}[\psi(X)] \geq 0$.

{\bf Hint}: find an expression for $\frac{d}{d \eta} \EE_{\eta}[\psi(X)]$ by using methods akin to the ones we used in class to derive the differential identities. You may assume it is justified to differentiate under the integral sign.



\item Conclude that $X$ is stochastically increasing in $\eta$; that is, show $\PP_\eta(X \leq c)$ is non-increasing in $\eta$, for every $c \in \RR$.



\end{enumerate}

\paragraph{Moral:} This exercise confirms something that we should intuitively expect to be true: that increasing the natural parameter $\eta$, which ``tilts'' the distribution toward larger values of $T(X)$, will also shift the distribution of $X$ to the right if $T$ is an increasing function. It also illustrates the usefulness of differential identities for understanding exponential families' structure.
\end{problem}


\begin{problem}[Mean parameterization of an exponential family]
\label{prob:mean-param}

  Consider the $s$-parameter exponential family $\cP = \{P_\eta:\; \eta \in \Xi\}$ on $\cX$ with densities $p_\eta(x) = e^{\eta'T(x) - A(\eta)}h(x)$ with respect to a common dominating measure $\nu$. Assume $\Xi=\Xi_1^{\circ}$, the interior of the full natural parameter space, and that $\text{Var}_\eta(a'T(X))>0$ for all $a \neq 0$ and $\eta\in \Xi$.

  Define the {\em mean parameter} 
  \[
  \mu(\eta) = \EE_\eta[T(X)].
  \]
  We will show that this is a one-to-one mapping, so $\cP$ can be alternatively be parameterized by $\mu(\eta)$ instead of $\eta$. The Bernoulli, Poisson, and exponential distributions are exponential families that are most often parameterized by their means, and parameterizations of other distributions like the normal and binomial are closely related to the mean parameterization.

  Throughout this problem, you may use without proof that if the variance of any statistic $S(X)$ is positive under one $P_\eta \in \cP$ then it is positive under all $P_\eta \in \cP$ (as an optional exercise, try to prove this).

  \begin{enumerate}[(a)]
    
  \item For $s=1$, show that $\eta \mapsto \EE_\eta[T(X)]$ is a one-to-one mapping; that is, show that if $\eta_1 \neq \eta_2$ then $\EE_{\eta_1}[T(X)] \neq \EE_{\eta_2}[T(X)]$.

    {\bf Hint:} You can use the differential identities.

    

  \item For $s>1$ and $\eta_1,\eta_2\in\Xi$, consider the subfamily whose parameter space is the line segment between $\eta_1$ and $\eta_2$. For $\theta \in [0,1]$, let
    \[
    \eta(\theta) = (1-\theta) \eta_1 + \theta \eta_2. 
    \]
    Show that this subfamily is a one-parameter exponential family on $\cX$ with natural parameter $\theta$, and write it in standard exponential family form.

    

  \item Combine (a) and (b) to show that $\eta \mapsto \EE_\eta[T(X)]$ is a one-to-one mapping for $s \geq 1$.

    


  \end{enumerate}

\paragraph{Moral:} We can always parameterize an exponential family by the mean of $T(X)$. This parameterization is very often more intuitive than the natural parameterization, and it gives us the standard parameterization for families like the Poisson (with sufficient statistic $T(X)=X$) and binomial (with sufficient statistic $T(X)=X/n$).
\end{problem}




\end{document}