em-vs-vi.tex

\frame{\frametitle{"Stochastizing" EM}
Recall from Liang and Klein (2009):
\begin{columns}
\begin{column}{.5\textwidth}
\begin{block}{}
\uncover<1-2>{
\footnotesize
\begin{enumerate}
\item[] $\mu = \text{initialization}$
\item[] for $t = 1, \ldots, T$:
\begin{enumerate}
\item[] for $i \in [n]$:
\begin{enumerate}
\item[] $s_i' = \sum_z p(z | x^{(i)}; \theta(\mu)) \phi(x^{(i)}, z)$
\item[] $\mu'_t \alt<2>{\alert{+=}}{+=} s_i'$
\end{enumerate}
\item[] $\mu = \mu'_t$
\end{enumerate}
\end{enumerate}
}
\end{block}
\end{column}
\begin{column}{.5\textwidth}
\begin{block}{}
\uncover<2>{
\footnotesize
\begin{enumerate}
\item[] $\mu = \text{initialization}$; $k = 0$
\item[] for $t = 1, \ldots, T$:
\begin{enumerate}
\item[] for $i \in [n]$ \alert{randomly}:
\begin{enumerate}
\item[] $s_i' = \sum_z p(z | x^{(i)}; \theta(\mu)) \phi(x^{(i)}, z)$
\item[] $\mu \alert{=} (1 - \rho_k) \mu + \rho_k s_i'$
\item[] $k += 1$
\end{enumerate}
\end{enumerate}
\end{enumerate}
}
\end{block}
\end{column}
\end{columns}
}

\frame{\frametitle{Stochastic Variational Inference: Main Idea}
Main points:
\begin{enumerate}
\item Sample, uniformly at random, a \textit{single} data point $x$
\item Assume $x$ was seen many ($N$) times
\item Form the \gvp{\text{\textit{initial} global}} updates
\item Interpolate these temporary updates with the current parameters
\end{enumerate}

\uncover<2->{
What about optimizing \lvp{\text{local}} variational parameters?
}
}

\frame{\frametitle{"Stochastizing" Variational Inference}
\begin{columns}
\begin{column}{.5\textwidth}
\begin{block}{}
\uncover<1-2>{
\footnotesize
\begin{enumerate}
\item[] $\gvp{\lambda^{(0)}}= \text{initialization}$
\item[] 
\item[] repeat:
\begin{enumerate}
\item[] for local $\lvp{\phi_{n,j}}$:
\begin{enumerate}
\item[] $\lvp{\phi_{n,j}},\lvp{\phi_{n,j}^{(t)}} = \mathbb{E}_{q^{(t-1)}}[\eta_{l,j}(x_n,z_{n\backslash j}, \beta)]$
\end{enumerate}
\item[] $\gvp{\lambda^{(t)}} = \mathbb{E}_{q^{(t)}}[\eta_g(z,x)]$
\end{enumerate}
\item[] until ELBO converges
\end{enumerate}
}
\end{block}
\end{column}
\begin{column}{.5\textwidth}
\begin{block}{}
\uncover<2>{
\footnotesize
\begin{enumerate}
\item[] $\gvp{\lambda^{(0)}}= \text{initialization}$
\item[] set step-size schedule, $\rho_t$
\item[] repeat:
\begin{itemize}
\item[] sample data point $x_i$ u.a.r.
\item $\lvp{\phi(\gvp{\lambda})} = \mathbb{E}_{\lambda^{(t-1)}}[\eta_g(x_n,z_{n\backslash j}, \beta)]$
\item $\gvp{\hat\lambda} = \mathbb{E}_{\lvp{\phi}} [ \eta_g(x_i^{(N)},z_i^{(N)})] $
\item $\gvp{\lambda^{(t)}} = (1-\rho_t)\lambda^{(t-1)} + \rho_t \hat\lambda$
\end{itemize}
\item[] forever
\end{enumerate}
}
\end{block}
\end{column}
\end{columns}
}