\documentclass{beamer}

%\usepackage[table]{xcolor}
\mode<presentation> {
  \usetheme{Boadilla}
%  \usetheme{Pittsburgh}
%\usefonttheme[2]{sans}
\renewcommand{\familydefault}{cmss}
%\usepackage{lmodern}
%\usepackage[T1]{fontenc}
%\usepackage{palatino}
%\usepackage{cmbright}
  \setbeamercovered{transparent}
\useinnertheme{rectangles}
}
%\usepackage{normalem}{ulem}
%\usepackage{colortbl, textcomp}
\setbeamercolor{normal text}{fg=black}
\setbeamercolor{structure}{fg= black}
\definecolor{trial}{cmyk}{1,0,0, 0}
\definecolor{trial2}{cmyk}{0.00,0,1, 0}
\definecolor{darkgreen}{rgb}{0,.4, 0.1}
\usepackage{array}
\beamertemplatesolidbackgroundcolor{white}  \setbeamercolor{alerted
text}{fg=red}

\setbeamertemplate{caption}[numbered]\newcounter{mylastframe}

%\usepackage{color}
\usepackage{tikz}
\usetikzlibrary{arrows}
\usepackage{colortbl}
%\usepackage[usenames, dvipsnames]{color}
%\setbeamertemplate{caption}[numbered]\newcounter{mylastframe}c
%\newcolumntype{Y}{\columncolor[cmyk]{0, 0, 1, 0}\raggedright}
%\newcolumntype{C}{\columncolor[cmyk]{1, 0, 0, 0}\raggedright}
%\newcolumntype{G}{\columncolor[rgb]{0, 1, 0}\raggedright}
%\newcolumntype{R}{\columncolor[rgb]{1, 0, 0}\raggedright}

%\begin{beamerboxesrounded}[upper=uppercol,lower=lowercol,shadow=true]{Block}
%$A = B$.
%\end{beamerboxesrounded}}
\renewcommand{\familydefault}{cmss}
%\usepackage[all]{xy}

\usepackage{tikz}
\usepackage{lipsum}

 \newenvironment{changemargin}[3]{%
 \begin{list}{}{%
 \setlength{\topsep}{0pt}%
 \setlength{\leftmargin}{#1}%
 \setlength{\rightmargin}{#2}%
 \setlength{\topmargin}{#3}%
 \setlength{\listparindent}{\parindent}%
 \setlength{\itemindent}{\parindent}%
 \setlength{\parsep}{\parskip}%
 }%
\item[]}{\end{list}}
\usetikzlibrary{arrows}
%\usepackage{palatino}
%\usepackage{eulervm}
\usecolortheme{lily}
%\usepackage[latin1]{inputenc}
\title[Text as Data] % (optional, nur bei langen Titeln nötig)
{Text as Data}

\author{Justin Grimmer}
\institute[Stanford University]{Associate Professor\\Department of Political Science \\  Stanford University}
\vspace{0.3in}


\date{November 18th, 2014}%[Big Data Workshop] 
%\date{\today}



\begin{document}
\begin{frame}
\titlepage
\end{frame}


\begin{frame}
\frametitle{Final Project}

Poster Session: 12/4 \pause \\
\begin{itemize}
\invisible<1>{\item[-] Make posters in \LaTeX\ or related software for asthetics} \pause 
\invisible<1-2>{\item[-] Landscape orientation is preferable} \pause 
\invisible<1-3>{\item[-] \alert{Less is More}} \pause 
\end{itemize}
\invisible<1-4>{Papers Due: 12/12, 5pm {\huge \alert{This is a hard deadline}!}} \pause 
\begin{itemize}
\invisible<1-5>{\item[-] An academic paper$\leadsto$ appropriate for your field} \pause 
\begin{itemize}
\invisible<1-6>{\item[-] Succinct (not cute) title} \pause 
\invisible<1-7>{\item[-] Abstract that explains your contribution} \pause 
\invisible<1-8>{\item[-] Introduction that explains why your paper exists} \pause 
\invisible<1-9>{\item[-] Presentation of Results} \pause 
\invisible<1-10>{\item[-] Avoid: Long/breezy lit reviews that merely list previous scholarship} 
\end{itemize}

\end{itemize}


\end{frame}

\begin{frame}
\frametitle{Supervised Learning: Ensemble Learning}
\begin{itemize}
\item[1)] Task \pause 
\begin{itemize}
\invisible<1>{\item[-] Subsidize hand coding$\leadsto$ learn a rule between labels and features} \pause 
\end{itemize}
\invisible<1-2>{\item[2)] Objective function } \pause 
\begin{eqnarray}
\invisible<1-3>{\text{Pr}(Y_{i} = C_{k} | \boldsymbol{x}_{i} ) & = & f(\boldsymbol{x}_{i}, \boldsymbol{\beta}, \boldsymbol{\lambda}) \nonumber } \pause 
\end{eqnarray}
\begin{itemize}
\invisible<1-4>{\item[-] ``Coefficients" $\boldsymbol{\beta}$} \pause 
\invisible<1-5>{\item[-] Tuning parameters $\boldsymbol{\lambda}$} \pause 
\invisible<1-6>{\item[-] Functional form $f$} \pause 
\invisible<1-7>{\item[-] \alert{Necessarily requires consequential assumptions}} \pause 
\invisible<1-8>{\item[-] Ensembles: aggregate classifiers to increase performance} \pause 
\end{itemize}
\invisible<1-9>{\item[3)] Optimization} \pause 
\begin{itemize}
\invisible<1-10>{\item[-] Optimization of individual classifiers$\leadsto$ methods already discussed} \pause 
\invisible<1-11>{\item[-] Determination of weights on models$\leadsto$ equal (bagging), out of sample performance via cross validation (super learning)} \pause 
\end{itemize}
\invisible<1-12>{\item[4)] Validation} \pause 
\begin{itemize}
\invisible<1-13>{\item[-] Out of sample predictive performance} 
\end{itemize}
\end{itemize}


\end{frame}





\begin{frame}
\frametitle{Ensemble Learning: Intuition} 

\alert{Heuristic} (upon which we'll improve):\pause\invisible<1>{ if classifiers are \alert{accurate} and \alert{diverse}$\rightarrow$ ensemble methods improve} \pause \\
\invisible<1-2>{\alert{Intuition}: } \pause 
\begin{itemize} 
\invisible<1-3>{\item[-] Classify documents into two categories (Category 1, Category 2). }  \pause 
\invisible<1-4>{\item[-] True labels: evenly distributed across two categories}  \pause 
\invisible<1-5>{\item[-] Three classifiers with $75\%$ accuracy, but independent  }\pause 
\invisible<1-6>{\item[-] Implement majority voting rule  }\pause 
\end{itemize}
\begin{eqnarray}
\invisible<1-7>{\text{Pr(Correct Guess}| \text{Votes} )} \pause \invisible<1-8>{ & = & \text{Pr(3 correct)} + \text{Pr(2 correct)} } \pause \nonumber \\
 \invisible<1-9>{& = & 0.75^3 + 3 \times (0.75^2 \times 0.25)} \pause  \nonumber \\
  \invisible<1-10>{& = &  0.844 \nonumber }
  \end{eqnarray}


\end{frame}


\begin{frame}
\frametitle{Ensemble Learning: Intuition} 



\only<1>{\scalebox{0.45}{\includegraphics{Ensemble1.pdf}}} 






\end{frame}


\begin{frame}
\frametitle{Ensemble Learning: Intuition} 

\alert{Diverse} and \alert{Accurate} matter.   

\only<1>{\scalebox{0.45}{\includegraphics{Ensemble2.pdf}}} 
\only<2>{\scalebox{0.45}{\includegraphics{Ensemble3.pdf}}} 


\end{frame}



\begin{frame}
\frametitle{Wisdom of the Crowds:}

Goal: estimate a document's category$\leadsto$ $Y \in \{0, 1\}$ \pause \\
\invisible<1>{Classifiers: (suppose) a sequence of identically distributed (\alert{not independent}) random variables. } \pause  \\
\invisible<1-2>{Suppose $Y = 1$} \pause \\
\invisible<1-3>{Guess from classifer $m$ is $B_{m}$ with Pr$(B_{i} = 1) = p>0.5$.  \\} \pause 
\begin{eqnarray}
\invisible<1-4>{\bar{B} & = & \sum_{m=1}^{M} \frac{B_{m}}{M} \nonumber } \pause 
\end{eqnarray}

\invisible<1-5>{\alert{Wisdom of crows} (Condorcet Jury Theorem)} \pause 
\begin{eqnarray}
\invisible<1-6>{\lim_{M\rightarrow \infty} P(\bar{B}> 0.5) & = & 1 \nonumber } 
\end{eqnarray}

\end{frame}


\begin{frame}
\frametitle{Wisdom of the Crowds}

Suppose $B_{m}$ have variance $\sigma^2$ and pairwise correlation $\rho$. \pause   \\
\invisible<1>{Then, } \pause 

\begin{eqnarray}
\invisible<1-2>{\text{var}(\bar{B}) }\pause\invisible<1-3>{& = & \text{var}\left(\sum_{i=1}^{M} \frac{B_{i}}{M}  \right)} \pause  \nonumber \\
 \invisible<1-4>{& = & \frac{1}{M^{2}} \sum_{i=1}^{M} \text{var}\left(B_{i}\right) + \frac{2}{M^2} \sum_{i<j} \text{cov}(B_{i}, B_{j}) \nonumber } \pause\\
 \invisible<1-5>{& = & \frac{M \sigma^2}{M^2}  + \frac{2}{M^2} \rho \sigma^2 {{M}\choose{2}} \nonumber } \pause\\
\invisible<1-6>{& = & \underbrace{\rho \sigma^2}_{\text{Resolve with independence}} + \underbrace{\frac{1 - \rho}{M} \sigma^2}_{\text{Resolve with $\uparrow$classifiers}} \nonumber } 
\end{eqnarray}


\end{frame}


\begin{frame}
\frametitle{\alert{Bagging}: bootstrap aggregation}

\begin{small}
Creating Weak Classifiers with resampling: \pause 
\begin{itemize}
\invisible<1>{\item[-] Suppose we have labels $\boldsymbol{Y}$ and document term matrix $\boldsymbol{X}$} \pause
\invisible<1-2>{\item[-] For each bootstrap step $m$, $(m = 1,2, \hdots, M)$ draw $N$ observations with replacement, $\tilde{\boldsymbol{Y}}_{m}$, $\tilde{\boldsymbol{X}}_{m}$.  } \pause 
\invisible<1-3>{\item[-] Train classifier on bootstrapped data, } \pause 
\begin{eqnarray}
\invisible<1-4>{\tilde{\boldsymbol{Y}}_{m} & = & f^{m}(\tilde{\boldsymbol{X}}_{m}, \widehat{\boldsymbol{\beta}}, \boldsymbol{\lambda} ) \nonumber } \pause 
\end{eqnarray}
\invisible<1-5>{\item[-] Aggregating across classifiers, } \pause 
\begin{eqnarray}
\invisible<1-6>{f_{\text{bag}}(\boldsymbol{x}_{i}) & = & \frac{1}{M} \sum_{m=1}^{M} f^{m}(\tilde{\boldsymbol{X}}_{m}, \widehat{\boldsymbol{\beta}}, \boldsymbol{\lambda}) \nonumber } \pause 
\end{eqnarray}
\invisible<1-7>{\item[-] Only leads to a difference in estimate if classifiers are non-linear. } \pause 
\invisible<1-8>{\item[-] Strong Correlation between classifiers}
\end{itemize}
\end{small}
\end{frame}


\begin{frame}
\frametitle{Classification and Regression Trees (CART): Intuition}

Consider regression $E[Y|\boldsymbol{x}_{i}]$. \pause  \\
\invisible<1>{With no assumptions, \alert{stratify}$\leadsto$ different mean for unique values of $\boldsymbol{x}_{i}$\\} \pause
\begin{itemize}
\invisible<1-2>{\item[-] Within each strata $p$, compute average $Y$} \pause 
\begin{eqnarray}
\invisible<1-3>{\bar{Y}|\boldsymbol{x}_{p} & = & \sum_{i=1}^{N} \frac{I(\boldsymbol{x}_{i} = \boldsymbol{x}_{p})Y_{i}}{\sum_{t=1}^{N} I(\boldsymbol{x}_{t} = \boldsymbol{x}_{p}) } \nonumber } \pause 
\end{eqnarray}
\end{itemize}

\invisible<1-4>{Implies that for test data we would fit:} \pause 


\begin{eqnarray}
\invisible<1-5>{\hat{f}(\boldsymbol{x}_{i}) & = & \sum_{p=1}^{P} \bar{Y}|\boldsymbol{x}_{p}  I(\boldsymbol{x}_{i} = \boldsymbol{x}_{p}) \nonumber }\pause\\
\invisible<1-6>{& = & \sum_{p=1}^{P} c_{p}  I(\boldsymbol{x}_{i} = \boldsymbol{x}_{p}) \nonumber }\pause 
\end{eqnarray}

\invisible<1-7>{Curse of dimensionality(!!!)\\} \pause 
\invisible<1-8>{Approximate with \alert{regions}$\leadsto$ search for splits of data to approximate stratification} 







\end{frame}




\begin{frame}
\frametitle{Classification and Regression Trees (CART): Objective function}

Labels $\boldsymbol{Y}_{i}$ and documents $\boldsymbol{x}_{i}$

\begin{eqnarray}
E[Y| \boldsymbol{x}_{i}] & = & \widehat{f}(\boldsymbol{x}_{i}) \nonumber \\
& = & \sum_{p=1}^{P} c_{p} I (\boldsymbol{x}_{i} \in R_{p}) \nonumber 
\end{eqnarray}

where:
\begin{itemize}
\item[-] $R_{p}$ describes a \alert{region} $\leadsto$ node
\item[-] $c_{p}$ describes values of $Y_{i}$ for document in $R_{p}$
\end{itemize}


\end{frame}


\begin{frame}
\frametitle{Classification and Regression Trees (CART): Optimization function}

Suppose we want to minimize sum of squared residuals with each \alert{node}\\ \pause 

\invisible<1>{Then $c_{p} = $ Average $Y$ for documents assigned to $R_{p}$ \\} \pause 
\begin{eqnarray}
\invisible<1-2>{\widehat{c}_{p} & =&  \sum_{i=1}^{N} \frac{Y_{i} I(\boldsymbol{x}_{i} \in R_{p} )  }{\sum_{j=1}^{N} I(\boldsymbol{x}_{j} \in R_{p} )  } \nonumber } \pause
\end{eqnarray}

\invisible<1-3>{Determining an optimal partition$\leadsto$ NP-Hard.  \\} \pause 
\invisible<1-4>{Suppose we are in some node (perhaps at the start).  \\} \pause 
\invisible<1-5>{Greedy algorithm:} \pause 
\begin{footnotesize}
\begin{eqnarray}
\invisible<1-6>{(j^{*}, s^{*})  & = & \text{arg min}_{j, s} \left[ \underbrace{\text{min}_{c_{1}} \sum_{i=1}^{N}I(x_{ij}< s)(Y_{i} - c_{1})^2}_{\text{``cost" group 1}}  + \underbrace{\text{min}_{c_{2}} \sum_{i=1}^{N}I(x_{ij}> s)(Y_{i} - c_{2})^2}_{\text{``cost" group 2}}   \right] \nonumber } 
\end{eqnarray}
\end{footnotesize}


\end{frame}


\begin{frame}
\frametitle{Classification and Regression Trees (CART): Algorithm}

\begin{itemize}
\item[-] Start in Node
\item[-] Partition according to Greedy algorithm
\item[-] Continue until some stopping rule: number of observations per node
\end{itemize}

\end{frame}


\begin{frame}
\frametitle{CART Picture (Spirling 2008)}

\scalebox{0.7}{\includegraphics{CART_Example.png}}

\end{frame}


\begin{frame}
\frametitle{Forests and Trees}

Recall: accurate (unbiased) and uncorrelated classifiers \pause 
\begin{itemize}
\invisible<1>{\item[-] Grow trees deeply$\leadsto$ unbiased classifers, though high variance} \pause
\invisible<1-2>{\item[-] \alert{Average}$\leadsto$ reduces variance, but will be correlated} \pause 
\invisible<1-3>{\item[-] Random forest$\leadsto$ introduce additional sampling to induce independence$\leadsto$ Only split on subset of variables} 
\end{itemize}


\end{frame}


\begin{frame}
\frametitle{Random Forest Algorithm (ESL, 588)}

\pause 
\begin{itemize}
\invisible<1>{\item[1)] For $m$ bootstrap samples $(m = 1,\hdots, M)$, draw $N$ observations with replacement, $\tilde{\boldsymbol{Y}_{m}}, \tilde{\boldsymbol{X}_{m}}$} \pause 
\invisible<1-2>{\item[2)] Until a minimum node size is reached:} \pause 
\begin{itemize}
\invisible<1-3>{\item[i)] \alert{Select $z$ of the $J$ variables}$\leadsto$ introduces independences across the trees} \pause 
\invisible<1-4>{\item[ii)] Among those $z$, select the best split node} \pause 
\invisible<1-5>{\item[iii)] Split into daughter nodes} \pause 
\end{itemize}
\invisible<1-6>{\item[3)] The result is an ensemble (forest) of trees $\boldsymbol{T} = (T_{1}, T_{2}, \hdots, T_{M})$, } \pause 
\begin{eqnarray}
\invisible<1-7>{\hat{f}(\boldsymbol{x}_{i}) & = & \frac{1}{M} \sum_{m=1}^{M} T_{m} (\boldsymbol{x}_{i}) \nonumber } \pause 
\end{eqnarray}
\end{itemize}

\invisible<1-8>{{\tt RandomForest}$\leadsto$ Not a silver bullet!} \pause 
\begin{itemize}
\invisible<1-9>{\item[-] With many poor predictors$\leadsto$ the $p$ selected may be meaningless} 
\end{itemize}


\end{frame}









% \begin{frame}
% \frametitle{Common Ensemble Methods} 

% \alert{Boosting}: sequential training of weak classifiers
% \begin{itemize}
% \item[-] Method for combining several \alert{weak} classifiers
% \item[-] Basic idea: 
% \begin{itemize}
% \item[-] Model 1: classify initially based on all data (equal weight)
% \item[-] Model 2: classify all data, more weight to incorrectly classified data
% \item[-] Model 3: classify all data, more weight to incorrectly classified data
% \item[] $\hdots $
% \item[-] Model M: classify all data, more weight to incorrectly classified data
% \end{itemize}
% \item[-] Aggregate using weighted committee 
% \end{itemize}



% \end{frame}






\begin{frame}
\frametitle{Super Learning}


\begin{itemize}
\item[1) ] Set of hand labeled documents.  For each $i$, $(i=1, \hdots, N_{\text{train}})$
\begin{itemize}
\invisible<1>{\item[] $Y_{i, \text{train} } \in \{C_{1} C_{2}, \hdots C_{K} \}$}
\end{itemize}
\invisible<1-2>{\item[2)] Estimate relationship between labels and words } 
\begin{itemize}
\invisible<1-3>{\item[-] Each document $i$  is a \alert{count vector} of $K$ words} 
\invisible<1-4>{\item[] $ \textbf{x}_{i, \text{train}}  =  (X_{i1}, X_{i2}, \hdots, X_{iK}  )$ } \nonumber 
\end{itemize}
\end{itemize}

\only<1->{
\begin{eqnarray}
\invisible<1-5>{\text{Pr}(Y_{i} = C_{k} | \textbf{x}_{i} )\invisible<1-6>{_{\text{train}}}\invisible<1-7>{& = & \widehat{g} (\textbf{x}_{i} )_{\text{train}} } \nonumber }
\end{eqnarray}
} 
%\only<7->{
%\begin{eqnarray}
%\text{Pr}(Y_{i}  = \text{Credit} | \textbf{w}_{i} )_{\text{train}} \invisible<1-7>{& = & \widehat{g} (\textbf{w}_{i} )_{\text{train}} } %\nonumber 
%\end{eqnarray}
%}

\begin{itemize}
\invisible<1-7>{\item[-] Identify systematic relationship between words, labels}  \invisible<1-8>{$\leadsto$ Data and \alert{assumptions}}
\begin{itemize}
\invisible<1-9>{\item[-] LASSO (Tibshirani 1996): \alert{sparsity} }
\invisible<1-10>{\item[-] KRLS (Hainmueller and Hazlett 2013): \alert{dense}, flexible surface}
\invisible<1-11>{\item[-] Ridge, Elastic-Net, SVM, Random Forests, BART, ...}
\end{itemize}
\invisible<1-12>{\item[-] Which model?  Difficult to know before hand}
\invisible<1-13>{\item[-] Assess out of sample performance with \alert{cross validation}}
\end{itemize}
\invisible<1-14>{\alert{Weighted ensemble}: weights determined by (unique) out of sample predictive performance}

\pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause 
\end{frame}


\begin{frame}


Committee Methods:\\

Fit many methods, average with equal weights 

\begin{itemize}
\item[-] Voting (classification)
\item[-] Averaging (predictions) 
\end{itemize}


Problem: many poor methods may overwhelm high quality fit (remember earlier figures)\\
Solution: learn weights via cross validation


\end{frame}




\begin{frame}
\frametitle{Weighted Ensemble to Classify Documents}

\begin{itemize}
\item[-] Suppose we have $M$ $(m = 1, \hdots, M)$ models.  
\end{itemize}
\begin{eqnarray}
\invisible<1>{\alert<14->{\text{Pr}(Y_{i}   =  \text{C}_{1} | \textbf{x}\only<1-13>{_{i}}\only<14->{_{i,\text{test}}} )_{\text{train}} } & = & \alert<14->{  \sum_{m=1}^{M} \alert<3>{\widehat{\pi}_{m} } \alert<4>{\widehat{g}_{m}(\textbf{x}\only<1-13>{_{i}}\only<14->{_{i, \text{test}}} )}} \nonumber } 
\end{eqnarray}

\begin{itemize}
\invisible<1-4>{\item[-] Estimate weights $(\widehat{\pi}_{m})$} 
\only<1-11>{
\begin{itemize}
\invisible<1-5>{\item[-] K-fold cross validation: generate $M$ out of sample predictions for each document in training set}
\invisible<1-6>{\item[] $\widehat{\textbf{Y}}_{i}  = (\widehat{Y}_{i1}, \widehat{Y}_{i2}, \hdots, \widehat{Y}_{iM} )$} 
\invisible<1-7>{\item[-] Estimate weights with constrained regression:}
\begin{eqnarray}
\invisible<1-8>{Y_{i} & = & \sum_{m=1}^{M} \pi_{m} \hat{Y}_{im} + \epsilon_{i} \nonumber }
\end{eqnarray}
\invisible<1-9>{\item[] where we impose constraints: $\pi_{m} \geq 0 $ and $\sum_{m=1}^{M} \pi_{m} = 1$. } 
\invisible<1-10>{\item[-] Result $\widehat{\pi}_{m}$ for each method } 
\end{itemize}
}
\invisible<1-11>{\item[-] Estimate  $\widehat{g}_{m}(\textbf{x}_{i} ) \leadsto $ Apply all $M$ models to entire training set} 
\end{itemize}
\invisible<1-12>{3) For each document $i$ in test set, $\textbf{x}_{i, \text{test}}$ } \\
%\begin{eqnarray}
%\invisible<1-11>{\text{Pr}(Y_{i} = \text{Credit} | \textbf{w}_{i} ) & = & \sum_{m=1}^{M} \widehat{\pi}_{m} \widehat{g}_{m}(\textbf{w}_{i} ) \nonumber } 
%\end{eqnarray}
\invisible<1-14>{(Classify if above threshold)} \\


\pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause 
\end{frame}




\begin{frame}


\scalebox{0.6}{\includegraphics{SuperLearner.png}}


\end{frame}



\begin{frame}
\frametitle{Why Super Learn?}

van der Laan et al (2007) prove:
\begin{itemize}
\item[-] \alert{Asymptotically}: super learners will perform as well the \alert{best} candidates for data 
\item[-] \alert{Oracle}: performs like the best possible method among candidate methods
\begin{itemize}
\item[-] Asymptotically outperforms constituent methods
\item[-] Performs as well as optimal combinations of those methods
\end{itemize}
\end{itemize}

Practical questions:
\begin{itemize}
\item[-] Final regression: 
\begin{itemize}
\item[-] Logistic
\item[-] Linear 
\item[-] \alert{Could super learn again!}
\end{itemize}
\item[-] How Many Folds?
\begin{itemize}
\item[-] van der Laan et al's proofs rely on growing folds with $N$ (but slowly)
\item[-] Use 10-fold cross validation for simulations
\end{itemize}
\end{itemize}


\end{frame}



\begin{frame}
\frametitle{Impression of Influence}

Estimate: $Y_{i} \in \{\text{Credit}, \text{Not Credit} \}$ \pause \\
\begin{itemize}
\invisible<1>{\item[-] Triple hand code 800 press releases} \pause 
\invisible<1-2>{\item[-] Resolve disagreement with voting$\leadsto$ few disagreements} \pause
\end{itemize}

\invisible<1-3>{Use five classifiers to form Ensemble (cross validating within each to tune parameters)}\pause 
\begin{itemize}
\invisible<1-4>{\item[-] LASSO} \pause \invisible<1-9>{0} 
\invisible<1-5>{\item[-] Elastic-Net}\pause \invisible<1-9>{ 23\%} 
\invisible<1-6>{\item[-] Random Forest} \pause \invisible<1-9>{61\%} 
\invisible<1-7>{\item[-] A Support Vector Machine}\pause \invisible<1-9>{16\%} 
\invisible<1-8>{\item[-] Kernel Regularized Least Squares (KRLS, Hainmueller and Hazlett 2014)}\pause \invisible<1-9>{ 0} 
\end{itemize}





\end{frame}



\begin{frame}
\frametitle{Strategic Credit Claiming to Build a Personal Vote}

\begin{tikzpicture}
\node (dummy1) at (-8, 8) [] {} ; 
\node at (-6, 8) [] {\scalebox{0.4}{\includegraphics{DensTikz.pdf}}};

\invisible<1>{\node (trial) at (-8.75,5.7) [] {} ; 
\only<1-9>{\node(Burton) at (-10, 10) [] {\scalebox{0.5}{\includegraphics{Burton.jpg}}}; 
\draw[->, line width = 1.5pt] (Burton) to [out = 270, in = 90] (trial) ; } 

\only<3>{\node (McGroff) at (-3, 11) [] {\alert{John McGroff}: ``voted for every spending bill "} ; }
\only<3>{\node (McGroff2) at (-3, 10.6) [] {that went through the office"} ;}
\only<4>{\node (McGroff3) at (-3, 11) [] {\alert{John McGroff}:  ``Not the actions of a fiscally "};}
\only<4>{\node (McGroff4) at (-3, 10.6) [] {conservative congressman who } ; } 
\only<4>{\node (McGroff5) at (-3, 10.2) [] {cares about personal responsibility"}; }
}
%\invisible<1-4>{\node (Price) at (-10, 8) [] {\scalebox{0.2}{\includegraphics{Price.jpg}}} ; 
%\node (trial1) at (-8.7,5.7) [] {} ; 
%\draw[->, line width = 1.5pt] (Price) to [out = 0, in = 90] (trial1) ; }

\only<1-9>{
\invisible<1-4>{\node (Flake) at (-8.2, 11) [] {\scalebox{0.1}{\includegraphics{Flake.jpg}}} ; 
\node (trial2) at (-8.6,5.7) [] {} ; 
\draw[->, line width = 1.5pt] (Flake) to [out = 270, in = 90] (trial2) ; }


\invisible<1-5>{\node (Degette) at (-10, 6) [] {\scalebox{0.1}{\includegraphics{Jan.jpg}}} ; 
\node (trial3) at (-8.65, 5.7) [] {} ; 
\draw[->, line width = 1.5pt] (Degette) to [out = 30, in = 90] (trial2) ; }

\invisible<1-6>{\node (Lobiondo) at (-6, 10) [] {\scalebox{0.08}{\includegraphics{Lobiondo.jpg}}} ; 
\node (trial4) at (-5.1, 5.7) [] {} ; 
\draw[->, line width = 1.5pt] (Lobiondo) to [out = 270, in = 90] (trial4) ; } 

\invisible<1-7>{\node (Lobiondo) at (-4, 9.5) [] {\scalebox{0.125}{\includegraphics{Edwards.jpg}}} ; 
\node (trial5) at (-4.95, 5.7) [] {} ; 
\draw[->, line width = 1.5pt] (Lobiondo) to [out = 270, in = 90] (trial5) ; }}

\invisible<1-8>{\node (Rogers) at (-1.8, 11) [] {\scalebox{0.15}{\includegraphics{Rogers.jpg}}}; 
\node (trial6) at (-2.4, 5.7) [] {} ; 
\draw[->, line width = 1.5pt] (Rogers) to [out = 270, in = 90] (trial6) ; }

\only<10>{\node (Pork1) at (-7, 11)  [] {``We just can't afford luxuries like ideology" } ; } 
\only<11>{\node (Pork2) at (-7, 11)  [] {Lexington Herald-Leader: \alert{Prince of Pork}} ; }


\end{tikzpicture}

\pause \pause \pause \pause \pause \pause \pause \pause \pause \pause 

%\only<1>{\scalebox{0.4}{\includegraphics{DensTikz.pdf}}}
%\only<1>{\scalebox{0.4}{\includegraphics{NewDensity1.pdf}}}\only<2>{\scalebox{0.4}{\includegraphics{NewDensity2.pdf}}}\only<3>{\scalebox{0.4}{\includegraphics{NewDensity3.pdf}}}\only<4>{\scalebox{0.4}{\includegraphics{NewDensity4.pdf}}}\only<5>{\scalebox{0.4}{\includegraphics{NewDensity5.pdf}}}\only<6>{\scalebox{0.4}{\includegraphics{NewDensity9.pdf}}}\only<7>{\scalebox{0.4}{\includegraphics{NewDensity10.pdf}}}\only<8>{\scalebox{0.4}{\includegraphics{NewDensity11.pdf}}}\only<9>{\scalebox{0.4}{\includegraphics{NewDensity12.pdf}}}\only<10>{\scalebox{0.4}{\includegraphics{NewDensity13.pdf}}}


\end{frame}





\begin{frame}
\frametitle{Other Reasons to Ensemble (Dietterich 2000) } 

Statistical \pause 
\begin{itemize}
\invisible<1>{\item[-] With little data, many algorithms offer similar performance } \pause 
\invisible<1-2>{\item[-] Ensemble ensures we avoid \alert{wrong} model in test set } \pause 
\end{itemize}
\invisible<1-3>{Computational } \pause 
\begin{itemize}
\invisible<1-4>{\item[-] Methods stuck in local modes} \pause 
\invisible<1-5>{\item[-] Result: no one run provides best model} \pause 
\invisible<1-6>{\item[-] Averages of runs may perform better } \pause 
\end{itemize}
\invisible<1-7>{Complex ``true" functional forms } \pause 
\begin{itemize} 
\invisible<1-8>{\item[-] One method may be unable to approximate true DGP } \pause 
\invisible<1-9>{\item[-] Mixtures of methods may approximate better } \pause 
\end{itemize} 


\end{frame}

\begin{frame}
\frametitle{Ensembles Beyond Text Data}


Machine Learning $\leftrightarrow$ Causal Inference



\end{frame}




\begin{frame}
\frametitle{\alert{An Example} Experiment } 


\only<1-2>{\invisible<1>{Rep. Harold ``Hal" Rogers (KY-05) announced today that Kentucky is slated to
receive \$962,500 to protect critical infrastructure- power plants, chemical
facilities, stadiums, and other high-risk assets, through the U.S. Department of
Homeland Security's buffer zone protection program} }

\only<3>{A federal grant will help keep the Brainerd Lakes Airport operating in winter
weather. Today, Congressman Jim Oberstar announced that the Federal Aviation
Administration (FAA) will award \$528,873 to the Brainerd airport. The funding
will be used to purchase new snow removal and deicing equipment.}


\only<4>{Congresswoman Darlene Hooley (OR-5) and Congressmen Earl Blumenauer (OR-3),
David Wu (OR-1) and Greg Walden (OR-2) joined together today in announcing
\$375,000 in federal funding for the Oregon Partnership to combat methamphetamine
abuse in Oregon.\\}


\only<5>{What information in credit claiming messages affect evaluations?}

\pause \pause \pause \pause


\end{frame}



\begin{frame}
\frametitle{Rewarding Actions and Type of Expenditure, Not Money}

Experiment: vary the \alert{recipient} of money and the \alert{action} reported in credit claiming statement (and many other features)

\invisible<1>{Treatments: \alert<2>{type}}\invisible<1-2>{, \alert<3>{stage}}\invisible<1-3>{, \alert<4>{money}}\invisible<1-4>{, \alert<5>{collaboration}}\invisible<1-5>{, \alert<6>{partisanship}} 

%\invisible<1>{Treatments: \alert<2>{type}}\invisible<1-2>{, \alert<3>{stage}} \invisible<1-3>{money, collaboration, and partisanship} 

\only<1>{\vspace{1.65in}}
\only<7>{\alert{Control Condition}:}


\only<2>{\begin{itemize}
\item[1)] Planned Parenthood
\item[2)] Parks
\item[3)] Gun Range
\item[4)] Fire Department
\item[5)] Police 
\item[6)] Roads
\end{itemize}
\vspace{0.125in}}

\only<4>{\begin{itemize}
\item[1)] \$50 Thousand
\item[2)] \$20 Million 
\end{itemize}
\vspace{1.05in}}

\only<3>{\begin{itemize}
\item[1)] Will request
\item[2)] Requested
\item[3)] Secured
\end{itemize}
\vspace{0.825in}

}

%\only<4>{ \invisible<1-8>{\begin{itemize}
%\item[1)] Will request
%\item[2)] Request
%\item[3)] Secured
%\end{itemize}
%\vspace{0.825in}
%}}


\only<5>{\begin{itemize}
\item[1)] Alone 
\item[2)] w/ Senate Democrat
\item[3)] w/ Senate Republican 
\end{itemize}
\vspace{.825in}
}

\only<6>{\begin{itemize}
\item[1)] Democrat
\item[2)] Republican
\end{itemize}
\vspace{1.05in}
}

\only<7>{\begin{itemize}
\item[] Advertising press release
\end{itemize}
\vspace{1.125in}
}


\pause \pause \pause \pause \pause 

\end{frame}



\begin{frame}
\frametitle{Rewarding Actions and Type of Expenditure, Not Money}

\only<1>{Example  Treatment: \\
\textbf{Headline}: Representative [blackbox] \alert{secured} \alert{\$50 Thousand} \alert{to purchase safety equipment for local firefighters}  \\
\textbf{Body}: Representative [blackbox] (\alert{Democrat}) and \alert{Senator [blackbox], a Democrat}, \alert{secured} \alert{\$50 Thousand} \alert{to purchase safety equipment for local firefighters}.  \\
Rep. [blackbox] said ``This money \alert{will help} \alert{our brave firefighters stay safe as they protect our businesses and homes}" \\}


\only<2>{Example Treatment:\\
\textbf{Headline}: Representative [blackbox] \alert{will request} \alert{\$20 million} \alert{for medical equipment at the local Planned Parenthood.}  \\
\textbf{Body}: Representative [blackbox] (\alert{Democrat}), \alert{will request} \alert{\$20 million} \alert{for medical equipment at the local Planned Parenthood}.  \\
Rep. [blackbox] said ``This money \alert{would help} \alert{provide state of the art care for women in our community.}" \\}

\only<3->{
\invisible<1-2>{\alert{214} other conditions} \\
\invisible<1-3>{Dependent variable: Approve of representative}\\

\vspace{0.175in}
\large 
\invisible<1-4>{\alert{Goal} $\leadsto$ measure effect of credit claiming content on approval ratings }
\invisible<1-5>{Mechanics$\leadsto$  Mechanical Turk sample (\alert{Findings are replicated in representative samples, using real representatives/senators}  )}
}


\pause \pause \pause \pause \pause %\pause \pause 
\end{frame}


\begin{frame}
\frametitle{Rewarding Actions and Type of Expenditure, Not Money}

\begin{itemize} 
\item[-] Participant  $i$ ($i=1, \hdots, N$), has treatment assignment $\textbf{T}_{i}$   \\
\invisible<1>{\item[-] If $\text{T}_{i} = 0$ for control condition} \\
\invisible<1-2>{\item[-] $\textbf{T}_{i} = (\text{T}_{i, \text{type}},  \text{T}_{i, \text{stage} },\text{T}_{i, \text{money}},  \text{T}_{i, \text{collab.}}, \text{T}_{i, \text{part.}} )$ }

\invisible<1-3>{\item[-] $Y_{i}(\textbf{T}_{i} )$ : participant $i$'s Approval decision under treatment $\textbf{T}_{i} $}  
\invisible<1-4>{\item[-] \alert{Quantities of Interest} } 
\only<1-10>{\invisible<1-5>{\item[-] Effect of particular component of message: }
\begin{itemize}
\invisible<1-6>{\item[-] T$_{\text{stage}} = $ Secured}
\invisible<1-7>{\item[-] T$_{\text{stage}}  = $ Requested }
\invisible<1-8>{\item[-] T$_{\text{stage}} = $ Will Request } 
\invisible<1-9>{\item[-] T$_{j} = k $} 
\end{itemize}
}


\invisible<1-10>{\item[-]  Marginal Average Treatment Effect  (MATE$_{\text{T}_{j} = k } $) }  
\end{itemize}
\only<1-13>{
\begin{eqnarray}
\invisible<1-11>{\text{MATE}_{\text{T}_{j} = k} & = & \int \text{E}[Y(\text{T}_{j} = k , \textbf{T}_{-j} )  - Y(0) ] \text{d}\text{F}_{\textbf{T}_{-j} | \text{T}_{j} = k } \nonumber \\}
\invisible<1-12>{\text{MATE}_{\text{T}_{j} = k}	& = & \text{E}[Y(\text{T}_{j} = k ) - Y(0) ] \nonumber }
\end{eqnarray}
\vspace{1.5in}

}
\only<14->{		
\begin{eqnarray}
		\text{MATE}_{\text{T}_{j} = k}	& = & \text{E}[Y(\text{T}_{j} = k )|\text{T}_{j} = k ]  - \text{E}[Y(0)|\text{T} = 0 ] \nonumber \\
\invisible<1-14>{\widehat{\text{MATE}_{\text{T}_{j} = k} } & = & 		\frac{\sum_{i=1}^{N} Y_{i} I(\text{T}_{ij} = k)  }{\sum_{i=1}^{N} I(\text{T}_{ij} = k) } - \frac{\sum_{i=1}^{N} Y_{i} I(\text{T}_{i}=0)  }{\sum_{i=1}^{N} I(\text{T}_{i}= 0) } \nonumber								}
\end{eqnarray}										
\vspace{1.5in}
}




\pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause 


\end{frame}



\begin{frame}
\frametitle{Rewarding Actions and Type of Expenditure, Not Money}
%(1) Marginal Average Treatment Effect (MATE$_{T_{j} = k } $): $\text{E}[Y(T_{j} = k ) - Y(0) ]$ \\
%\vspace{0.025in} 
%\pause 
\begin{itemize}
\item[-] Response may be conditional on respondent characteristics $\textbf{x}$ \pause \\
\invisible<1>{\item[-] For example $\textbf{x}  = $ (Conservative, Republican)} \pause \\
\invisible<1-2>{\item[-] Marginal Conditional Average Treatment Effect (MCATE$_{\text{T}_{j} = k,\textbf{x}}$)} \pause 
\end{itemize}
\begin{eqnarray}
\invisible<1-3>{\text{MCATE}_{\text{T}_{j} = k,\textbf{x}} & = & \text{E}[Y(\text{T}_{j} = k) - Y(0) | \textbf{x}] }    \nonumber\\
\invisible<1-4>{\text{MCATE}_{\text{T}_{j} = k, \textbf{x}} & = & \text{E}[Y(\text{T}_{j} = k)|\textbf{x} ] - \text{E}[Y(0) | \textbf{x} ] \nonumber }  \nonumber 
\end{eqnarray}


\end{frame}

\begin{frame}
\frametitle{Rewarding Actions and Type of Expenditure, Not Money}
\begin{eqnarray}
\text{MCATE}_{\text{T}_{j} = k, \textbf{x}} & = & \text{E}[Y(\text{T}_{j} = k)|\textbf{x} ] - \text{E}[Y(0) | \textbf{x} ] \nonumber \\
\only<1-8>{\invisible<1>{\widehat{\text{MCATE}_{\text{T}_{j} = k,\textbf{x}}}& = &\frac{\sum_{i=1}^{N} Y_{i} I(\text{T}_{j} = k ,  \textbf{x}_{i} = \textbf{x} )  }{\sum_{i=1}^{N} I(\text{T}_{j} = k , \textbf{x}_{i} = \textbf{x} ) }    - \frac{\sum_{i=1}^{N} Y_{i} I(\text{T}_{i}=0 , \textbf{x}_{i} = \textbf{x})  }{\sum_{i=1}^{N} I(\text{T}_{i}= 0 , \textbf{x}_{i} = \textbf{x}) }  \nonumber } \pause }
\only<9->{
\widehat{\text{MCATE}_{\text{T}_{j} = k,\textbf{x}}} & = & \widehat{g}_{m} (\text{T}_{j} = k, \textbf{x} ) - \widehat{g}_{m}(0, \textbf{x} ) \nonumber 
}
\end{eqnarray}



\begin{itemize}
\invisible<1-2>{\item[-] \alert{Curse of Dimensionality}: highly variable estimates, (sometimes) empty strata} 
\invisible<1-3>{\item[-] Separate systematic differences from noise $\leadsto$ \alert{data} and \alert{assumptions} } \invisible<1-4>{ Heterogeneous treatment effect methods} % ($g_{m} (\text{T}_{j} = k, \textbf{x} ) $)}
\begin{itemize}
\invisible<1-5>{\item[-] LASSO, Find It (Imai and Ratkovic, 2013)$\leadsto$ sparsity} 
\invisible<1-6>{\item[-] Ridge, KRLS (Hainmueller and Hazlett, 2013)$\leadsto$ flexible surface, dense} 
\invisible<1-7>{\item[-] Model $m$ to estimate some function $g_{m}(\text{T}_{j} = k, \boldsymbol{x})$}
\end{itemize}
\invisible<1-8>{\item[-] Perform well: $g_{m}(\text{T}_{j} = k, \boldsymbol{x})$ accurately estimates response surface ($\text{E}[Y(\text{T}_{j} = k)|\textbf{x} ]$)}
\invisible<1-9>{\item[-] Perform well: accurate out of sample prediction and classification (van der Laan et al 2007, Raftery et al 2005) }
\end{itemize}

\invisible<1-10>{Create ensemble: weighting methods by (unique) out of sample predictive performance}

\pause \pause \pause \pause \pause \pause \pause \pause \pause \pause 



\end{frame}

\begin{frame}
\frametitle{Weighted Ensemble to Measure Credit Claiming Rate}

\begin{itemize}
\item[-] Suppose we have $M$ $(m = 1, \hdots, M)$ models.  
\end{itemize}


\begin{eqnarray}
\only<1-13>{\invisible<1>{\widehat{\text{MCATE}_{\text{T}_{j} = k,\textbf{x}}}  & = & \sum_{m=1}^{M} \alert<3>{\widehat{\pi}_{m}} (\alert<4>{\widehat{g}_{m} (\text{T}_{j} = k, \textbf{x} )  }-\alert<4>{\widehat{ g}_{m} (0, \textbf{x} )  })} \nonumber }
\only<14->{\alert{\widehat{\text{MCATE}_{\text{T}_{j} = k,\textbf{x}_{\text{new}}}}  }& = & \sum_{m=1}^{M} \alert{\widehat{\pi}_{m}} (\alert{\widehat{g}_{m} (\text{T}_{j} = k, \textbf{x}_{\text{new}} ) } -\alert{\widehat{ g}_{m} (0, \textbf{x}_{\text{new}} )  } )\nonumber }
\end{eqnarray}

\begin{itemize}
\invisible<1-4>{\item[-] Estimate weights $(\widehat{\pi}_{m})$} 
\only<1-12>{
\begin{itemize}
\invisible<1-5>{\item[-] 10-fold cross validation: generate $M$ out of sample predictions for each observation}
\invisible<1-6>{\item[] $\widehat{\textbf{Y}}_{i}  = (\widehat{Y}_{i1}, \widehat{Y}_{i2}, \hdots, \widehat{Y}_{iM} )$} 
\only<1-11>{\invisible<1-7>{\item[-] Estimate weights with constrained regression:}
\begin{eqnarray}
\invisible<1-8>{Y_{i} & = & \sum_{m=1}^{M} \pi_{m} \hat{Y}_{im} + \epsilon_{i} \nonumber }
\end{eqnarray}
\invisible<1-9>{\item[] where we impose constraints: $\pi_{m} \geq 0 $ and $\sum_{m=1}^{M} \pi_{m} = 1$. } 
\invisible<1-10>{\item[-] Result $\widehat{\pi}_{m}$ for each method } }
\only<12>{\item[-] (Alternatively) Estimate weights from mixture model (EBMA) (Raftery et al 2005;  Montgomery, Hollenback, Ward 2012)$\leadsto$ EM, Gibbs, Variational Approximation} 
\end{itemize}
}
\invisible<1-12>{\item[-] Estimate  $\widehat{g}_{m}(\text{T}_{j} = k, \textbf{x} ) \leadsto $ Apply all $M$ models to entire data set} 

\invisible<1-13>{\item[-] Generate effects  of interest (perhaps weighting to other population)  $\textbf{x}_{\text{new}}$ } 
\end{itemize}
%\invisible<1-14>{Applied to this experiment: Positive weight on three methods:} \\
%\invisible<1-15>{(1) LASSO (0.62), (2) KRLS (0.24), (3) Find It (0.14)} 
%\begin{eqnarray}
%\invisible<1-11>{\text{Pr}(Y_{i} = \text{Credit} | \textbf{w}_{i} ) & = & \sum_{m=1}^{M} \widehat{\pi}_{m} \widehat{g}_{m}(\textbf{w}_{i} ) \nonumber } 
%\end{eqnarray}
%\invisible<1-14>{(Classify if above threshold)} \\
%\invisible<1-15>{{\large \alert{90\% accuracy}} }\\

\pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause \pause 
\end{frame}




\begin{frame}
\frametitle{Monte Carlo Evidence}
\begin{center}
\only<1>{\scalebox{0.4}{\includegraphics{Plot1.pdf}}}
\only<2>{\scalebox{0.4}{\includegraphics{Plot2.pdf}}}
\only<3->{\scalebox{0.3}{\includegraphics{Plot2.pdf}}}
\end{center}
\only<3->{\large Ensembles outperform constituent methods $\leadsto$ ensembles place weight on  better performing method}

\end{frame}


\begin{frame}
\frametitle{Returning to \alert{Example} Experiment}

\alert{Recall}: experiment to assess effects of credit claiming on approval $\leadsto$ 1,074 participants (MTurk) \pause  \\
\invisible<1>{Apply ensemble method (7 constituent methods, 10 fold cross validation), including treatments and Partisanship and Ideology.  } \pause \\
\invisible<1-2>{Positive  weight on three methods:} \pause 
\begin{itemize}
\invisible<1-3>{\item[1)] LASSO (0.62)} \pause 
\invisible<1-4>{\item[2)] KRLS (0.24)} \pause 
\invisible<1-5>{\item[3)] Find it (0.14) } 
\end{itemize}

\end{frame}

\begin{frame}
\begin{tikzpicture}
\node (dumm) at (-8, 8) [] {} ; 
\only<1-2, 4, 6>{\node (plot1) at (-5, 8) [] {\scalebox{0.25}{\includegraphics{HetExpIdeoFig2.pdf}}}; }
\only<8>{\node (plot_no) at (-3, 8) [] {\scalebox{0.3}{\includegraphics{SmallTypeMoneyIdeology.pdf}}}; }
\only<9->{\node (plot_no) at (-3, 8) [] {\scalebox{0.2}{\includegraphics{SmallTypeMoneyIdeology.pdf}}}; }
\node (d_plan) at (-8, 4.5) [] {} ; 
\node (d_plan3) at (-1.25, 4.5) [] {} ; 
\node (d_plan2) at (-8, 5.75) [] {} ; 
\node (d_plan4) at (-1.25, 5.75) [] {} ; 

\only<2>{\draw[-, line width = 2pt, red] (d_plan) to [out = 0, in = 180] (d_plan3) ; 
\draw[-, line width = 2pt, red] (d_plan2) to [out = 0, in = 180] (d_plan4) ; }

\only<3>{\node (plot2) at (-5, 8) [] {\scalebox{0.4}{\includegraphics{HetPlanned.pdf}}}; }


\node (d_gun) at (-8, 6.85) [] {} ; 
\node (d_gun3) at (-1.25, 6.85) [] {} ; 
\node (d_gun2) at (-8, 8.05) [] {} ; 
\node (d_gun4) at (-1.25, 8.05) [] {} ; 

\only<4>{\draw[-, line width = 2pt, red] (d_gun) to [out = 0, in = 180] (d_gun3) ; 
\draw[-, line width = 2pt, red] (d_gun2) to [out = 0, in = 180] (d_gun4) ; }

\node (d_gun) at (-8, 9.25) [] {} ; 
\node (d_gun3) at (-1.25, 9.25) [] {} ; 
\node (d_gun2) at (-8, 10.4) [] {} ; 
\node (d_gun4) at (-1.25, 10.4) [] {} ; 


\only<5>{\node (plot3) at (-5, 8) [] {\scalebox{0.4}{\includegraphics{HetGun.pdf}}}; }

\only<6>{\draw[-, line width = 2pt, red] (d_gun) to [out = 0, in = 180] (d_gun3) ; 
\draw[-, line width = 2pt, red] (d_gun2) to [out = 0, in = 180] (d_gun4) ; }

\only<7>{\node (plot3) at (-5, 8) [] {\scalebox{0.4}{\includegraphics{HetPolice.pdf}}}; }



\end{tikzpicture}

\only<9>{$\leadsto$ Constituents evaluate expenditures using \alert{qualitative} information, rather than numerical facts}

\pause \pause \pause \pause \pause \pause \pause \pause 


\end{frame}



\begin{frame}
\frametitle{Ensembles to Estimate Heterogeneous Effects}

Ensembles: prediction, classification, \alert{estimation of heterogeneous effects} \pause \\
\invisible<1>{{\tt R} package$\leadsto$ implement methods, create synthetic observations, visualize results } \pause \\
\invisible<1-2>{\alert{Ensembles} as companion:} \pause 
\begin{itemize}
\invisible<1-3>{\item[-] Better individual methods$\leadsto$ better ensembles } \pause 
\invisible<1-4>{\item[-] Evaluate new methods $\leadsto$ more weight from ensemble } \pause 
\begin{itemize}
\invisible<1-5>{\item[1)] Distinct} \pause 
\invisible<1-6>{\item[2)] Accurate} \pause 
\end{itemize}
\end{itemize}

\invisible<1-7>{\alert{Ensembles} $\leadsto$ leverage many contributions to build better estimates.}





\end{frame}


\end{document}