%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Beamer Presentation
% LaTeX Template
% Version 1.0 (10/11/12)
%
% This template has been downloaded from:
% http://www.LaTeXTemplates.com
%
% License:
% CC BY-NC-SA 3.0 (http://creativecommons.org/licenses/by-nc-sa/3.0/)
%
% Changed theme to WSU by William King
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%----------------------------------------------------------------------------------------
%    PACKAGES AND THEMES
%----------------------------------------------------------------------------------------

\documentclass[xcolor=dvipsnames,aspectratio=169]{beamer}


%Import Preamble bits
\input{../assets/preambles/FormattingPreamble.tex}
\input{../assets/preambles/TikzitPreamble.tex}
\input{../assets/preambles/MathPreamble.tex}
\input{../assets/preambles/BibPreamble.tex}
\input{../assets/preambles/GeneralPreamble.tex}


%----------------------------------------------------------------------------------------
%    TITLE PAGE
%----------------------------------------------------------------------------------------

\title[Clinical Trials]{The Effects of Market Conditions on Recruitment and Completion of Clinical Trials} 

\author{Will King} % Your name
\institute[WSU] % Your institution as it will appear on the bottom of every slide, may be shorthand to save space
{
Washington State University \\ % Your institution for the title page
\medskip
\textit{william.f.king@wsu.edu} % Your email address
}
\date{\today} % Date, can be changed to a custom date





\begin{document}
\begin{frame}
\titlepage % Print the title page as the first slide
\end{frame}
%----------------------------------
\begin{frame} %Allow frame breaks
\frametitle{Clinical Trials} % Table of contents slide, comment this out to remove it
    % - Intro and hook (Clinical Trials are key part of pharmacological pipeline)
    Pharmaceuticals are a frequently discussed aspect of health care cost management.
    Their development is dictated by scientific and regulatory hurdles 
    including passing clinical trials
    (\cite{noauthor_fda_nodate}), 
    while their market is characterized by strategic competition and ambiguous 
    patent protection
    (\cite{van_der_gronde_addressing_2017}).

    \vspace{12pt}

    This research investigates the ways by which market conditions
    affect clinical trial completion.
\end{frame}
%--------------------------------
\begin{frame}[allowframebreaks] %Allow frame breaks
\frametitle{Overview} % Table of contents slide, comment this out to remove it
\tableofcontents 
% - Intro and hook
% - Literature review
% - Causal Identification
% - Data
% - Econometric model
% - Results
% - Improvements
\end{frame}
%-------------------------------

%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Introduction and Background %%%%%%%%%%%%%%%%%%%%%%%%
\section{Background}
% TOC
% - Background on drug process
% - Literature on clinical trials
% - My questions
% add info about trials
% - Requirements (pre registered design [2007], updated "regularly" on clinicaltrials.gov)
% - Phases (1,2,3,4, mixed)
% - Safety and Ethicas (oversight boards, restrictions on payments)
% - Approval processes (biologics vs small-molecule)
% add info about drugs
%-------------------------------------------------------------------------------------

%-------------------------------
\begin{frame}
    \frametitle{Clinical Trials and Drug develoment}

    The FDA requires clinical trials before approving new drug compounds
    \begin{itemize}
        \item Pre-registered design
        \item Updated regularly on clinicaltrials.gov
        \item Often requires an oversight board.
        \item Goal is to prove efficacy and safety of a compound/dosage/route.
        \item A new drug candidate (NDC) must complete 3 phases of clinical trials before approval.
        \item Phases are reviewed with FDA.
        \item Not all clinical trials are for new drugs.
    \end{itemize}
\end{frame}
%-----------------------------
\begin{frame}
    \frametitle{Literature Highlights}
    \begin{itemize}
        \item \cite{van_der_gronde_addressing_2017}: 
            High level synthesis of overall discussion regarding drug costs. 
            Both academic and non-academic sources.
        \item \cite{hwang_failure_2016}:
            Answered the question "Why do late-stage (phase III) trials fail?"
            Found that efficacy, safety, and competition reasons accounted for 
            57\%, 17\%, and 22\% respectively.
        \item \cite{abrantes-metz_pharmaceutical_2004}:
            Described how drugs progress through the 3 phases of clinical trials
            and correlations between various trial characteristics and the 
            clinical trial failures.
        \item \cite{khmelnitskaya_competition_2021}: 
            Modeled clinical trial life-cycle of drugs, found method to separate
            scientific from competitive reasons for failure to progress to the 
            next phase.
%        \item \cite{}: 

    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{This research, in context}

    In contrast to previous work looking at multiple phases of trials,
    I seek to figure out what causes individual trials to fail.

%    \vspace{12pt}
%
%    Instead of focusing on the drug development pipeline, I attempt to 
%    investigate the population of drug-based, phase III trials.
%
    \vspace{12pt}

    Questions
    \begin{itemize}
        \item How do the competitors on the market affect clinical trial completion?
        \item Can we tell how this effect moderated by the enrollment of participants?
        \item Is this effect consistent across different disease categories?
    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Audience Questions}

    \center{What can I clarify?}
\end{frame}
%%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%%% Causality and Data %%%%%%%%%%%%%%%%%%%%%%%%
%\section{Causal Story and Data}
%% TOC
%% - Causal Story (no subsection)
%%    - Clinical trials: targets specific drug/condition combination.
%%    - Enrollment process: patients counsel with providers
%%    - Trials terminate if unsafe, ineffective, unprofitable, or cannot enroll patients
%%    - Ethical concerns exist throughout.
%%    - This is complicated by the fact that the experiment reveals information over time.
%% - Formalization
%% - Data Sources
%% Data Generating process
%% - Agents and their decisions
%% - Factors that influence each decision
%%-------------------------------------------------------------------------------------
%
%-------------------------------
\begin{frame}[shrink=10] %evil option is helpful here.
    \frametitle{How do clinical trials proceed?}
    \begin{columns}[T]
        \begin{column}{0.5\textwidth}
            What does a \textbf{Completed} trial look like?
            \begin{enumerate}
                \item Study sponsor comes up with design
                \item Apply for NCT ID from ClinicalTrials.gov
                \item Begin enrolling participants
                \item Update ClinicalTrials.gov to recruit
                \item Close Enrollment
                \item Update ClinicalTrials.gov as not recruiting*
                \item Reach primary objectives
                \item Update ClinicalTrials.gov as complete
                \item Reach secondary objectives
                \item Update ClinicalTrials.gov with more information
            \end{enumerate}
        \end{column}
        \begin{column}{0.5\textwidth}
            What does an \textbf{Terminated} trial look like?
            \begin{enumerate}
                \item Study sponsor comes up with design
                \item Apply for NCT ID from ClinicalTrials.gov
                \item Begin enrolling participants
                \item Update ClinicalTrials.gov to advertise
                \item Run into issues: 
                    \begin{itemize}
                        \item Safety
                        \item Efficacy
                        \item Profitability
                        \item Feasiblity (enrollment, PI leaves, etc.)
                    \end{itemize}
                \item Close Enrollment*
                \item Decide to terminate clinical trial.
                \item Update ClinicalTrials.gov as terminated.
            \end{enumerate}
        \end{column}
    \end{columns}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{ClinicalTrials.gov}
    Thus ClinicalTrials.gov becomes an (append only) repository of 
    the ``current'' status of clincal trials.

    As it is designed to help faciltate enrollment in clinical trials, 
    the record includes important information such as 

    \begin{itemize}
        \item drugs
        \item study arms
        \item conditions
        \item expected and final enrollment figures
        \item current status
    \end{itemize}

    ClinicalTrials.gov also reports the history from previous
    updates.
\end{frame}

%-------------------------------
\begin{frame}
    \frametitle{Decision-Making Process}
    % study sponsors
    Study Sponsors Decide to start a Phase 3 trial and whether to terminate it.
    \\
    They ask themselves:
    \begin{itemize}
        \item Do safety incidents require terminating a trial?
        \item Do efficacy results indicate the trial is worth continuing?
        \item Is recruiting sufficient to achieve our results and contain costs?
        \item Do expectations about future returns justify our expenditures?
    \end{itemize}
    
    They plan their enrollment considering
    \begin{itemize}
        \item Total population affected
        \item Likely participant response rates
        \item Their network of clinicians
    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Decision-Making Process}
    % participants
    Participants decide to enroll (and dis-enroll) themselves in a trial based on
    \begin{itemize}
        \item Doctor Recommendations
        \item Disease severity
        \item Relative safety/efficacy compared to other treatments
    \end{itemize}

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Questions?}
    \center{What clarifying questions do you have?}
\end{frame}
%-------------------------------
%--------------------------------
%%%%%%%%%%%%%%%%%%%% Causal Formalization
\subsection{Formalization}
% - Introduce basic triangle
%    - discuss total vs direct effects
%    - 
% - Add confounders and controls
% - Introduce backdoor criterion
%--------------------------------
%%-------------------------------
%%\begin{frame} %Allow frame breaks
%%\frametitle{Why this approach?} 
%%    \begin{figure}
%%        \includegraphics[height=0.8\textheight]{../assets/img/methodology_trial.png}
%%        \label{FIG:xkcd2726}
%%        \caption{``If you think THAT'S unethical, you should see the stuff we approved via our Placebo IRB.'' 
%%        - \url{https://xkcd.com/2726}
%%        }
%%    \end{figure}
%%\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Framing my Questions}
    \begin{columns}[T]
        \begin{column}{0.5\textwidth}
            Two potential causes of trial termination include 
            \begin{enumerate}
                \item Alternative (competitor) treatments exist
                    \begin{itemize}
                        \item reduces future profitability.
                        \item reduces incentives to enroll as participants.
                    \end{itemize}
                \item It can be difficult to recruit patients
                    \begin{itemize}
                        \item Are there few patients?
                        \item Are potential participants choosing other alternatives?
                    \end{itemize}
            \end{enumerate}
        \end{column}
        \begin{column}{0.5\textwidth}
            Overall this can be described graphically as:

            \begin{figure}
                \scalebox{0.8}{
                    \tikzfig{../assets/tikzit/4Node}
                }
                \label{FIG:4Node}
                \caption{Total Effect}
            \end{figure}
        \end{column}
    \end{columns}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Causal Effects}
    %Discuss the two different effects: total effect, direct effects

    \begin{columns}
        \begin{column}{0.5\textwidth}
            Total Effect of Competitors

                \begin{figure}
                    \scalebox{0.8}{
                        \tikzfig{../assets/tikzit/4Node_total}
                    }
                    \label{FIG:4Node}
                    \caption{Total Effect}
                \end{figure}
        \end{column}
        \begin{column}{0.5\textwidth}
            Direct Effects of Competitors and Enrollment

                \begin{figure}
                    \scalebox{0.8}{
                        \tikzfig{../assets/tikzit/4Node_direct}
                    }
                    \label{FIG:4Node}
                    \caption{Direct Effect}
                \end{figure}
        \end{column}
    \end{columns}
\end{frame}

%-------------------------------
\begin{frame}
    \frametitle{Rephrasing Questions}
    To rephrase my questions
    \begin{enumerate}
        \item How large is the total effect of increasing the number 
            of competing drugs on completing clinical trials?
        \item How large is the direct effect of increasing the number 
            of competing drugs on completing clincial trials?
    \end{enumerate}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Additional Concerns}
    %Confounders 
    Of course, there are other confounding relationships
    \begin{enumerate}
        \item Population Effects
        \item Fundamental Safety and Efficacy of compound/dosage/route
        \item How long the trial is taking
        \item the total effects
    \end{enumerate}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Complete graph}
    %introduce backdoor criterion

            \begin{figure}
                \scalebox{0.6}{
                    \tikzfig{../assets/tikzit/CausalGraph}
                }
                \label{FIG:CausalGraph}
                \caption{Full Causal Graph}
            \end{figure}
            Discuss concerns about Elapsed Duration and Enrollment

\end{frame}
%%-------------------------------
\begin{frame}
    \frametitle{Causal Diagram: Backdoor Criterion}
    \small
    \begin{block}{$d$-separation}
        A set $S$ of nodes blocks a path $p$ on a DAG if either
        \begin{enumerate}
            \item $p$ contains at least one arrow-emitting node in $S$
            \item $p$ contains at least one collision node $c$ that is outside $S$ 
                and has no descendants in $S$.
        \end{enumerate}
        If $S$ blocks all paths from X to Y, then it is said to ``$d$-separate'' 
        $X$ and $Y$, and then $X \perp Y | S$.
    \end{block}
    \begin{block}{Back-Door Criterion}
        A set $S$ of covariates is admissible as controls on the 
        causal relationship $X \rightarrow Y$ if:
        \begin{enumerate}
            \item No element of $S$ is a descendant of $X$
            \item The elements of $S$ d-separate all paths from $X$ to $Y$ that include
                parents of $X$.
        \end{enumerate}
    \end{block}
    \cite{pearl_causality_2000}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Sufficent Adjustment Set}
    %introduce backdoor criterion

    Thus the required adjustment set depends on the effects of interest.

    For the total effect these are controls for:
    \begin{itemize}
        \item Proceed with Phase III
        \item Condition
        \item Population
    \end{itemize}
    Discuss Regime Switching

    For the direct effect these are controls for:
    \begin{itemize}
        \item Proceed with Phase III
        \item Condition
        \item Population (optional)
        \item Enrollment
    \end{itemize}
    Not causally identified due to Regime Switching


\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Other testable hypotheses}
    One advantage of this approach is that there are tools that can automatically
    \begin{itemize}
        \item verify causal identification
        \item generate hypotheses to verify model
    \end{itemize}

    In this case, the following are testable hyptheses
    \begin{itemize}
        \item Decision to continue Phase III $\perp$ Elapsed Duration
        \item Decision to continue Phase III  $\perp$ Market Conditions $\Vert$ Condition
        \item Decision to continue Phase III $\perp$ Population $\Vert$ Condition
        \item Elapsed Duration $\perp$ Market Conditions
        \item Elapsed Duration $\perp$ Condition
        \item Elapsed Duration $\perp$  Population
        \item Population $\perp$  Terminated $\Vert$ Condition, Decision to continue Phase III, Elapsed Duration, Enrollment Status, Market Conditions
    \end{itemize}

    \href{http://dagitty.net/mLyFuc5}{Dagitty.net model}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Questions?}

\end{frame}
%-------------------------------
%--------------------------------
%%%%%%%%%%%%%%%%%%%% Data sources
\subsection{Data Sources}
% TOC
% - Main Data Sources
%    - ClinicalTrials.gov and AACT
%    - IHME Burden of Disease
%    - Marketing Data
%    - MeSH, RxNorm/RxNav
% - How did I Link Data Sources
% - Data Sizes
%--------------------------------

%-------------------------------
\begin{frame} %Allow frame breaks
    \frametitle{Data Sources}
    \begin{itemize}
        \item ClinicalTrials.gov - AACT \& custom scripts
            \begin{itemize}
                \item Select trials of interest
                \item Trial details: 
                    \begin{itemize}
                        \item conditions
                        \item final status
                        \item drugs/interventions
                    \end{itemize}
                \item Trial snapshots:
                    \begin{itemize}
                        \item elapsed duration
                        \item enrollment status (NYE,EBI,R,ANR)
                    \end{itemize}
            \end{itemize}
        \item Medical Subject Headings (MeSH) Thesaurus
            \begin{itemize}
                \item A standardized nomenclature used to classify interventions 
                    and conditions in the clinical trials database.
            \end{itemize}
    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame} %Allow frame breaks
    \frametitle{Data Sources}
    \begin{itemize}
        \item USP Drug Classification (2023)
            \begin{itemize}
                \item Used to measure which drugs
            \end{itemize}
        \item NSDE Files (New drug code Structured product labels Data Element)
            \begin{itemize}
                \item Contains information about when a given drug was on the market.
            \end{itemize}
        \item RxNorm
            \begin{itemize}
                \item Links pharmaceuticals between MeSH standardized terms and 
                    NSDE files.
                \item Used to find brand names that share active ingredients with those from trial.
            \end{itemize}
    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame} %Allow frame breaks
    \frametitle{Data Sources}
    \begin{itemize}
        \item Global Disease Burden Survey (2019)
            \begin{itemize}
                \item Estimates of DALYs for categories of disease
                \item Links of Categories to ICD-10 Codes
            \end{itemize}
        \item ICD-10 (2019)
            \begin{itemize}
                \item WHO version
                \item CMS version (Clinical Management)
                \item Used to group disease conditions in hierarchical model
            \end{itemize}
        \item Unified Medical Language System Thesaurus
            \begin{itemize}
                \item Used to link MeSH standardized terms and ICD-10 conditions
                \item Manual matching process
            \end{itemize}
    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Linking data}
    %
    The following linking process was used:
    \begin{enumerate}
        \item AACT trials to snapshots (internal ID)
        \item AACT trials to ICD-10 (hand match)
        \item ICD-10 to IHME (IHME)
        \item Snapshots to drug brands (RxNorm/RxNav/MeSh, SPL)
        \item AACT to USP DC alternates (RxNorm, USP DC, hand match)
    \end{enumerate}

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Data used}
    The following data points were used.
    \begin{itemize}
        \item elapsed duration 
        \item enrollment status
        \item asinh(brands with identical ingredients)
        \item asinh(brands in USP-DC category)
        \item asinh(high sdi DALY estimate)
        \item asinh(high-medium sdi DALY estimate)
        \item asinh(medium sdi DALY estimate)
        \item asinh(low-medium sdi DALY estimate)
        \item asinh(low sdi DALY estimate)
    \end{itemize}
    The asinh operator was used because it parallels $\text{ln}(x)$ for 
    large values of $x$ but also handles $\text{asinh}(0)=0$.
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Measures of Causes and Effects}
    
    Here are the actual measures used for causes
    \begin{itemize}
        \item Final Status: Measured from AACT - status when trial is over.
        \item Competitors on Market: Measured by the number of drugs
            \begin{itemize}
                \item with same active ingredients (at the time of the snapshot)
                \item sharing the USP DC category and class (in 2023)
            \end{itemize}
    \end{itemize}

    Effects are measured in parameter values and changes in probability

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Adjustment set}
    Here are the actual measures of the adjustment set
    \begin{itemize}
        \item Enrollment: Measured by enrollment status at the snapshot level.
        \item Elapsed Duration: Measured at snapshot level
            by $\frac{\text{Current Date} - \text{Start Date}}{\text{Planned Completion Date} - \text{Start Date}}$
        \item Population Measures
            \begin{itemize}
                \item IHME Global Disease Burden: DALYs, spread over 5 levels of the Social Development Index
            \end{itemize}
        \item Beliefs about safety \& efficacy: Restricted to Phase 3 trials.
        \item Disease Type: Hierarchal parameters in model
    \end{itemize}

    Note the implicit conditioning on trials treating diseases with IHME data\footnote{
        IHME does not track data for W61.62XD: Struck by duck, subsequent encounter
    }.

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Other Details}

    Other Trial Selection Criteria
    \begin{itemize}
        \item Interventional Study
        \item Involved an FDA Regulated Drug
        \item Phase 3 trial
        \item Started after 2010-01-01
        \item Ended before 2022-01-01
    \end{itemize}

\end{frame}
%%----------------------------------
%%%%%%%%%%%%%%%%%%%%% Summary
%\subsection{Data Summary}
%%----------------------------------
%%-------------------------------
%\begin{frame}
%    \frametitle{Data Summaries}
%    %TODO: Update
%    \begin{itemize}
%        \item Number of Phase III, FDA monitored Drug Trials: 1,981
%        \item Number of Trials matched to ICD-10: 186
%        \item Number of Trials matched to ICD-10 with population measures: 67 
%            (51 completed, 16 terminated)
%        \item Number of Snapshots: 616
%    \end{itemize}
%\end{frame}
%%----------------------------------
%\begin{frame}
%    \frametitle{Summaries: Trial Durations}
%    \begin{figure}
%        \includegraphics[height=0.8\textheight]{../assets/img/2023-04-12_durations_hist.png}
%        \label{FIG:durations}
%        \caption{Trial Durations (days)}
%    \end{figure}
%\end{frame}
%%----------------------------------
%\begin{frame}
%    \frametitle{Summaries: snapshots}
%    \begin{figure}
%        \includegraphics[height=0.8\textheight]{../assets/img/2023-04-12_snapshots_hist.png}
%        \label{FIG:snapshots}
%        \caption{Number of Snapshots per matched trial}
%    \end{figure}
%\end{frame}
%%----------------------------------
%\begin{frame}
%    \frametitle{Summaries: snapshots}
%    \begin{figure}
%        \includegraphics[height=0.8\textheight]{../assets/img/2023-04-12_status_duration_snapshots_points.png}
%        \label{FIG:snapshot_duration_scatter}
%        \caption{Scatterplot of snapshot count and durations}
%    \end{figure}
%\end{frame}
%%-------------------------------
%\begin{frame}
%    \frametitle{Questions?}
%
%\end{frame}
%-------------------------------
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%% Analysis %%%%%%%%%%%%%%%%%%%%%%%%
\section{Analysis}
% TOC
% - Review questions and datasets to use for each
% - 
% - 
%-------------------------------------------------------------------------------------
%-------------------------------
\begin{frame}
    \frametitle{General Approach}

    \begin{itemize}
        \item Logistic model
        \item Bayesian Hierarchal model
            \begin{itemize}
                \item Allows for transfer of probability between groups
            \end{itemize}
    \end{itemize}
    
\end{frame}

%-------------------------------
\begin{frame}
    \frametitle{Total Effects Model}
    \begin{align}
        y_i &\sim \text{Bernoulli}(p_i) \\
        p_i &= \text{logistic}(X_i \vec\beta_{c(i)}) \\
        \vec\beta_{c(i)} &\sim \text{MvNormal}(\vec\mu,\vec\sigma I)
    \end{align}
    %TODO: describe X\beta
\end{frame}

%-------------------------------
\begin{frame}
    \frametitle{Analysis}

    Review:
    \begin{itemize}
        \item Hyperparameters
        \item Parameters of Interest
        \item Distrbution of Predicted Differences (Hypothetical Causal Intervention)
    \end{itemize}

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Questions?}

\end{frame}
%--------------------------------
%--------------------------------
%%%%%%%%%%%%%%%%%%%% Results
\subsection{Results}
%--------------------------------
%--------------------------------
\subsubsection{Total Effect}
% - Review Parameter Values
%    - hyperparameters
%        - Table of MLE
%        - Distributions
%    - betas
%        - Table of MLE
%        - Distributions
% - Review Posterior Prediction for interventions
%--------------------------------
%-------------------------------
\begin{frame}
    \frametitle{Results}
    Because Bayesian estimation is typically done numerically, we will first 
    validate convergence.

    Then we will take a look at preliminary results.

    Sampling details

    %TODO: Update
    \begin{itemize}
        \item 4 chains
        \item 2,500 warm-up, 2,500 sampling runs
        \item seed = 11021585
    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Questions?}

\end{frame}
%%-------------------------------
%%--------------------------------
%%\subsubsection{Direct Effects}
%%% - Review Parameter Values
%%%    - hyperparameters
%%%        - Table of MLE
%%%        - Distributions
%%%    - betas
%%%        - Table of MLE
%%%        - Distributions
%%% - Review Posterior Prediction for interventions
%%%--------------------------------
%%%-------------------------------
%%\begin{frame}
%%    \frametitle{Convergence}
%%    Sampling details
%%    %TODO: UPDATE
%%    \begin{itemize}
%%        \item 6 chains
%%        \item 2,500 warm-up, 2,500 sampling runs
%%        \item seed = 11021585
%%    \end{itemize}
%%\end{frame}
%%%-------------------------------
%%\begin{frame}
%%    \frametitle{Questions?}
%%
%%\end{frame}
%%-------------------------------
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Conclusion %%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
%-------------------------------------------------------------------------------------

%-------------------------------
\begin{frame}
    \frametitle{Proposed improvements}
    \begin{enumerate}
        \item Match more trials to ICD-10 codes and Formularies
        \item Add more formularies
        \item Remove disease categories that don't exist in the data from the priors
        \item Imputing Enrollment
    \end{enumerate}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Summary}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Final Questions}

    \center{\huge{Time is yours to ask any remaining questions.}}
\end{frame}
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Appendicies %%%%%%%%%%%%%%%%%%%%%%%%
\section{Appendices}
%-------------------------------------------------------------------------------------
%----------------------------------
%%%%%%%%%%%%%%%%%%%% Convergence Tests
\subsection{Convergence}
%----------------------------------
%-------------------------------
\begin{frame}
    \frametitle{Warnings}
    %TODO: UPDATE

    \begin{itemize}
        \item There were no diverging transitions.
        \item There were 15,000 transitions that exceeded max treedepth. 
            Sampling efficiency is poor.
        \item All chains had low Bayesian Fraction of Missing Information. 
            Some areas of the distribution were poorly explored.
        \item R-hat = $1.23$, ideal is around 1, chains did not mix well.
        \item Bulk and Tail Effective Sample sizes were low, 
            suggesting mean and variance/quantile estimates will be unreliable.
    \end{itemize}
    \cite{mc-stan}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Convergence: Mu}
    \begin{figure}
        %TODO: UPDATE
        %\includegraphics[height=0.9\textheight]{../assets/img/2023-04-11_mu_points.png}
        \label{FIG:caption}
        \caption{Hyperparameter Points Plots: Mu}
    \end{figure}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Convergence: Sigma}
    \begin{figure}
        %TODO: UPDATE
        %\includegraphics[height=0.9\textheight]{../assets/img/2023-04-11_mu_points.png}
        \label{FIG:caption}
        \caption{Hyperparameter Points Plots: Sigma}
    \end{figure}
\end{frame}
%-------------------------------
\begin{frame}[allowframebreaks]
    \frametitle{Bibliography}
    \printbibliography
\end{frame}
%-------------------------------
\end{document} 
%=========================================
%\begin{frame}
%    \frametitle{MarginalRevenue}
%    \begin{figure}
%        \tikzfig{../Assets/owned/ch8_MarginalRevenue}
%        \includegraphics[height=\textheight]{../Assets/copyrighted/KrugmanObsterfeldMeliz_fig8-7.jpg}
%        \label{FIG:costs}
%        \caption{Average Cost Curve as firms enter.}
%    \end{figure}
%\end{frame}
%-------------------------------
%\begin{frame}
%    \frametitle{Columns}
%     \begin{columns}
%        \begin{column}{0.5\textwidth}
%        \end{column}
%        \begin{column}{0.5\textwidth}
%               \begin{figure}
%                   \tikzfig{../Assets/owned/ch7_EstablishedAdvantageExample2}
%                   \label{FIG:costs}
%                   \caption{Setting the Stage}
%               \end{figure}
%        \end{column}
%     \end{columns}
%\end{frame}
% %---------------------------------------------------------------
