%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Beamer Presentation
% LaTeX Template
% Version 1.0 (10/11/12)
%
% This template has been downloaded from:
% http://www.LaTeXTemplates.com
%
% License:
% CC BY-NC-SA 3.0 (http://creativecommons.org/licenses/by-nc-sa/3.0/)
%
% Changed theme to WSU by William King
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%----------------------------------------------------------------------------------------
%    PACKAGES AND THEMES
%----------------------------------------------------------------------------------------

\documentclass[xcolor=dvipsnames,aspectratio=169]{beamer}


%Import Preamble bits
\input{../assets/preambles/FormattingPreamble.tex}
\input{../assets/preambles/TikzitPreamble.tex}
\input{../assets/preambles/MathPreamble.tex}
\input{../assets/preambles/BibPreamble.tex}
\input{../assets/preambles/GeneralPreamble.tex}




%----------------------------------------------------------------------------------------
%    TITLE PAGE
%----------------------------------------------------------------------------------------

\title[Clinical Trials]{The Effects of Market Conditions on Recruitment and Completion of Clinical Trials} 

\author{Will King} % Your name
\institute[WSU] % Your institution as it will appear on the bottom of every slide, may be shorthand to save space
{
Washington State University \\ % Your institution for the title page
\medskip
\textit{william.f.king@wsu.edu} % Your email address
}
\date{\today} % Date, can be changed to a custom date





\begin{document}
\begin{frame}
\titlepage % Print the title page as the first slide
\end{frame}
%----------------------------------
\begin{frame} %Allow frame breaks
\frametitle{Clinical Trials} % Table of contents slide, comment this out to remove it
    % - Intro and hook (Clinical Trials are key part of pharmacological pipeline)
    Pharmaceuticals are a frequently discussed aspect of health care cost management.
    Their development is dictated by scientific and regulatory hurdles 
    including passing clinical trials
    (\cite{noauthor_fda_nodate}), 
    while their market is characterized by strategic competition and ambiguous 
    patent protection
    (\cite{van_der_gronde_addressing_2017}).

    \vspace{12pt}

    This research investigates the pathways by which market conditions
    affect clinical trial completion.
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{This research}
    \textbf{Questions:} 
    \begin{enumerate}
        \item Does the existence of alternative drugs on the market make it
            harder for clinical trials to complete successfully? 
        \item How much of this is occurs due to increased recruitment difficulty?
    \end{enumerate}
    
\end{frame}
%--------------------------------
\begin{frame}
\frametitle{Thanks} % Table of contents slide, comment this out to remove it
    Thanks to Chris Adams and Rebecca Sachs of the Congressional Budget Office.
\end{frame}
%--------------------------------
\begin{frame}[allowframebreaks] %Allow frame breaks
\frametitle{Overview} % Table of contents slide, comment this out to remove it
\tableofcontents 
% - Intro and hook
% - Literature review
% - Causal Identification
% - Data
% - Econometric model
% - Results
% - Improvements
\end{frame}
%-------------------------------

%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Introduction and Background %%%%%%%%%%%%%%%%%%%%%%%%
\section{Background}
% TOC
% - Background on drug process
% - Literature on clinical trials
% - My questions
%-------------------------------------------------------------------------------------

%-------------------------------
\begin{frame}
    \frametitle{Clinical Trials and Drug develoment}
    % add info about trials
    % - Requirements (pre registered design [2007], updated "regularly" on clinicaltrials.gov)
    % - Phases (1,2,3,4, mixed)
    % - Safety and Ethicas (oversight boards, restrictions on payments)
    % - Approval processes (biologics vs small-molecule)
    % add info about drugs

    The FDA requires clinical trials before approving new drug compounds
    \begin{itemize}
        \item Pre-registered design
        \item Updated regularly on clinicaltrials.gov
        \item Often requires an oversight board.
        \item Goal is to prove efficacy and safety of a compound/dosage/route.
        \item A new drug candidate (NDC) must complete 3 phases of clinical trials before approval.
        \item Phases are reviewed with FDA.
        \item Not all clinical trials are for new drugs.
    \end{itemize}
\end{frame}
%-----------------------------
\begin{frame}
    \frametitle{Literature Highlights}
    \begin{itemize}
        \item \cite{van_der_gronde_addressing_2017}: 
            High level synthesis of overall discussion regarding drug costs. 
            Both academic and non-academic sources.
        \item \cite{hwang_failure_2016}:
            Answered the question "Why do late-stage (phase III) trials fail?"
            Found that efficacy, safety, and competition reasons accounted for 
            57\%, 17\%, and 22\% respectively.
        \item \cite{abrantes-metz_pharmaceutical_2004}:
            Described how drugs progress through the 3 phases of clinical trials
            and correlations between various trial characteristics and the 
            clinical trial failures.
        \item \cite{khmelnitskaya_competition_2021}: 
            Modeled clinical trial life-cycle of drugs, found method to separate
            scientific from competitive reasons for failure to progress to the 
            next phase.
%        \item \cite{}: 

    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{This research, in context}

    In contrast to previous work looking at multiple phases of trials,
    I seek to figure out what causes individual trials to fail.

    \vspace{12pt}

    Instead of focusing on the drug development pipeline, I attempt to 
    investigate the population of drug-based, phase III trials.
\end{frame}
%-------------------------------
\begin{frame} %Allow frame breaks
\frametitle{Why this approach?} % Table of contents slide, comment this out to remove it

    \begin{figure}
        \includegraphics[height=0.8\textheight]{../assets/img/methodology_trial.png}
        \label{FIG:xkcd2726}
        \caption{``If you think THAT'S unethical, you should see the stuff we approved via our Placebo IRB.'' 
        - \url{https://xkcd.com/2726}
        }
    \end{figure}
\end{frame}
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Causal Identification / DGP%%%%%%%%%%%%%%%%%%%%%%%%
\section{Causal Model}
% Data Generating process
% - Agents and their decisions
% - Factors that influence each decision
% - 
% - 
%-------------------------------------------------------------------------------------
%-------------------------------
\begin{frame}
    \frametitle{Data Generating Process}
    % study sponsors
    Study Sponsors Decide to start a Phase 3 trial and whether to terminate it.
    \\
    They ask themselves:
    \begin{itemize}
        \item Do safety incidents require terminating a trial?
        \item Do efficacy results indicate the trial is worth continuing?
        \item Is recruiting sufficient to achieve our results and contain costs?
        \item Do expectations about future returns justify our expenditures?
    \end{itemize}
    
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Data Generating Process}
    % participants
    Participants decide to enroll (and dis-enroll) themselves in a trial based 
    \begin{itemize}
        \item Disease severity
        \item Relative safety/efficacy compared to other treatments
    \end{itemize}

    Study sponsors plan their enrollment considering
    \begin{itemize}
        \item Total population affected
        \item Likely participant response rates
    \end{itemize}

\end{frame}
%-----------------------------
\begin{frame}
    \frametitle{Questions of Interest}

    \begin{itemize}
        \item How do the competitors on the market affect clinical trial completion?
        \item How is this effect moderated by the enrollment of participants?
    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Audience Questions}

    \center{What can I clarify?}
\end{frame}
%-------------------------------
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Causality and Data %%%%%%%%%%%%%%%%%%%%%%%%
\section{Causal Story and Data}
% TOC
% - Causal Story (no subsection)
%    - Clinical trials: targets specific drug/condition combination.
%    - Enrollment process: patients counsel with providers
%    - Trials terminate if unsafe, ineffective, unprofitable, or cannot enroll patients
%    - Ethical concerns exist throughout.
%    - This is complicated by the fact that the experiment reveals information over time.
% - Formalization
% - Data Sources
%-------------------------------------------------------------------------------------

%-------------------------------
\begin{frame}[shrink=10] %evil option is helpful here.
    \frametitle{How do clinical trials proceed?}
    \begin{columns}[T]
    \begin{column}{0.5\textwidth}
    What does a complete trial look like.
    \begin{enumerate}
        \item Study sponsor comes up with design
        \item Apply for NCT ID from ClinicalTrials.gov
        \item Begin enrolling participants
        \item Update ClinicalTrials.gov to recruit
        \item Close Enrollment
        \item Update ClinicalTrials.gov as not recruiting*
        \item Reach primary objectives
        \item Update ClinicalTrials.gov as complete
        \item Reach secondary objectives
        \item Update ClinicalTrials.gov with more information
    \end{enumerate}
    \end{column}
    \begin{column}{0.5\textwidth}
    What does an incomplete trial look like?
    \begin{enumerate}
        \item Study sponsor comes up with design
        \item Apply for NCT ID from ClinicalTrials.gov
        \item Begin enrolling participants
        \item Update ClinicalTrials.gov to advertise
        \item Run into issues: 
            \begin{itemize}
                \item Safety
                \item Efficacy
                \item Profitability
                \item Feasiblity (enrollment, PI leaves, etc.)
            \end{itemize}
        \item Close Enrollment*
        \item Decide to terminate clinical trial.
        \item Update ClinicalTrials.gov as terminated.
    \end{enumerate}
    \end{column}
    \end{columns}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{ClinicalTrials.gov}
    Thus ClinicalTrials.gov becomes an (append only) repository of 
    the ``current'' status of clincal trials.


    As it is designed to help faciltate enrollment in clinical trials, 
    the record includes important information such as 

    \begin{itemize}
        \item drugs
        \item study arms
        \item conditions
        \item expected and final enrollment figures
        \item current status
    \end{itemize}

    ClinicalTrials.gov also reports the history from previous
    updates.
\end{frame}

%-------------------------------
\begin{frame}
    \frametitle{Questions?}

\end{frame}
%-------------------------------
%--------------------------------
%%%%%%%%%%%%%%%%%%%% Causal Formalization
\subsection{Formalization}
% - Introduce basic triangle
%    - discuss total vs direct effects
%    - 
% - Add confounders and controls
% - Introduce backdoor criterion
%--------------------------------

%-------------------------------
\begin{frame}
    \frametitle{Framing my Questions}
    Two potential causes of trial termination include 
    \begin{enumerate}
        \item Alternative (competitor) treatments exist
            \begin{itemize}
                \item reduces future profitability.
                \item reduces incentives to enroll as participants.
            \end{itemize}
        \item It can be difficult to recruit patients
            \begin{itemize}
                \item Are there few patients?
                \item Are potential participants choosing other alternatives?
            \end{itemize}
    \end{enumerate}
    Overall this can be described graphically as:

    INSERT IMAGE OF 4 NODES HERE


\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Causal Identification: Backdoor Criterion}
    %Discuss the two different effects: total effect, direct effects

    \begin{columns}
    \begin{column}{0.5\textwidth}
        Total Effect of Competitors

        INSERT TOTAL EFFECT GRAPH
    \end{column}
    \begin{column}{0.5\textwidth}
        Direct Effects of Competitors and Enrollment

        INSERT DIRECT EFFECT GRAPH
    \end{column}
    \end{columns}
\end{frame}

%-------------------------------
\begin{frame}
    \frametitle{Rephrasing Questions}
    To rephrase my questions
    \begin{enumerate}
        \item How large is the total effect of increasing the number of competing drugs on completing clinical trials?
        \item How large is the direct effect of increasing the number of competing drugs on completing clincial trials?
    \end{enumerate}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Additional Concerns}
    %Confounders 
    Of course, there are other confounding relationships
    \begin{enumerate}
        \item Population Effects
        \item Fundamental Safety and Efficacy of compound/dosage/route
    \end{enumerate}

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Complete graph}
    %introduce backdoor criterion

    INSERT COMPLETE GRAPH HERE

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Causal Identification: Backdoor Criterion}
    %introduce backdoor criterion

    \cite{PEARLYYYY} developed a method of verifying causal identification from DAGs like the one I presented.

    Of particular interest is the rule called the Backdoor criterion:

    INSERT DESCRIPTION OF THE BACK DOOR CRITERION    


\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Sufficent Adjustment Set}
    %introduce backdoor criterion

    INSERT COMPLETE GRAPH HERE with adjustment set highlighted

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Sufficent Adjustment Set}
    %introduce backdoor criterion

    Thus the required adjustment set includes:

    \begin{itemize}
        \item Population Measures
        \item Safety and Efficacy Measures
        \item INSERT MORE
    \end{itemize}


\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Questions?}

\end{frame}
%-------------------------------
%--------------------------------
%%%%%%%%%%%%%%%%%%%% Data sources
\subsection{Data Sources}
% TOC
% - Main Data Sources
%    - ClinicalTrials.gov and AACT
%    - IHME Burden of Disease
%    - Marketing Data
%    - MeSH, RxNorm/RxNav
% - How did I Link Data Sources
% - Data Sizes
%--------------------------------

%-------------------------------
\begin{frame}
    \frametitle{Data Sources}
    %TODO: add citations
    Data sources
    \begin{itemize}
        \item ClinicalTrials.gov
            \begin{itemize}
                \item AACT-CTTI
                \item Scraping historical snapshots
            \end{itemize}
        \item ICD-10 (CMS and WHO)
        \item IHME Global Burden of Disease
        \item Structured Product Labels
        \item USP Drug Classification
        \item Drugs@FDA: RxNav / RxNorm / MeSH
    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Linking data}
    %
    The following linking process was used:
    \begin{enumerate}
        \item AACT trials to snapshots (internal ID)
        \item AACT trials to ICD-10 (hand match)
        \item ICD-10 to IHME (IHME)
        \item Snapshots to drug brands (RxNorm/RxNav/MeSh, SPL)
        \item AACT to USP DC alternates (RxNorm, USP DC, hand match)
    \end{enumerate}

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Measures of Causes and Effects}
    
    \begin{itemize}
        \item Final Status: Measured from AACT - status when trial is over.
        \item Competitors on Market: Measured by the number of drugs
            \begin{itemize}
                \item with same active ingredients (at the time of the snapshot)
                \item sharing the USP DC category and class (in 2023)
            \end{itemize}
        \item Enrollment: Measured by enrollment status at the snapshot level.
    \end{itemize}

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Adjustment set}
    \begin{itemize}
        \item Population Measures
            \begin{itemize}
                \item IHME Global Disease Burden: QUALYs, spread over 5 levels of the Social Development Index
            \end{itemize}
        \item Beliefs about safety \& efficacy: Restricted to Phase 3 trials.
        \item Disease Type: Hierarchal parameters in model
    \end{itemize}

    Note the implicit conditioning on trials treating diseases with IHME data\footnote{
        IHME does not track data for W61.62XD: Struck by duck, subsequent encounter
    }.

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Other Details}

    Other Trial Selection Criteria
    \begin{itemize}
        \item Interventional Study
        \item Involved an FDA Regulated Drug
        \item Phase 3 trial
        \item Started after 2010-01-01
        \item Ended before 2022-01-01
    \end{itemize}

\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Questions?}

\end{frame}
%-------------------------------
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Analysis %%%%%%%%%%%%%%%%%%%%%%%%
\section{Analysis}
% TOC
% - Review questions and datasets to use for each
% - 
% - 
%-------------------------------------------------------------------------------------

%-------------------------------
\begin{frame}
    \frametitle{Questions?}

\end{frame}
%--------------------------------
%%%%%%%%%%%%%%%%%%%% Econometric Model
\subsection{Econometric Model}
% - Present model per effect
% - 
% - 
%--------------------------------

%-------------------------------
\begin{frame}
    \frametitle{Questions?}

\end{frame}
%--------------------------------
%%%%%%%%%%%%%%%%%%%% Results
\subsection{Results}
%--------------------------------
%--------------------------------
\subsubsection{Total Effect}
% - Review Parameter Values
%    - hyperparameters
%        - Table of MLE
%        - Distributions
%    - betas
%        - Table of MLE
%        - Distributions
% - Review Posterior Prediction for interventions
%--------------------------------
%-------------------------------
\begin{frame}
    \frametitle{Questions?}

\end{frame}
%-------------------------------
%--------------------------------
\subsubsection{Direct Effects}
% - Review Parameter Values
%    - hyperparameters
%        - Table of MLE
%        - Distributions
%    - betas
%        - Table of MLE
%        - Distributions
% - Review Posterior Prediction for interventions
%--------------------------------

%-------------------------------
\begin{frame}
    \frametitle{Questions?}

\end{frame}
%-------------------------------
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Conclusion %%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
%-------------------------------------------------------------------------------------

%-------------------------------
\begin{frame}
    \frametitle{Final Questions}

    \center{\huge{Time is yours to ask any remaining questions.}}
\end{frame}
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Appendicies %%%%%%%%%%%%%%%%%%%%%%%%
\section{Appendices}
%-------------------------------------------------------------------------------------

%-------------------------------
\begin{frame}
    \frametitle{Data Generating Process}
    % Trial Snapshots and dependencies.
    During a trial, the study sponsor reports snapshots of their trial.
    This includes updates to:

    \begin{itemize}
        \item enrollment (actual or anticipated)
        \item current recruitment status (Recruiting, Active not recruiting, etc)
        \item study sponsor
        \item planned completion dates
        \item elapsed duration
    \end{itemize}

    Note that final enrollment and the final status (Completed or Terminated) 
    of the trial are jointly determined.
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Causal Diagram: Key Pathways}
    % Estimating Direct vs Total Effects
    \begin{figure}
        \resizebox{!}{0.5\textheight}{
            \tikzfig{../assets/tikzit/CausalGraph}
        }
        \label{FIG:CausalDiagram}
        \caption{Causal Diagram highlighting direct and total pathways}
    \end{figure}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Causal Diagram: Backdoor Criterion}
    \small
    \begin{block}{$d$-separation}
        A set $S$ of nodes blocks a path $p$ if either
        \begin{enumerate}
            \item $p$ contains at least one arrow-emitting node in $S$
            \item $p$ contains at least one collision node $c$ that is outside $S$ 
                and has no descendants in $S$.
        \end{enumerate}
        If $S$ blocks all paths from X to Y, then it is said to ``$d$-separate'' 
        $X$ and $Y$, and then $X \perp Y | S$.
    \end{block}
    \begin{block}{Back-Door Criterion}
        A set $S$ of covariates is admissible as controls on the 
        causal relationship $X \rightarrow Y$ if:
        \begin{enumerate}
            \item No element of $S$ is a descendant of $X$
            \item The elements of $S$ d-separate all paths from $X$ to $Y$ that include
                parents of $X$.
        \end{enumerate}
    \end{block}
    \cite{pearl_causality_2000}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Causal Diagram}
    Key takeaways
    \begin{itemize}
        \item Measuring enrollment prior to trial completion is necessary for causal identification.
        \item The backdoor criterion gives us the following adjustment sets:
            \begin{itemize}
                \item Total Effect for Market on Termination; Population, Condition, Phase III
                \item Direct Effects for Enrollment, Market on Termination; Population, Condition Phase III, 
                    Elapsed Duration, Planned Enrollment
            \end{itemize}
        \item Enrollment requires imputation
    \end{itemize}
\end{frame}
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Data %%%%%%%%%%%%%%%%%%%%%%%%
\section{Data}
%-------------------------------------------------------------------------------------
%----------------------------------
%%%%%%%%%%%%%%%%%%%% Sources
\subsection{Sources}
%----------------------------------
%-------------------------------
\begin{frame} %Allow frame breaks
    \frametitle{Data Sources}
    \begin{itemize}
        \item ClinicalTrials.gov - AACT \& custom scripts
            \begin{itemize}
                \item Select trials of interest
                \item Trial details: 
                    \begin{itemize}
                        \item conditions
                        \item final status
                        \item drugs/interventions
                    \end{itemize}
                \item Trial snapshots:
                    \begin{itemize}
                        \item enrollment (anticipated, planned, or actual)
                        \item elapsed duration
                        \item current status
                    \end{itemize}
            \end{itemize}
        \item Medical Subject Headings (MeSH) Thesaurus
            \begin{itemize}
                \item A standardized nomenclature used to classify interventions 
                    and conditions in the clinical trials database.
            \end{itemize}
    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame} %Allow frame breaks
    \frametitle{Data Sources}
    \begin{itemize}
        \item NSDE Files (New drug code Structured product labels Data Element)
            \begin{itemize}
                \item Contains information about when a given drug was on the market.
            \end{itemize}
        \item RxNorm
            \begin{itemize}
                \item Links pharmaceuticals between MeSH standardized terms and 
                    NSDE files.
            \end{itemize}
        \item Global Disease Burden Survey (2019)
            \begin{itemize}
                \item Estimates of DALYs for categories of disease
                \item Links of Categories to ICD-10 Codes
            \end{itemize}
        \item ICD-10 (2019)
            \begin{itemize}
                \item WHO version
                \item CMS version (Clinical Management)
                \item Used to group disease conditions in hierarchical model
            \end{itemize}
        \item Unified Medical Language System Thesaurus
            \begin{itemize}
                \item Used to link MeSH standardized terms and ICD-10 conditions
                \item Manual matching process
            \end{itemize}
    \end{itemize}
\end{frame}
%----------------------------------
%%%%%%%%%%%%%%%%%%%% Integration
\subsection{Integration}
%----------------------------------
%-------------------------------
\begin{frame}
    \frametitle{Data Summaries}
    %put summaries now
    \begin{itemize}
        \item Number of Phase III, FDA monitored Drug Trials: 1,981
        \item Number of Trials matched to ICD-10: 186
        \item Number of Trials matched to ICD-10 with population measures: 67 
            (51 completed, 16 terminated)
        \item Number of Snapshots: 616
    \end{itemize}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Data used}
    The following data points were used.
    \begin{itemize}
        \item elapsed duration 
        \item asinh(number of brands)
        \item asinh(high sdi DALY estimate)
        \item asinh(high-medium sdi DALY estimate)
        \item asinh(medium sdi DALY estimate)
        \item asinh(low-medium sdi DALY estimate)
        \item asinh(low sdi DALY estimate)
    \end{itemize}
    The asinh operator was used because it parallels $\text{ln}(x)$ for 
    large values of $x$ but also handles $\text{asinh}(0)=0$.
\end{frame}
%----------------------------------
\begin{frame}
    \frametitle{Summaries: Trial Durations}
    \begin{figure}
        \includegraphics[height=0.8\textheight]{../assets/img/2023-04-12_durations_hist.png}
        \label{FIG:durations}
        \caption{Trial Durations (days)}
    \end{figure}
\end{frame}
%----------------------------------
\begin{frame}
    \frametitle{Summaries: snapshots}
    \begin{figure}
        \includegraphics[height=0.8\textheight]{../assets/img/2023-04-12_snapshots_hist.png}
        \label{FIG:snapshots}
        \caption{Number of Snapshots per matched trial}
    \end{figure}
\end{frame}
%----------------------------------
\begin{frame}
    \frametitle{Summaries: snapshots}
    \begin{figure}
        \includegraphics[height=0.8\textheight]{../assets/img/2023-04-12_status_duration_snapshots_points.png}
        \label{FIG:snapshot_duration_scatter}
        \caption{Scatterplot of snapshot count and durations}
    \end{figure}
\end{frame}
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Econometric Model %%%%%%%%%%%%%%%%%%%%%%%%
\section{Econometric model}
%-------------------------------------------------------------------------------------
%-------------------------------
\begin{frame}
    \frametitle{Econometric Model}
    Estimating the total effect of brands on market
    \begin{align}
        y_n &\sim \text{Bernoulli}(p_n) \\
        p_n &= \text{logisticfn}(x_n * \beta(d_n)) \\
        \beta_k(d) &\sim \text{Normal}(\mu_k, \sigma_k) \\
        \mu_k &\sim \text{Normal}(0,1) \\
        \sigma_k &\sim \text{Gamma}(2,1)
    \end{align}
    $k$ indexes parameters and $d_n$ represents the ICD-10 group the trial corresponds to.
\end{frame}
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Results %%%%%%%%%%%%%%%%%%%%%%%%
\section{Results}
%-------------------------------------------------------------------------------------
%-------------------------------
\begin{frame}
    \frametitle{Results}
    Because Bayesian estimation is typically done numerically, we will first 
    validate convergence.

    Then we will take a look at preliminary results.

    Sampling details
    \begin{itemize}
        \item 6 chains
        \item 2,500 warm-up, 2,500 sampling runs
        \item seed = 11021585
    \end{itemize}
\end{frame}
%----------------------------------
%%%%%%%%%%%%%%%%%%%% Convergence Tests
\subsection{Convergence}
%----------------------------------
%-------------------------------
\begin{frame}
    \frametitle{Warnings}

    \begin{itemize}
        \item There were no diverging transitions.
        \item There were 15,000 transitions that exceeded max treedepth. 
            Sampling efficiency is poor.
        \item All chains had low Bayesian Fraction of Missing Information. 
            Some areas of the distribution were poorly explored.
        \item R-hat = $1.23$, ideal is around 1, chains did not mix well.
        \item Bulk and Tail Effective Sample sizes were low, 
            suggesting mean and variance/quantile estimates will be unreliable.
    \end{itemize}
    \cite{mc-stan}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Convergence: Mu}
    \begin{figure}
        \includegraphics[height=0.9\textheight]{../assets/img/2023-04-11_mu_points.png}
        \label{FIG:caption}
        \caption{Hyperparameter Points Plots: Mu}
    \end{figure}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Convergence: Sigma}
    \begin{figure}
        \includegraphics[height=0.8\textheight]{../assets/img/2023-04-11_sigma_points.png}
        \label{FIG:caption}
        \caption{Hyperparameter Points Plots: Sigma}
    \end{figure}
\end{frame}
%----------------------------------
%%%%%%%%%%%%%%%%%%%% Preliminary Results
\subsection{Preliminary Results}
%----------------------------------
%-------------------------------
\begin{frame}
    \frametitle{Preliminary Results: Mu}

    \begin{columns}
        \begin{column}{0.3\textwidth}
            \begin{enumerate}
                \item elapsed duration 
                \item asinh(n\_brands)
                \item asinh(high sdi)
                \item asinh(high-medium sdi)
                \item asinh(medium sdi)
                \item asinh(low-medium sdi)
                \item asinh(low sdi)
            \end{enumerate}
        \end{column}
        \begin{column}{0.7\textwidth}
            \begin{figure}
                \includegraphics[height=0.8\textheight]{../assets/img/2023-04-11_mu_dist.png}
                \label{FIG:caption}
                \caption{Hyperparameter Distribution: Mu}
            \end{figure}
        \end{column}
    \end{columns}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Preliminary Results: Sigma}

    \begin{figure}
        \includegraphics[height=0.8\textheight]{../assets/img/2023-04-11_sigma_dist.png}
        \label{FIG:caption}
        \caption{Hyperparameter Distribution: Sigma}
    \end{figure}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Interpretation}
    All of the following interpretations are done in the context of insufficient data

    \begin{enumerate}
        \item Elapsed Duration (Mu[1]): Trending Negative, reduced probability of termination.
        \item Number of Brands(Mu[2]): Trending Positive, increased probability of termination.
        \item Population Measures (Mu[3]-Mu[7])
            \begin{enumerate}
                \item What is most surprising is that these are both positive and negative.
                    Probably need more data.
            \end{enumerate}
        \item It is surprising to see the wide distribution in sigma values.
    \end{enumerate}
\end{frame}
%-------------------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%% Improvements %%%%%%%%%%%%%%%%%%%%%%%%
\section{Improvements}
%-------------------------------------------------------------------------------------
%-------------------------------
\begin{frame}
    \frametitle{Proposed improvements}
    \begin{enumerate}
        \item Match more trials to ICD-10 codes
        \item Improve Measures of Market Conditions
        \item Adjust Covariance Structure
        \item Find Reasonable Priors
        \item Remove disease categories that don't exist in the data from the priors
        \item Imputing Enrollment
        \item Improve Population Estimates
    \end{enumerate}
\end{frame}
%-------------------------------
\begin{frame}
    \frametitle{Questions?}
    \center{\huge{Questions?}}

\end{frame}
%-------------------------------
\begin{frame}[allowframebreaks]
    \frametitle{Bibliography}
    \printbibliography
\end{frame}
%-------------------------------
\end{document} 
%=========================================
%\begin{frame}
%    \frametitle{MarginalRevenue}
%    \begin{figure}
%        \tikzfig{../Assets/owned/ch8_MarginalRevenue}
%        \includegraphics[height=\textheight]{../Assets/copyrighted/KrugmanObsterfeldMeliz_fig8-7.jpg}
%        \label{FIG:costs}
%        \caption{Average Cost Curve as firms enter.}
%    \end{figure}
%\end{frame}
%-------------------------------
%\begin{frame}
%    \frametitle{Columns}
%     \begin{columns}
%        \begin{column}{0.5\textwidth}
%        \end{column}
%        \begin{column}{0.5\textwidth}
%               \begin{figure}
%                   \tikzfig{../Assets/owned/ch7_EstablishedAdvantageExample2}
%                   \label{FIG:costs}
%                   \caption{Setting the Stage}
%               \end{figure}
%        \end{column}
%     \end{columns}
%\end{frame}
% %---------------------------------------------------------------