Nearly finished interface to manually review icd-10 codes.

Scripts/DevelopingLinks.sql

@@ -0,0 +1,41 @@
+drop table if exists "DiseaseBurden".trial_to_icd10;
+
+CREATE TABLE "DiseaseBurden".trial_to_icd10 (
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
+    nct_id varchar NOT NULL,
+    "condition" varchar NOT NULL,
+    ui varchar NULL,
+    uri varchar NULL,
+    rootsource varchar NULL,
+    "name" varchar NULL,
+    CONSTRAINT trial_to_icd10_pk PRIMARY KEY (id)
+);
+
+
+
+
+DROP TABLE if exists "DiseaseBurden".icd10_to_cause;
+ 
+CREATE TABLE "DiseaseBurden".icd10_to_cause (
+    id SERIAL NOT NULL ,
+    code varchar NOT NULL,
+    cause_text varchar NOT NULL,
+    CONSTRAINT icd10_to_cause_pk PRIMARY KEY (id)
+);
+
+
+
+drop table if exists "DiseaseBurden".match_status;
+
+create type "DiseaseBurden".validation_type as enum ('accepted', 'rejected', 'unmatched');
+comment on type "DiseaseBurden".validation_type is 'This is used to record interactions with each type. It can be accepted (yes this should be used), rejected (no this doesn`t match), or unmatched (where non of the proposed options match)';
+
+
+CREATE TABLE "DiseaseBurden".match_status (
+    id serial4 NOT NULL,
+    approved "DiseaseBurden".validation_type NOT NULL,
+    approval_timestamp timestamp NOT NULL,
+    CONSTRAINT match_status_pk PRIMARY KEY (id, approval_timestamp)
+);
+COMMENT ON TABLE "DiseaseBurden".match_status IS 'This allows me to record if a particular proposed match is approved or not.';
+

scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/__init__.py

@@ -0,0 +1,43 @@
+from flask import Flask
+import os
+from dotenv import dotenv_values
+
+
+
+env_path = "../../containers/.env"
+ENV = dotenv_values(env_path)
+
+def create_app(test_config=None):
+    # create and configure the app
+    app = Flask(__name__, instance_relative_config=True)
+    app.config.from_mapping(
+        SECRET_KEY='dev',
+        POSTGRES_DB=ENV["POSTGRES_DB"],
+        POSTGRES_USER=ENV["POSTGRES_USER"],
+        POSTGRES_HOST=ENV["POSTGRES_HOST"],
+        POSTGRES_PORT=ENV["POSTGRES_PORT"],
+        POSTGRES_PASSWORD=ENV["POSTGRES_PASSWORD"],
+    )
+
+
+    # ensure the instance folder exists
+    try:
+        os.makedirs(app.instance_path)
+    except OSError:
+        pass
+
+    # a simple page that says hello
+    @app.route('/')
+    def hello():
+        return 'Hello, World!'
+
+
+    from . import db_interface
+    db_interface.init_database(app)
+
+    from . import validation
+    app.register_blueprint(validation.bp)
+
+    return app
+
+

scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/db_interface.py

@@ -0,0 +1,160 @@
+import psycopg2 as psyco
+from psycopg2 import extras
+
+import click #used for cli commands. Not needed for what I am doing.
+from flask import current_app, g
+
+def get_db(**kwargs):
+
+    if "db" not in g:
+        g.db = psyco.connect(
+            dbname=current_app.config["POSTGRES_DB"]
+            ,user=current_app.config["POSTGRES_USER"]
+            ,host=current_app.config["POSTGRES_HOST"]
+            ,port=current_app.config["POSTGRES_PORT"]
+            ,password=current_app.config["POSTGRES_PASSWORD"]
+            ,**kwargs
+        )
+    return g.db
+
+def close_db(e=None):
+    db = g.pop('db', None)
+
+    if db is not None:
+        db.close()
+
+def check_initialization(app):
+    db = get_db()
+    with db.cursor() as curse:
+        curse.execute("select count(*) from \"DiseaseBurden\".trial_to_icd10")
+    curse.fetchall()
+    #just checking if everything is going to fail
+
+def init_database(app):
+    #check_initialization(app)
+    app.teardown_appcontext(close_db)
+
+
+
+
+def select_remaing_trials_to_analyze(db_conn):
+    '''
+    This will get the set of trials that need to be analyzed.
+    '''
+    sql = '''
+    select distinct nct_id 
+    from "DiseaseBurden".trial_to_icd10 tti 
+    where tti.id not in 
+        (
+            select distinct id
+            from "DiseaseBurden".match_status
+        )
+    order by nct_id
+    ;
+    '''
+    with db_conn.cursor() as cursor:
+        cursor.execute(sql)
+        return cursor.fetchall()
+
+
+def select_analyzed_trials(db_conn):
+    '''
+    This will get the set of trials that have been analyzed.
+    '''
+    sql = '''
+    select distinct nct_id
+    from "DiseaseBurden".trial_to_icd10 tti 
+    where tti.id in 
+        (
+            select distinct id
+            from "DiseaseBurden".match_status
+            where approved in ('accepted','rejected')
+        )
+    order by nct_id
+    ;
+    '''
+    with db_conn.cursor() as cursor:
+        cursor.execute(sql)
+        return cursor.fetchall()
+
+def select_unmatched_trials(db_conn):
+    '''
+    This will get the set of trials that have been analyzed.
+    '''
+    sql = '''
+    select distinct nct_id
+    from "DiseaseBurden".trial_to_icd10 tti 
+    where tti.id in 
+        (
+            select distinct id
+            from "DiseaseBurden".match_status
+            where approved = 'unmatched'
+        )
+    order by nct_id
+    ;
+    '''
+    with db_conn.cursor() as cursor:
+        cursor.execute(sql)
+        return cursor.fetchall()
+
+
+def get_trial_conditions_and_proposed_matches(db_conn, nct_id):
+    sql = '''
+        select *
+        from "DiseaseBurden".trial_to_icd10 tti 
+        where nct_id = %s
+        '''
+    with db_conn.cursor() as cursor:
+        cursor.execute(sql,[nct_id])
+        return cursor.fetchall()
+
+
+def store_validation(db_conn, list_of_inserts):
+    sql = """
+    insert into "DiseaseBurden".match_status (id, approved, approval_timestamp)
+    values %s
+    ; 
+    """
+    with db_conn.cursor() as cursor:
+        extras.execute_values(cursor, sql, list_of_inserts)
+    db_conn.commit()
+
+def get_trial_summary(db_conn,nct_id):
+    sql_summary ="""
+select 
+    s.nct_id,
+    brief_title ,
+    official_title ,
+    bs.description as brief_description,
+    dd.description as detailed_description
+from ctgov.studies s 
+    left join ctgov.brief_summaries bs
+        on bs.nct_id = s.nct_id 
+    left join ctgov.detailed_descriptions dd 
+        on dd.nct_id = s.nct_id 
+where s.nct_id = %s
+;
+"""
+    sql_conditions="""
+--conditions mentioned
+select * from ctgov.conditions c 
+where c.nct_id = %s
+;
+"""
+    sql_keywords="""
+select nct_id ,downcase_name 
+from ctgov.keywords k 
+where k.nct_id = %s
+;
+"""
+    with db_conn.cursor() as curse:
+        curse.execute(sql_summary,[nct_id])
+        summary = curse.fetchall()
+
+        curse.execute(sql_keywords,[nct_id])
+        keywords = curse.fetchall()
+
+        curse.execute(sql_conditions,[nct_id])
+        conditions = curse.fetchall()
+
+    return {"summary":summary, "keywords":keywords, "conditions":conditions}

scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/base.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<title>{% block title %}{% endblock %} - ClinicalTrialsProject</title>
+<!--<link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">-->
+
+<nav>
+  <h1>Nav</h1>
+  <ul>
+    <a href="{{ url_for('validation.remaining') }}">Validation Home</a>
+  </ul>
+</nav>
+
+<section class="content">
+  <header>
+    {% block header %}{% endblock %}
+  </header>
+  {% block content %}{% endblock %}
+</section>

scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_index.html

@@ -0,0 +1,48 @@
+{% extends 'base.html' %}
+
+{% block header %}
+  <h1>{% block title %} ICD-10 to Trial Conditions Validation {% endblock %}</h1>
+{% endblock %}
+
+{% block content %}
+
+<h2>Trials to Validate</h2>
+
+<table>
+<th>Trials</th>
+{% for trial in list_to_validate %}
+<tr><td>
+    <a href="{{ url_for('.validate_trial', nct_id=trial[0] ) }}">
+        {{ trial [0] }}
+    </a>
+</td></tr> 
+{% endfor %}
+</table>
+
+<h2>Trials that have been Validated</h2>
+
+<table>
+<th>Trials Links</th>
+{% for trial in validated_list %}
+<tr><td>
+    <a href="{{ url_for('.validate_trial', nct_id=trial[0] ) }}">
+        {{ trial [0] }}
+    </a>
+</td></tr> 
+{% endfor %}
+</table>
+
+<h2>Trials that don't have a good match</h2>
+
+<table>
+<th>Trial Links</th>
+{% for trial in unmatched_list %}
+<tr><td>
+    <a href="{{ url_for('.validate_trial', nct_id=trial[0] ) }}">
+        {{ trial [0] }}
+    </a>
+</td></tr> 
+{% endfor %}
+</table>
+
+{% endblock %}

scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_of_trial.html

@@ -0,0 +1,92 @@
+{% extends 'base.html' %}
+
+{% block header %}
+  <h1> ICD-10 to Trial Conditions Validation: {{ nct_id }} </h1>
+{% endblock %}
+
+{% block content %}
+
+<section class="summary">
+  <h3>Trial Summary</h3>
+
+  <div class="text_summary">
+    <ul>
+      <li>NCT: {{ summary_dats["summary"][0][0] }}</li>
+      <li>Brief Title: {{ summary_dats["summary"][0][1] }}</li>
+      <li>Long Title: {{ summary_dats["summary"][0][2] }}</li>
+      <li>Brief Description: {{ summary_dats["summary"][0][3] }}</li>
+      <li>Long Description: {{ summary_dats["summary"][0][4] }}</li>
+    </ul>
+  </div>
+  <div class="keywords">
+    <h4>Keywords</h4>
+    <ul>
+      {% for keyword in  summary_dats["keywords"] %}
+        <li>
+          {{ keyword[1] }}
+        </li>
+        {% endfor %}
+    </ul>
+  </div>
+  <div class="conditions">
+    <h4>Raw Conditions </h4>
+    <ul>
+      {% for condition in  summary_dats["conditions"] %}
+        <li>
+          {{ condition[3] }}
+        </li>
+        {% endfor %}
+    </ul>
+  </div>
+</section>
+
+<section class="proposed_conditions">
+  <h3>Proposed Conditions</h3>
+  <form method="post">
+    <table>
+      <tr>
+      <th>Approve</th>
+      <th>Condition (MeSH normalized)</th>
+      <th>Identifier</th>
+      <th>Source</th>
+      <th>Description</th>
+      </tr>
+      {% for condition in condition_list %}
+
+      <tr>
+        <td> <input type="checkbox" id="{{ condition[0] }}" name="{{condition[0]}}" value="accepted"> </td>
+        <td> {{condition[2]}} </td>
+        <td> {{condition[3]}} </td>
+        <td> {{condition[5]}} </td>
+        <td> {{condition[6]}} </td>
+      </tr> 
+
+      {% endfor %}
+    </table>
+    <input type="submit" name="submission" value="Submit approvals">
+    <br/>
+    <input type="submit" name="marked_unmatched" value="Mark unmmatched">
+  </form>
+</section>
+
+<section class="submit_alternate">
+  <h3>Submit Alternate Conditions</h3>
+  <!--For each listed condition, provide a spot to enter a ICT10 code-->
+  <form method="post">
+    <label for="alternate_sub">Please Select a code that appears to be the best match:</label>
+    <select name="alt_sub" id="alternate_sub">
+        <option value="" selected disabled>--Please choose an option--</option>
+        {% for code in icd10_codes %}
+        <option value="{{ code }}"> {{ code }} </option>
+        {% endfor %}
+    </select>
+    <br/>
+    <label for="notes">Helpful Notes:</label> 
+    <input type="text" id="notes">
+    <br/>
+    <input type="submit" name="alternate_submission" value="Submit alternate ICD-10 code">
+  </form>
+</section>
+
+
+{% endblock %}

scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/validation.py

@@ -0,0 +1,81 @@
+import functools
+from flask import (Blueprint, flash, g, redirect, render_template, request, session, url_for)
+from Icd10ConditionsMatching.db_interface import (
+    get_db,select_remaing_trials_to_analyze, 
+    select_analyzed_trials,
+    select_unmatched_trials,
+    get_trial_conditions_and_proposed_matches,
+    store_validation,
+    get_trial_summary,
+    )
+from datetime import datetime
+
+#### First Blueprint: Checking Data
+bp = Blueprint("validation", __name__, url_prefix="/validation")
+
+@bp.route("/",methods=["GET"])
+def remaining():
+    db_conn = get_db()
+
+
+    to_validate = select_remaing_trials_to_analyze(db_conn)
+    validated = select_analyzed_trials(db_conn)
+    unmatched_list = select_unmatched_trials(db_conn)
+ 
+    
+    return render_template(
+        "validation_index.html", 
+        list_to_validate=to_validate,
+        validated_list = validated,
+        unmatched_list = unmatched_list
+        )
+
+@bp.route("/<nct_id>", methods=["GET","POST"])
+def validate_trial(nct_id):
+        
+    if request.method == "GET":
+        db_conn = get_db()
+
+        condition_list =  get_trial_conditions_and_proposed_matches(db_conn, nct_id)
+        summary_dats = get_trial_summary(db_conn, nct_id)
+        icd10_codes = [1,2,3]
+
+        return render_template(
+            "validation_of_trial.html", 
+            nct_id=nct_id,
+            condition_list=condition_list,
+            summary_dats=summary_dats,
+            icd10_codes = icd10_codes
+            )
+    elif request.method == "POST":
+        db_conn = get_db()
+
+        list_of_insert_data = []
+        
+        db_conn = get_db()
+
+        condition_list =  get_trial_conditions_and_proposed_matches(db_conn, nct_id)
+
+        print(request.form)
+
+        if "submission" in request.form:
+            #if it is a submission: 
+                #grab all match ids from db
+                #if match id in submitted form, mark as approved, otherwise mark as rejected
+            for condition in condition_list:
+                id = condition[0]
+                list_of_insert_data.append((id, request.form.get(str(id),"rejected"),datetime.now()))
+
+        elif "marked_unmatched" in request.form:
+            #if this was marked as "unmatched", store that for each entry.
+            for condition in condition_list:
+                id = condition[0]
+                list_of_insert_data.append((id, "unmatched",datetime.now()))
+        elif "alternate_submission" in request.form:
+            pass
+        
+        store_validation(db_conn, list_of_insert_data)
+
+        return redirect(url_for("validation.remaining"))
+        
+

scripts/Icd10ConditionsMatching/match_conditions_to_icd10.txt

@@ -0,0 +1,16 @@
+The purpose of this application is to provide an easy way to manually review the 
+proposed matches between trial conditions and proposed icd10 codes.
+
+Plan:
+ - get list of proposed matches For each NCT_ID
+    - present appropriate information from trial
+        - nctid, descriptions, browse_conditions info (including mesh-ansestors)
+    - for each condition
+        - show proposed codes
+        - each code has a check box next to it.
+        - If there are no proposed codes, an entry box is made available.
+
+Design steps:
+
+Setup SQL queries and tables.
+Setup Flask App

scripts/Icd10ConditionsMatching/setup.py

@@ -0,0 +1,11 @@
+from setuptools import setup
+
+setup(
+    name='Icd10ConditionsMatching',
+    packages=['Icd10ConditionsMatching'],
+    include_package_data=True,
+    install_requires=[
+        'flask',
+        'psycopg2',
+    ],
+)