Added Current status of IDC10 to trial linking tool

Icd10ConditionsMatching/__init__.py

@@ -0,0 +1,44 @@
+from flask import Flask
+import os
+from dotenv import dotenv_values
+
+
+
+env_path = "../../containers/.env"
+ENV = dotenv_values(env_path)
+
+def create_app(test_config=None):
+    # create and configure the app
+    app = Flask(__name__, instance_relative_config=True)
+    app.config.from_mapping(
+        SECRET_KEY='6e674d6e41b733270fd01c6257b3a1b4769eb80f3f773cd0fe8eff25f350fc1f',
+        POSTGRES_DB=ENV["POSTGRES_DB"],
+        POSTGRES_USER=ENV["POSTGRES_USER"],
+        POSTGRES_HOST=ENV["POSTGRES_HOST"],
+        POSTGRES_PORT=ENV["POSTGRES_PORT"],
+        POSTGRES_PASSWORD=ENV["POSTGRES_PASSWORD"],
+    )
+
+
+
+    # ensure the instance folder exists
+    try:
+        os.makedirs(app.instance_path)
+    except OSError:
+        pass
+
+    # a simple page that says hello
+    @app.route('/')
+    def hello():
+        return 'Hello, World!'
+
+
+    from . import db_interface
+    db_interface.init_database(app)
+
+    from . import validation
+    app.register_blueprint(validation.bp)
+
+    return app
+
+

Icd10ConditionsMatching/db_interface.py

@@ -0,0 +1,175 @@
+import psycopg2 as psyco
+from psycopg2 import extras
+from datetime import datetime
+
+import click #used for cli commands. Not needed for what I am doing.
+from flask import current_app, g
+
+def get_db(**kwargs):
+
+    if "db" not in g:
+        g.db = psyco.connect(
+            dbname=current_app.config["POSTGRES_DB"]
+            ,user=current_app.config["POSTGRES_USER"]
+            ,host=current_app.config["POSTGRES_HOST"]
+            ,port=current_app.config["POSTGRES_PORT"]
+            ,password=current_app.config["POSTGRES_PASSWORD"]
+            ,**kwargs
+        )
+    return g.db
+
+def close_db(e=None):
+    db = g.pop('db', None)
+
+    if db is not None:
+        db.close()
+
+def check_initialization(app):
+    db = get_db()
+    with db.cursor() as curse:
+        curse.execute("select count(*) from \"DiseaseBurden\".trial_to_icd10")
+    curse.fetchall()
+    #just checking if everything is going to fail
+
+def init_database(app):
+    #check_initialization(app)
+    app.teardown_appcontext(close_db)
+
+
+
+
+def select_remaing_trials_to_analyze(db_conn):
+    '''
+    This will get the set of trials that need to be analyzed.
+    '''
+    sql = '''
+    select distinct nct_id 
+    from "DiseaseBurden".trial_to_icd10 tti 
+    where tti.approved is null
+    order by nct_id
+    ;
+    '''
+    with db_conn.cursor() as cursor:
+        cursor.execute(sql)
+        return cursor.fetchall()
+
+
+def select_analyzed_trials(db_conn):
+    '''
+    This will get the set of trials that have been analyzed.
+    '''
+    sql = '''
+    select distinct nct_id, max(approval_timestamp)
+    from "DiseaseBurden".trial_to_icd10 tti 
+    where tti.approved in ('accepted','rejected')
+    group by nct_id
+    order by max(approval_timestamp) desc
+    ;
+    '''
+    with db_conn.cursor() as cursor:
+        cursor.execute(sql)
+        return cursor.fetchall()
+
+def select_unmatched_trials(db_conn):
+    '''
+    This will get the set of trials that have been analyzed.
+    '''
+    sql = '''
+    select distinct nct_id
+    from "DiseaseBurden".trial_to_icd10 tti 
+    where tti.approved = 'unmatched'
+    order by nct_id
+    ;
+    '''
+    with db_conn.cursor() as cursor:
+        cursor.execute(sql)
+        return cursor.fetchall()
+
+
+def get_trial_conditions_and_proposed_matches(db_conn, nct_id):
+    sql = '''
+        select *
+        from "DiseaseBurden".trial_to_icd10 tti 
+        where nct_id = %s
+        '''
+    with db_conn.cursor() as cursor:
+        cursor.execute(sql,[nct_id])
+        return cursor.fetchall()
+
+
+def store_validation(db_conn, list_of_insert_data):
+    sql = """
+    update "DiseaseBurden".trial_to_icd10
+    set approved=%s, approval_timestamp=%s
+    where id=%s
+    ; 
+    """
+    with db_conn.cursor() as cursor:
+        for l in list_of_insert_data:
+            cursor.execute(sql, l)
+    db_conn.commit()
+
+def get_trial_summary(db_conn,nct_id):
+    sql_summary ="""
+select 
+    s.nct_id,
+    brief_title ,
+    official_title ,
+    bs.description as brief_description,
+    dd.description as detailed_description
+from ctgov.studies s 
+    left join ctgov.brief_summaries bs
+        on bs.nct_id = s.nct_id 
+    left join ctgov.detailed_descriptions dd 
+        on dd.nct_id = s.nct_id 
+where s.nct_id = %s
+;
+"""
+    sql_conditions="""
+--conditions mentioned
+select * from ctgov.conditions c 
+where c.nct_id = %s
+;
+"""
+    sql_keywords="""
+select nct_id ,downcase_name 
+from ctgov.keywords k 
+where k.nct_id = %s
+;
+"""
+    with db_conn.cursor() as curse:
+        curse.execute(sql_summary,[nct_id])
+        summary = curse.fetchall()
+
+        curse.execute(sql_keywords,[nct_id])
+        keywords = curse.fetchall()
+
+        curse.execute(sql_conditions,[nct_id])
+        conditions = curse.fetchall()
+
+    return {"summary":summary, "keywords":keywords, "conditions":conditions}
+
+def get_list_icd10_codes(db_conn):
+    sql = """
+    select distinct code 
+    from "DiseaseBurden".icd10_to_cause itc 
+    order by code;
+    """
+    with db_conn.cursor() as curse:
+        curse.execute(sql)
+        codes = curse.fetchall()
+    
+    return [ x[0] for x in codes ]
+
+def record_suggested_matches(db_conn, nct_id,condition,icd10_code):
+    sql1 = """
+    INSERT INTO "DiseaseBurden".trial_to_icd10 
+    (nct_id,"condition",ui,"source",approved,approval_timestamp)
+    VALUES (%s,%s,%s,'hand matched','accepted',%s)
+    ;
+    """
+
+
+    with db_conn.cursor() as curse:
+        curse.execute(sql1,[nct_id,condition,icd10_code,datetime.now()])
+    db_conn.commit()

Icd10ConditionsMatching/login.py

@@ -0,0 +1 @@
+#at some point I need to add a login or something.

Icd10ConditionsMatching/templates/base.html

@@ -0,0 +1,25 @@
+<!doctype html>
+<title>{% block title %}{% endblock %} - ClinicalTrialsProject</title>
+<!--<link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">-->
+
+<nav>
+  <h1>Nav</h1>
+  <ul>
+    <li>
+      <a href="{{ url_for('validation.remaining') }}">Validation Home</a>
+    </li>
+    <li>
+      <a href="https://icd.who.int/browse10/2019/en">WHO ICD-10 Codes (2019)</a>
+    </li>
+    <li>
+      <a href="https://uts.nlm.nih.gov/uts/umls/home">UMLS Metathesaurs browser (requires login)</a>
+    </li>
+  </ul>
+</nav>
+
+<section class="content">
+  <header>
+    {% block header %}{% endblock %}
+  </header>
+  {% block content %}{% endblock %}
+</section>

Icd10ConditionsMatching/templates/validation_index.html

@@ -0,0 +1,49 @@
+{% extends 'base.html' %}
+
+{% block header %}
+  <h1>{% block title %} ICD-10 to Trial Conditions Validation {% endblock %}</h1>
+{% endblock %}
+
+{% block content %}
+
+<h2>Trials to Validate</h2>
+
+<table>
+<th>Trials</th>
+{% for trial in list_to_validate %}
+<tr><td>
+    <a href="{{ url_for('.validate_trial', nct_id=trial[0] ) }}">
+        {{ trial [0] }}
+    </a>
+</td></tr> 
+{% endfor %}
+</table>
+
+<h2>Trials that have been Validated</h2>
+
+<table>
+<th>Trials Links</th>
+{% for trial in validated_list %}
+<tr><td>
+    <a href="{{ url_for('.validate_trial', nct_id=trial[0] ) }}">
+        {{ trial [0] }}
+    </a> 
+    (Most recently updated {{trial[1]}})
+</td></tr> 
+{% endfor %}
+</table>
+
+<h2>Trials that don't have a good match</h2>
+
+<table>
+<th>Trial Links</th>
+{% for trial in unmatched_list %}
+<tr><td>
+    <a href="{{ url_for('.validate_trial', nct_id=trial[0] ) }}">
+        {{ trial [0] }}
+    </a>
+</td></tr> 
+{% endfor %}
+</table>
+
+{% endblock %}

Icd10ConditionsMatching/templates/validation_of_trial.html

@@ -0,0 +1,95 @@
+{% extends 'base.html' %}
+
+{% block header %}
+  <h1> ICD-10 to Trial Conditions Validation: {{ nct_id }} </h1>
+{% endblock %}
+
+{% block content %}
+
+<section class="summary">
+  <h3>Trial Summary</h3>
+
+  <div class="text_summary">
+    <ul>
+      <li>NCT: {{ summary_dats["summary"][0][0] }}</li>
+      <li>Brief Title: {{ summary_dats["summary"][0][1] }}</li>
+      <li>Long Title: {{ summary_dats["summary"][0][2] }}</li>
+      <li>Brief Description: {{ summary_dats["summary"][0][3] }}</li>
+      <li>Long Description: {{ summary_dats["summary"][0][4] }}</li>
+    </ul>
+  </div>
+  <div class="keywords">
+    <h4>Keywords</h4>
+    <ul>
+      {% for keyword in  summary_dats["keywords"] %}
+        <li>
+          {{ keyword[1] }}
+        </li>
+        {% endfor %}
+    </ul>
+  </div>
+  <div class="conditions">
+    <h4>Raw Conditions </h4>
+    <ul>
+      {% for condition in  summary_dats["conditions"] %}
+        <li>
+          {{ condition[3] }}
+        </li>
+        {% endfor %}
+    </ul>
+  </div>
+</section>
+
+<section class="proposed_conditions">
+  <h3>Proposed Conditions</h3>
+  <form method="post">
+    <table>
+      <tr>
+      <th>Approve</th>
+      <th>Condition (MeSH normalized)</th>
+      <th>Identifier</th>
+      <th>Source</th>
+      <th>Description</th>
+      <th>Source</th>
+      </tr>
+      {% for condition in condition_list %}
+
+      <tr>
+        <td> <input type="checkbox" id="{{ condition[0] }}" name="{{condition[0]}}" value="accepted" {% if condition[8] == "accepted" %}checked{% endif %}> </td>
+        <td> {{condition[2]}} </td>
+        <td> {{condition[3]}} </td>
+        <td> {{condition[5]}} </td>
+        <td> {{condition[6]}} </td>
+        <td> {{condition[7]}} </td>
+      </tr> 
+
+      {% endfor %}
+    </table>
+    <input type="submit" name="submission" value="Submit approvals">
+    <br/>
+    <input type="submit" name="marked_unmatched" value="Mark unmmatched">
+  </form>
+</section>
+
+<section class="submit_alternate">
+  <h3>Submit Alternate Conditions</h3>
+  <!--For each listed condition, provide a spot to enter a ICT10 code-->
+  <form method="post">
+    <label for="alternate_sub">Please enter the proposed code that appears to be the best match:</label>
+    <input name="alt_sub" id="alternate_sub">
+    <br/>
+    <label for="condition">
+      Please give a name to the condition you used to match this<br/>
+      Condition:
+    </label> 
+    <input name="condition", id="condition">
+    <br/>
+    <input type="submit" name="alternate_submission" value="Submit alternate ICD-10 code">
+  </form>
+</section>
+
+<section class="approved">
+  <!--TODO:This will list the already approved values-->
+</section>
+
+{% endblock %}

Icd10ConditionsMatching/validation.py

@@ -0,0 +1,98 @@
+import functools
+from flask import (Blueprint, flash, g, redirect, render_template, request, session, url_for)
+from Icd10ConditionsMatching.db_interface import (
+    get_db,select_remaing_trials_to_analyze, 
+    select_analyzed_trials,
+    select_unmatched_trials,
+    get_trial_conditions_and_proposed_matches,
+    store_validation,
+    get_trial_summary,
+    get_list_icd10_codes,
+    record_suggested_matches,
+    )
+from datetime import datetime
+
+#### First Blueprint: Checking Data
+bp = Blueprint("validation", __name__, url_prefix="/validation")
+
+
+
+@bp.route("/",methods=["GET"])
+def remaining():
+    db_conn = get_db()
+
+
+    to_validate = select_remaing_trials_to_analyze(db_conn)
+    validated = select_analyzed_trials(db_conn)
+    unmatched_list = select_unmatched_trials(db_conn)
+ 
+    
+    return render_template(
+        "validation_index.html", 
+        list_to_validate=to_validate,
+        validated_list = validated,
+        unmatched_list = unmatched_list
+        )
+
+
+@bp.route("/<nct_id>", methods=["GET","POST"])
+def validate_trial(nct_id):
+        
+    if request.method == "GET":
+        db_conn = get_db()
+
+        condition_list =  get_trial_conditions_and_proposed_matches(db_conn, nct_id)
+        summary_dats = get_trial_summary(db_conn, nct_id)
+
+        return render_template(
+            "validation_of_trial.html", 
+            nct_id=nct_id,
+            condition_list=condition_list,
+            summary_dats=summary_dats,
+            )
+    elif request.method == "POST":
+        db_conn = get_db()
+
+        list_of_insert_data = []
+        
+        db_conn = get_db()
+
+        condition_list =  get_trial_conditions_and_proposed_matches(db_conn, nct_id)
+
+        print(request.form)
+
+        if "submission" in request.form:
+            #if it is a submission: 
+                #grab all match ids from db
+                #if match id in submitted form, mark as approved, otherwise mark as rejected
+            for condition in condition_list:
+                id = condition[0]
+                list_of_insert_data.append((request.form.get(str(id),"rejected"), datetime.now(),id))
+               
+            store_validation(db_conn, list_of_insert_data)
+            return redirect(url_for("validation.remaining"))
+        elif "marked_unmatched" in request.form:
+            #if this was marked as "unmatched", store that for each entry.
+            for condition in condition_list:
+                id = condition[0]
+                list_of_insert_data.append(( "unmatched", datetime.now(), id))
+
+            store_validation(db_conn, list_of_insert_data)
+            return redirect(url_for("validation.remaining"))
+        elif "alternate_submission" in request.form:
+            code = request.form["alt_sub"]
+            code = code.strip().replace(".",'').ljust(7,"-")
+
+            condition = request.form["condition"].strip()
+
+            codelist = get_list_icd10_codes(db_conn)
+            if code in codelist:
+                record_suggested_matches(db_conn, nct_id, condition, code)
+                return redirect(request.path)
+            else:
+                record_suggested_matches(db_conn, nct_id, condition + "| Code not in GBD list", code)
+                return """
+                Entered `{}`, which is not in the list of available ICD-10 codes. <a href={}>Return to trial summary</a>
+                """.format(code.strip("-"),request.path), 422
+        
+

setup.py

@@ -0,0 +1,13 @@
+from setuptools import setup
+
+setup(
+    name='Icd10ConditionsMatching',
+    packages=['Icd10ConditionsMatching'],
+    include_package_data=True,
+    install_requires=[
+        'flask',
+        'psycopg2',
+        'datetime',
+        'python-dotenv',
+    ],
+)

start.sh

@@ -0,0 +1 @@
+waitress-serve --port=5000 --call 'Icd10ConditionsMatching:create_app'