From ae8966d9afa1f1b6a427f6e93d3b2707ef315d9e Mon Sep 17 00:00:00 2001 From: Will King Date: Tue, 19 Sep 2023 21:37:45 -0700 Subject: [PATCH] Added Current status of IDC10 to trial linking tool --- Icd10ConditionsMatching/__init__.py | 44 +++++ Icd10ConditionsMatching/db_interface.py | 175 ++++++++++++++++++ Icd10ConditionsMatching/login.py | 1 + Icd10ConditionsMatching/model.py | 0 Icd10ConditionsMatching/templates/base.html | 25 +++ .../templates/validation_index.html | 49 +++++ .../templates/validation_of_trial.html | 95 ++++++++++ Icd10ConditionsMatching/validation.py | 98 ++++++++++ setup.py | 13 ++ start.sh | 1 + 10 files changed, 501 insertions(+) create mode 100644 Icd10ConditionsMatching/__init__.py create mode 100644 Icd10ConditionsMatching/db_interface.py create mode 100644 Icd10ConditionsMatching/login.py create mode 100644 Icd10ConditionsMatching/model.py create mode 100644 Icd10ConditionsMatching/templates/base.html create mode 100644 Icd10ConditionsMatching/templates/validation_index.html create mode 100644 Icd10ConditionsMatching/templates/validation_of_trial.html create mode 100644 Icd10ConditionsMatching/validation.py create mode 100644 setup.py create mode 100755 start.sh diff --git a/Icd10ConditionsMatching/__init__.py b/Icd10ConditionsMatching/__init__.py new file mode 100644 index 0000000..6e82eb6 --- /dev/null +++ b/Icd10ConditionsMatching/__init__.py @@ -0,0 +1,44 @@ +from flask import Flask +import os +from dotenv import dotenv_values + + + +env_path = "../../containers/.env" +ENV = dotenv_values(env_path) + +def create_app(test_config=None): + # create and configure the app + app = Flask(__name__, instance_relative_config=True) + app.config.from_mapping( + SECRET_KEY='6e674d6e41b733270fd01c6257b3a1b4769eb80f3f773cd0fe8eff25f350fc1f', + POSTGRES_DB=ENV["POSTGRES_DB"], + POSTGRES_USER=ENV["POSTGRES_USER"], + POSTGRES_HOST=ENV["POSTGRES_HOST"], + POSTGRES_PORT=ENV["POSTGRES_PORT"], + POSTGRES_PASSWORD=ENV["POSTGRES_PASSWORD"], + ) + + + + # ensure the instance folder exists + try: + os.makedirs(app.instance_path) + except OSError: + pass + + # a simple page that says hello + @app.route('/') + def hello(): + return 'Hello, World!' + + + from . import db_interface + db_interface.init_database(app) + + from . import validation + app.register_blueprint(validation.bp) + + return app + + diff --git a/Icd10ConditionsMatching/db_interface.py b/Icd10ConditionsMatching/db_interface.py new file mode 100644 index 0000000..523ee52 --- /dev/null +++ b/Icd10ConditionsMatching/db_interface.py @@ -0,0 +1,175 @@ +import psycopg2 as psyco +from psycopg2 import extras +from datetime import datetime + +import click #used for cli commands. Not needed for what I am doing. +from flask import current_app, g + +def get_db(**kwargs): + + if "db" not in g: + g.db = psyco.connect( + dbname=current_app.config["POSTGRES_DB"] + ,user=current_app.config["POSTGRES_USER"] + ,host=current_app.config["POSTGRES_HOST"] + ,port=current_app.config["POSTGRES_PORT"] + ,password=current_app.config["POSTGRES_PASSWORD"] + ,**kwargs + ) + return g.db + +def close_db(e=None): + db = g.pop('db', None) + + if db is not None: + db.close() + +def check_initialization(app): + db = get_db() + with db.cursor() as curse: + curse.execute("select count(*) from \"DiseaseBurden\".trial_to_icd10") + curse.fetchall() + #just checking if everything is going to fail + +def init_database(app): + #check_initialization(app) + app.teardown_appcontext(close_db) + + + + +def select_remaing_trials_to_analyze(db_conn): + ''' + This will get the set of trials that need to be analyzed. + ''' + sql = ''' + select distinct nct_id + from "DiseaseBurden".trial_to_icd10 tti + where tti.approved is null + order by nct_id + ; + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql) + return cursor.fetchall() + + +def select_analyzed_trials(db_conn): + ''' + This will get the set of trials that have been analyzed. + ''' + sql = ''' + select distinct nct_id, max(approval_timestamp) + from "DiseaseBurden".trial_to_icd10 tti + where tti.approved in ('accepted','rejected') + group by nct_id + order by max(approval_timestamp) desc + ; + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql) + return cursor.fetchall() + +def select_unmatched_trials(db_conn): + ''' + This will get the set of trials that have been analyzed. + ''' + sql = ''' + select distinct nct_id + from "DiseaseBurden".trial_to_icd10 tti + where tti.approved = 'unmatched' + order by nct_id + ; + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql) + return cursor.fetchall() + + +def get_trial_conditions_and_proposed_matches(db_conn, nct_id): + sql = ''' + select * + from "DiseaseBurden".trial_to_icd10 tti + where nct_id = %s + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql,[nct_id]) + return cursor.fetchall() + + +def store_validation(db_conn, list_of_insert_data): + sql = """ + update "DiseaseBurden".trial_to_icd10 + set approved=%s, approval_timestamp=%s + where id=%s + ; + """ + with db_conn.cursor() as cursor: + for l in list_of_insert_data: + cursor.execute(sql, l) + db_conn.commit() + +def get_trial_summary(db_conn,nct_id): + sql_summary =""" +select + s.nct_id, + brief_title , + official_title , + bs.description as brief_description, + dd.description as detailed_description +from ctgov.studies s + left join ctgov.brief_summaries bs + on bs.nct_id = s.nct_id + left join ctgov.detailed_descriptions dd + on dd.nct_id = s.nct_id +where s.nct_id = %s +; +""" + sql_conditions=""" +--conditions mentioned +select * from ctgov.conditions c +where c.nct_id = %s +; +""" + sql_keywords=""" +select nct_id ,downcase_name +from ctgov.keywords k +where k.nct_id = %s +; +""" + with db_conn.cursor() as curse: + curse.execute(sql_summary,[nct_id]) + summary = curse.fetchall() + + curse.execute(sql_keywords,[nct_id]) + keywords = curse.fetchall() + + curse.execute(sql_conditions,[nct_id]) + conditions = curse.fetchall() + + return {"summary":summary, "keywords":keywords, "conditions":conditions} + +def get_list_icd10_codes(db_conn): + sql = """ + select distinct code + from "DiseaseBurden".icd10_to_cause itc + order by code; + """ + with db_conn.cursor() as curse: + curse.execute(sql) + codes = curse.fetchall() + + return [ x[0] for x in codes ] + +def record_suggested_matches(db_conn, nct_id,condition,icd10_code): + sql1 = """ + INSERT INTO "DiseaseBurden".trial_to_icd10 + (nct_id,"condition",ui,"source",approved,approval_timestamp) + VALUES (%s,%s,%s,'hand matched','accepted',%s) + ; + """ + + + with db_conn.cursor() as curse: + curse.execute(sql1,[nct_id,condition,icd10_code,datetime.now()]) + db_conn.commit() diff --git a/Icd10ConditionsMatching/login.py b/Icd10ConditionsMatching/login.py new file mode 100644 index 0000000..68058a6 --- /dev/null +++ b/Icd10ConditionsMatching/login.py @@ -0,0 +1 @@ +#at some point I need to add a login or something. \ No newline at end of file diff --git a/Icd10ConditionsMatching/model.py b/Icd10ConditionsMatching/model.py new file mode 100644 index 0000000..e69de29 diff --git a/Icd10ConditionsMatching/templates/base.html b/Icd10ConditionsMatching/templates/base.html new file mode 100644 index 0000000..75ecb98 --- /dev/null +++ b/Icd10ConditionsMatching/templates/base.html @@ -0,0 +1,25 @@ + +{% block title %}{% endblock %} - ClinicalTrialsProject + + + + +
+
+ {% block header %}{% endblock %} +
+ {% block content %}{% endblock %} +
\ No newline at end of file diff --git a/Icd10ConditionsMatching/templates/validation_index.html b/Icd10ConditionsMatching/templates/validation_index.html new file mode 100644 index 0000000..761bc31 --- /dev/null +++ b/Icd10ConditionsMatching/templates/validation_index.html @@ -0,0 +1,49 @@ +{% extends 'base.html' %} + +{% block header %} +

{% block title %} ICD-10 to Trial Conditions Validation {% endblock %}

+{% endblock %} + +{% block content %} + +

Trials to Validate

+ + + +{% for trial in list_to_validate %} + +{% endfor %} +
Trials
+ + {{ trial [0] }} + +
+ +

Trials that have been Validated

+ + + +{% for trial in validated_list %} + +{% endfor %} +
Trials Links
+ + {{ trial [0] }} + + (Most recently updated {{trial[1]}}) +
+ +

Trials that don't have a good match

+ + + +{% for trial in unmatched_list %} + +{% endfor %} +
Trial Links
+ + {{ trial [0] }} + +
+ +{% endblock %} \ No newline at end of file diff --git a/Icd10ConditionsMatching/templates/validation_of_trial.html b/Icd10ConditionsMatching/templates/validation_of_trial.html new file mode 100644 index 0000000..be256fd --- /dev/null +++ b/Icd10ConditionsMatching/templates/validation_of_trial.html @@ -0,0 +1,95 @@ +{% extends 'base.html' %} + +{% block header %} +

ICD-10 to Trial Conditions Validation: {{ nct_id }}

+{% endblock %} + +{% block content %} + +
+

Trial Summary

+ +
+
    +
  • NCT: {{ summary_dats["summary"][0][0] }}
    • +
  • Brief Title: {{ summary_dats["summary"][0][1] }}
    • +
  • Long Title: {{ summary_dats["summary"][0][2] }}
    • +
  • Brief Description: {{ summary_dats["summary"][0][3] }}
    • +
  • Long Description: {{ summary_dats["summary"][0][4] }}
    • +
+
+
+

Keywords

+
    + {% for keyword in summary_dats["keywords"] %} +
  • + {{ keyword[1] }} +
    • + {% endfor %} +
+
+
+

Raw Conditions

+
    + {% for condition in summary_dats["conditions"] %} +
  • + {{ condition[3] }} +
    • + {% endfor %} +
+
+
+ +
+

Proposed Conditions

+
+ + + + + + + + + + {% for condition in condition_list %} + + + + + + + + + + + {% endfor %} +
ApproveCondition (MeSH normalized)IdentifierSourceDescriptionSource
{{condition[2]}} {{condition[3]}} {{condition[5]}} {{condition[6]}} {{condition[7]}}
+ +
+ +
+
+ +
+

Submit Alternate Conditions

+ +
+ + +
+ + +
+ +
+
+ +
+ +
+ +{% endblock %} \ No newline at end of file diff --git a/Icd10ConditionsMatching/validation.py b/Icd10ConditionsMatching/validation.py new file mode 100644 index 0000000..62e1838 --- /dev/null +++ b/Icd10ConditionsMatching/validation.py @@ -0,0 +1,98 @@ +import functools +from flask import (Blueprint, flash, g, redirect, render_template, request, session, url_for) +from Icd10ConditionsMatching.db_interface import ( + get_db,select_remaing_trials_to_analyze, + select_analyzed_trials, + select_unmatched_trials, + get_trial_conditions_and_proposed_matches, + store_validation, + get_trial_summary, + get_list_icd10_codes, + record_suggested_matches, + ) +from datetime import datetime + +#### First Blueprint: Checking Data +bp = Blueprint("validation", __name__, url_prefix="/validation") + + + +@bp.route("/",methods=["GET"]) +def remaining(): + db_conn = get_db() + + + to_validate = select_remaing_trials_to_analyze(db_conn) + validated = select_analyzed_trials(db_conn) + unmatched_list = select_unmatched_trials(db_conn) + + + return render_template( + "validation_index.html", + list_to_validate=to_validate, + validated_list = validated, + unmatched_list = unmatched_list + ) + + +@bp.route("/", methods=["GET","POST"]) +def validate_trial(nct_id): + + if request.method == "GET": + db_conn = get_db() + + condition_list = get_trial_conditions_and_proposed_matches(db_conn, nct_id) + summary_dats = get_trial_summary(db_conn, nct_id) + + return render_template( + "validation_of_trial.html", + nct_id=nct_id, + condition_list=condition_list, + summary_dats=summary_dats, + ) + elif request.method == "POST": + db_conn = get_db() + + list_of_insert_data = [] + + db_conn = get_db() + + condition_list = get_trial_conditions_and_proposed_matches(db_conn, nct_id) + + print(request.form) + + if "submission" in request.form: + #if it is a submission: + #grab all match ids from db + #if match id in submitted form, mark as approved, otherwise mark as rejected + for condition in condition_list: + id = condition[0] + list_of_insert_data.append((request.form.get(str(id),"rejected"), datetime.now(),id)) + + store_validation(db_conn, list_of_insert_data) + return redirect(url_for("validation.remaining")) + elif "marked_unmatched" in request.form: + #if this was marked as "unmatched", store that for each entry. + for condition in condition_list: + id = condition[0] + list_of_insert_data.append(( "unmatched", datetime.now(), id)) + + store_validation(db_conn, list_of_insert_data) + return redirect(url_for("validation.remaining")) + elif "alternate_submission" in request.form: + code = request.form["alt_sub"] + code = code.strip().replace(".",'').ljust(7,"-") + + condition = request.form["condition"].strip() + + codelist = get_list_icd10_codes(db_conn) + if code in codelist: + record_suggested_matches(db_conn, nct_id, condition, code) + return redirect(request.path) + else: + record_suggested_matches(db_conn, nct_id, condition + "| Code not in GBD list", code) + return """ + Entered `{}`, which is not in the list of available ICD-10 codes. Return to trial summary + """.format(code.strip("-"),request.path), 422 + + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..764b8b4 --- /dev/null +++ b/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup + +setup( + name='Icd10ConditionsMatching', + packages=['Icd10ConditionsMatching'], + include_package_data=True, + install_requires=[ + 'flask', + 'psycopg2', + 'datetime', + 'python-dotenv', + ], +) diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..6868a58 --- /dev/null +++ b/start.sh @@ -0,0 +1 @@ +waitress-serve --port=5000 --call 'Icd10ConditionsMatching:create_app'