diff --git a/Scripts/DevelopingLinks.sql b/Scripts/DevelopingLinks.sql new file mode 100644 index 0000000..b146bc4 --- /dev/null +++ b/Scripts/DevelopingLinks.sql @@ -0,0 +1,41 @@ +drop table if exists "DiseaseBurden".trial_to_icd10; + +CREATE TABLE "DiseaseBurden".trial_to_icd10 ( + id integer NOT NULL GENERATED ALWAYS AS IDENTITY, + nct_id varchar NOT NULL, + "condition" varchar NOT NULL, + ui varchar NULL, + uri varchar NULL, + rootsource varchar NULL, + "name" varchar NULL, + CONSTRAINT trial_to_icd10_pk PRIMARY KEY (id) +); + + + + +DROP TABLE if exists "DiseaseBurden".icd10_to_cause; + +CREATE TABLE "DiseaseBurden".icd10_to_cause ( + id SERIAL NOT NULL , + code varchar NOT NULL, + cause_text varchar NOT NULL, + CONSTRAINT icd10_to_cause_pk PRIMARY KEY (id) +); + + + +drop table if exists "DiseaseBurden".match_status; + +create type "DiseaseBurden".validation_type as enum ('accepted', 'rejected', 'unmatched'); +comment on type "DiseaseBurden".validation_type is 'This is used to record interactions with each type. It can be accepted (yes this should be used), rejected (no this doesn`t match), or unmatched (where non of the proposed options match)'; + + +CREATE TABLE "DiseaseBurden".match_status ( + id serial4 NOT NULL, + approved "DiseaseBurden".validation_type NOT NULL, + approval_timestamp timestamp NOT NULL, + CONSTRAINT match_status_pk PRIMARY KEY (id, approval_timestamp) +); +COMMENT ON TABLE "DiseaseBurden".match_status IS 'This allows me to record if a particular proposed match is approved or not.'; + diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/__init__.py b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/__init__.py new file mode 100644 index 0000000..9374946 --- /dev/null +++ b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/__init__.py @@ -0,0 +1,43 @@ +from flask import Flask +import os +from dotenv import dotenv_values + + + +env_path = "../../containers/.env" +ENV = dotenv_values(env_path) + +def create_app(test_config=None): + # create and configure the app + app = Flask(__name__, instance_relative_config=True) + app.config.from_mapping( + SECRET_KEY='dev', + POSTGRES_DB=ENV["POSTGRES_DB"], + POSTGRES_USER=ENV["POSTGRES_USER"], + POSTGRES_HOST=ENV["POSTGRES_HOST"], + POSTGRES_PORT=ENV["POSTGRES_PORT"], + POSTGRES_PASSWORD=ENV["POSTGRES_PASSWORD"], + ) + + + # ensure the instance folder exists + try: + os.makedirs(app.instance_path) + except OSError: + pass + + # a simple page that says hello + @app.route('/') + def hello(): + return 'Hello, World!' + + + from . import db_interface + db_interface.init_database(app) + + from . import validation + app.register_blueprint(validation.bp) + + return app + + diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/db_interface.py b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/db_interface.py new file mode 100644 index 0000000..4947195 --- /dev/null +++ b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/db_interface.py @@ -0,0 +1,160 @@ +import psycopg2 as psyco +from psycopg2 import extras + +import click #used for cli commands. Not needed for what I am doing. +from flask import current_app, g + +def get_db(**kwargs): + + if "db" not in g: + g.db = psyco.connect( + dbname=current_app.config["POSTGRES_DB"] + ,user=current_app.config["POSTGRES_USER"] + ,host=current_app.config["POSTGRES_HOST"] + ,port=current_app.config["POSTGRES_PORT"] + ,password=current_app.config["POSTGRES_PASSWORD"] + ,**kwargs + ) + return g.db + +def close_db(e=None): + db = g.pop('db', None) + + if db is not None: + db.close() + +def check_initialization(app): + db = get_db() + with db.cursor() as curse: + curse.execute("select count(*) from \"DiseaseBurden\".trial_to_icd10") + curse.fetchall() + #just checking if everything is going to fail + +def init_database(app): + #check_initialization(app) + app.teardown_appcontext(close_db) + + + + +def select_remaing_trials_to_analyze(db_conn): + ''' + This will get the set of trials that need to be analyzed. + ''' + sql = ''' + select distinct nct_id + from "DiseaseBurden".trial_to_icd10 tti + where tti.id not in + ( + select distinct id + from "DiseaseBurden".match_status + ) + order by nct_id + ; + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql) + return cursor.fetchall() + + +def select_analyzed_trials(db_conn): + ''' + This will get the set of trials that have been analyzed. + ''' + sql = ''' + select distinct nct_id + from "DiseaseBurden".trial_to_icd10 tti + where tti.id in + ( + select distinct id + from "DiseaseBurden".match_status + where approved in ('accepted','rejected') + ) + order by nct_id + ; + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql) + return cursor.fetchall() + +def select_unmatched_trials(db_conn): + ''' + This will get the set of trials that have been analyzed. + ''' + sql = ''' + select distinct nct_id + from "DiseaseBurden".trial_to_icd10 tti + where tti.id in + ( + select distinct id + from "DiseaseBurden".match_status + where approved = 'unmatched' + ) + order by nct_id + ; + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql) + return cursor.fetchall() + + +def get_trial_conditions_and_proposed_matches(db_conn, nct_id): + sql = ''' + select * + from "DiseaseBurden".trial_to_icd10 tti + where nct_id = %s + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql,[nct_id]) + return cursor.fetchall() + + +def store_validation(db_conn, list_of_inserts): + sql = """ + insert into "DiseaseBurden".match_status (id, approved, approval_timestamp) + values %s + ; + """ + with db_conn.cursor() as cursor: + extras.execute_values(cursor, sql, list_of_inserts) + db_conn.commit() + +def get_trial_summary(db_conn,nct_id): + sql_summary =""" +select + s.nct_id, + brief_title , + official_title , + bs.description as brief_description, + dd.description as detailed_description +from ctgov.studies s + left join ctgov.brief_summaries bs + on bs.nct_id = s.nct_id + left join ctgov.detailed_descriptions dd + on dd.nct_id = s.nct_id +where s.nct_id = %s +; +""" + sql_conditions=""" +--conditions mentioned +select * from ctgov.conditions c +where c.nct_id = %s +; +""" + sql_keywords=""" +select nct_id ,downcase_name +from ctgov.keywords k +where k.nct_id = %s +; +""" + with db_conn.cursor() as curse: + curse.execute(sql_summary,[nct_id]) + summary = curse.fetchall() + + curse.execute(sql_keywords,[nct_id]) + keywords = curse.fetchall() + + curse.execute(sql_conditions,[nct_id]) + conditions = curse.fetchall() + + return {"summary":summary, "keywords":keywords, "conditions":conditions} \ No newline at end of file diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/model.py b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/model.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/base.html b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/base.html new file mode 100644 index 0000000..cfbf613 --- /dev/null +++ b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/base.html @@ -0,0 +1,17 @@ + +{% block title %}{% endblock %} - ClinicalTrialsProject + + + + +
+
+ {% block header %}{% endblock %} +
+ {% block content %}{% endblock %} +
\ No newline at end of file diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_index.html b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_index.html new file mode 100644 index 0000000..6de2145 --- /dev/null +++ b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_index.html @@ -0,0 +1,48 @@ +{% extends 'base.html' %} + +{% block header %} +

{% block title %} ICD-10 to Trial Conditions Validation {% endblock %}

+{% endblock %} + +{% block content %} + +

Trials to Validate

+ + + +{% for trial in list_to_validate %} + +{% endfor %} +
Trials
+ + {{ trial [0] }} + +
+ +

Trials that have been Validated

+ + + +{% for trial in validated_list %} + +{% endfor %} +
Trials Links
+ + {{ trial [0] }} + +
+ +

Trials that don't have a good match

+ + + +{% for trial in unmatched_list %} + +{% endfor %} +
Trial Links
+ + {{ trial [0] }} + +
+ +{% endblock %} \ No newline at end of file diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_of_trial.html b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_of_trial.html new file mode 100644 index 0000000..1fa6981 --- /dev/null +++ b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_of_trial.html @@ -0,0 +1,92 @@ +{% extends 'base.html' %} + +{% block header %} +

ICD-10 to Trial Conditions Validation: {{ nct_id }}

+{% endblock %} + +{% block content %} + +
+

Trial Summary

+ +
+ +
+
+

Keywords

+ +
+
+

Raw Conditions

+ +
+
+ +
+

Proposed Conditions

+
+ + + + + + + + + {% for condition in condition_list %} + + + + + + + + + + {% endfor %} +
ApproveCondition (MeSH normalized)IdentifierSourceDescription
{{condition[2]}} {{condition[3]}} {{condition[5]}} {{condition[6]}}
+ +
+ +
+
+ +
+

Submit Alternate Conditions

+ +
+ + +
+ + +
+ +
+
+ + +{% endblock %} \ No newline at end of file diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/validation.py b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/validation.py new file mode 100644 index 0000000..058bb79 --- /dev/null +++ b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/validation.py @@ -0,0 +1,81 @@ +import functools +from flask import (Blueprint, flash, g, redirect, render_template, request, session, url_for) +from Icd10ConditionsMatching.db_interface import ( + get_db,select_remaing_trials_to_analyze, + select_analyzed_trials, + select_unmatched_trials, + get_trial_conditions_and_proposed_matches, + store_validation, + get_trial_summary, + ) +from datetime import datetime + +#### First Blueprint: Checking Data +bp = Blueprint("validation", __name__, url_prefix="/validation") + +@bp.route("/",methods=["GET"]) +def remaining(): + db_conn = get_db() + + + to_validate = select_remaing_trials_to_analyze(db_conn) + validated = select_analyzed_trials(db_conn) + unmatched_list = select_unmatched_trials(db_conn) + + + return render_template( + "validation_index.html", + list_to_validate=to_validate, + validated_list = validated, + unmatched_list = unmatched_list + ) + +@bp.route("/", methods=["GET","POST"]) +def validate_trial(nct_id): + + if request.method == "GET": + db_conn = get_db() + + condition_list = get_trial_conditions_and_proposed_matches(db_conn, nct_id) + summary_dats = get_trial_summary(db_conn, nct_id) + icd10_codes = [1,2,3] + + return render_template( + "validation_of_trial.html", + nct_id=nct_id, + condition_list=condition_list, + summary_dats=summary_dats, + icd10_codes = icd10_codes + ) + elif request.method == "POST": + db_conn = get_db() + + list_of_insert_data = [] + + db_conn = get_db() + + condition_list = get_trial_conditions_and_proposed_matches(db_conn, nct_id) + + print(request.form) + + if "submission" in request.form: + #if it is a submission: + #grab all match ids from db + #if match id in submitted form, mark as approved, otherwise mark as rejected + for condition in condition_list: + id = condition[0] + list_of_insert_data.append((id, request.form.get(str(id),"rejected"),datetime.now())) + + elif "marked_unmatched" in request.form: + #if this was marked as "unmatched", store that for each entry. + for condition in condition_list: + id = condition[0] + list_of_insert_data.append((id, "unmatched",datetime.now())) + elif "alternate_submission" in request.form: + pass + + store_validation(db_conn, list_of_insert_data) + + return redirect(url_for("validation.remaining")) + + diff --git a/scripts/Icd10ConditionsMatching/match_conditions_to_icd10.txt b/scripts/Icd10ConditionsMatching/match_conditions_to_icd10.txt new file mode 100644 index 0000000..69805ae --- /dev/null +++ b/scripts/Icd10ConditionsMatching/match_conditions_to_icd10.txt @@ -0,0 +1,16 @@ +The purpose of this application is to provide an easy way to manually review the +proposed matches between trial conditions and proposed icd10 codes. + +Plan: + - get list of proposed matches For each NCT_ID + - present appropriate information from trial + - nctid, descriptions, browse_conditions info (including mesh-ansestors) + - for each condition + - show proposed codes + - each code has a check box next to it. + - If there are no proposed codes, an entry box is made available. + +Design steps: + +Setup SQL queries and tables. +Setup Flask App diff --git a/scripts/Icd10ConditionsMatching/setup.py b/scripts/Icd10ConditionsMatching/setup.py new file mode 100644 index 0000000..8a35e34 --- /dev/null +++ b/scripts/Icd10ConditionsMatching/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +setup( + name='Icd10ConditionsMatching', + packages=['Icd10ConditionsMatching'], + include_package_data=True, + install_requires=[ + 'flask', + 'psycopg2', + ], +)