From 3e65b9fbddf495a3ee66a7a86d47028fc4b4b33a Mon Sep 17 00:00:00 2001 From: Will King Date: Fri, 22 Sep 2023 08:46:36 -0700 Subject: [PATCH 1/4] can request and display some data in json format --- .gitignore | 6 + .../FormulariesMatching/__init__.py | 50 +++++ .../FormulariesMatching/db_interface.py | 83 +++++++++ .../formularies_db.py.template | 25 +++ .../formularies_matching.py | 47 +++++ .../FormulariesMatching/icd10_db.py | 175 ++++++++++++++++++ .../templates/.formulary_index.html.swp | Bin 0 -> 12288 bytes .../FormulariesMatching/templates/base.html | 22 +++ .../templates/formulary_index.html | 49 +++++ .../templates/icd10_index.html | 49 +++++ .../FormulariesMatching/templates/index.html | 24 +++ FormulariesMatching/requirements.txt | 6 + FormulariesMatching/setup.py | 12 ++ FormulariesMatching/start.sh | 1 + sample.env | 2 +- 15 files changed, 550 insertions(+), 1 deletion(-) create mode 100644 FormulariesMatching/FormulariesMatching/__init__.py create mode 100644 FormulariesMatching/FormulariesMatching/db_interface.py create mode 100644 FormulariesMatching/FormulariesMatching/formularies_db.py.template create mode 100644 FormulariesMatching/FormulariesMatching/formularies_matching.py create mode 100644 FormulariesMatching/FormulariesMatching/icd10_db.py create mode 100644 FormulariesMatching/FormulariesMatching/templates/.formulary_index.html.swp create mode 100644 FormulariesMatching/FormulariesMatching/templates/base.html create mode 100644 FormulariesMatching/FormulariesMatching/templates/formulary_index.html create mode 100644 FormulariesMatching/FormulariesMatching/templates/icd10_index.html create mode 100644 FormulariesMatching/FormulariesMatching/templates/index.html create mode 100644 FormulariesMatching/requirements.txt create mode 100644 FormulariesMatching/setup.py create mode 100755 FormulariesMatching/start.sh diff --git a/.gitignore b/.gitignore index 5d381cc..4cb5c72 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,9 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +*bin/ +avenv +lib64 +pyvenv.cfg +*.swp diff --git a/FormulariesMatching/FormulariesMatching/__init__.py b/FormulariesMatching/FormulariesMatching/__init__.py new file mode 100644 index 0000000..53f2575 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/__init__.py @@ -0,0 +1,50 @@ +from flask import (Flask,render_template) +import os +from .db_interface import get_connection_details, check_connection + + + + + + +def create_app(test_config=None): + # create and configure the app + app = Flask(__name__, instance_relative_config=True) + app.config.from_mapping( + SECRET_KEY='6e674d6e41b733270fd01c6257b3a1b4769eb80f3f773cd0fe8eff25f350fc1f', + POSTGRES_DB="aact_db", + POSTGRES_USER="root", + POSTGRES_HOST="localhost", + POSTGRES_PORT=5432, + POSTGRES_PASSWORD="root", + ) + + + # ensure the instance folder exists + try: + os.makedirs(app.instance_path) + except OSError: + pass + + # a simple page that says hello + @app.route('/') + def hello(): + return render_template("index.html") + + + @app.route('/debug') + def debug_info(): + return { + "connection": get_connection_details(), + } + + + + from . import db_interface + #db_interface.check_connection(app) + + from . import formularies_matching + app.register_blueprint(formularies_matching.bp) + + + return app diff --git a/FormulariesMatching/FormulariesMatching/db_interface.py b/FormulariesMatching/FormulariesMatching/db_interface.py new file mode 100644 index 0000000..93f5fe9 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/db_interface.py @@ -0,0 +1,83 @@ +import psycopg as psyco +from datetime import datetime +from flask import current_app,g + + +def get_db(**kwargs): + + if "db" not in g: + g.db = psyco.connect( + dbname=current_app.config["POSTGRES_DB"] + ,user=current_app.config["POSTGRES_USER"] + ,host=current_app.config["POSTGRES_HOST"] + ,port=current_app.config["POSTGRES_PORT"] + ,password=current_app.config["POSTGRES_PASSWORD"] + ,**kwargs + ) + return g.db + +def close_db(e=None): + db = g.pop('db', None) + + if db is not None: + db.close() + +def get_connection_details(): + return { + "dbname":current_app.config["POSTGRES_DB"] + ,"user":current_app.config["POSTGRES_USER"] + ,"host":current_app.config["POSTGRES_HOST"] + ,"port":current_app.config["POSTGRES_PORT"] + ,"password":current_app.config["POSTGRES_PASSWORD"] + } + +def check_connection(app): + db = get_db() + with db.cursor() as curse: + curse.execute("select count(*) from \"DiseaseBurden\".trial_to_icd10") + curse.fetchall() + #just checking if everything is going to fail + return True + + + +def get_trial_summary(db_conn,nct_id): + sql_summary =""" +select + s.nct_id, + brief_title , + official_title , + bs.description as brief_description, + dd.description as detailed_description +from ctgov.studies s + left join ctgov.brief_summaries bs + on bs.nct_id = s.nct_id + left join ctgov.detailed_descriptions dd + on dd.nct_id = s.nct_id +where s.nct_id = %s +; +""" + sql_conditions=""" +--conditions mentioned +select * from ctgov.conditions c +where c.nct_id = %s +; +""" + sql_keywords=""" +select nct_id ,downcase_name +from ctgov.keywords k +where k.nct_id = %s +; +""" + with db_conn.cursor() as curse: + curse.execute(sql_summary,[nct_id]) + summary = curse.fetchall() + + curse.execute(sql_keywords,[nct_id]) + keywords = curse.fetchall() + + curse.execute(sql_conditions,[nct_id]) + conditions = curse.fetchall() + + return {"summary":summary, "keywords":keywords, "conditions":conditions} + diff --git a/FormulariesMatching/FormulariesMatching/formularies_db.py.template b/FormulariesMatching/FormulariesMatching/formularies_db.py.template new file mode 100644 index 0000000..e4732b9 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/formularies_db.py.template @@ -0,0 +1,25 @@ +import psycopg as psyco +from datetime import datetime + +from flask import current_app, g + + + + +def get_all_formulary_groups(db_conn): + ''' + Get the list of active formulary groups + TODO: IMplement for the given formulary + ''' + pass + +def get_formulary_groups_per_NCTID(db_conn, nct_id): + ''' + Get the list of formulary groups associated with + the drugs found in a trial identified by NCTID + ''' + pass + + +def store_trial_to_formulary_group_matches(): + pass diff --git a/FormulariesMatching/FormulariesMatching/formularies_matching.py b/FormulariesMatching/FormulariesMatching/formularies_matching.py new file mode 100644 index 0000000..8c9d3fe --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/formularies_matching.py @@ -0,0 +1,47 @@ +import functools +from flask import (Blueprint, flash, g, redirect, render_template, request, session, url_for) +from datetime import datetime + +from FormulariesMatching.db_interface import ( + get_db, + get_connection_details, + get_trial_summary, + ) + + + +#setup blueprint +bp = Blueprint("formularies validation", __name__, url_prefix='/link/formularies') + +@bp.route("/", methods=['GET']) +def get_remaining(): + #get db connection + #db_conn = get_db() + + #get required data + connection_valid = get_connection_details() + + #return html + return connection_valid + + +@bp.route("/", methods=['GET',"POST"]) +def match_trial_to_formulary_groups(nct_id): + #get db connection + db_conn = get_db() + + if request.method == "GET": + pass + #get list of potential matches for each of the formularies + #get trial summary + summary = get_trial_summary(db_conn,nct_id) + + #render template + return summary + + elif request.method == "POST": + pass + #build array of data to insert + + else: + pass diff --git a/FormulariesMatching/FormulariesMatching/icd10_db.py b/FormulariesMatching/FormulariesMatching/icd10_db.py new file mode 100644 index 0000000..523ee52 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/icd10_db.py @@ -0,0 +1,175 @@ +import psycopg2 as psyco +from psycopg2 import extras +from datetime import datetime + +import click #used for cli commands. Not needed for what I am doing. +from flask import current_app, g + +def get_db(**kwargs): + + if "db" not in g: + g.db = psyco.connect( + dbname=current_app.config["POSTGRES_DB"] + ,user=current_app.config["POSTGRES_USER"] + ,host=current_app.config["POSTGRES_HOST"] + ,port=current_app.config["POSTGRES_PORT"] + ,password=current_app.config["POSTGRES_PASSWORD"] + ,**kwargs + ) + return g.db + +def close_db(e=None): + db = g.pop('db', None) + + if db is not None: + db.close() + +def check_initialization(app): + db = get_db() + with db.cursor() as curse: + curse.execute("select count(*) from \"DiseaseBurden\".trial_to_icd10") + curse.fetchall() + #just checking if everything is going to fail + +def init_database(app): + #check_initialization(app) + app.teardown_appcontext(close_db) + + + + +def select_remaing_trials_to_analyze(db_conn): + ''' + This will get the set of trials that need to be analyzed. + ''' + sql = ''' + select distinct nct_id + from "DiseaseBurden".trial_to_icd10 tti + where tti.approved is null + order by nct_id + ; + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql) + return cursor.fetchall() + + +def select_analyzed_trials(db_conn): + ''' + This will get the set of trials that have been analyzed. + ''' + sql = ''' + select distinct nct_id, max(approval_timestamp) + from "DiseaseBurden".trial_to_icd10 tti + where tti.approved in ('accepted','rejected') + group by nct_id + order by max(approval_timestamp) desc + ; + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql) + return cursor.fetchall() + +def select_unmatched_trials(db_conn): + ''' + This will get the set of trials that have been analyzed. + ''' + sql = ''' + select distinct nct_id + from "DiseaseBurden".trial_to_icd10 tti + where tti.approved = 'unmatched' + order by nct_id + ; + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql) + return cursor.fetchall() + + +def get_trial_conditions_and_proposed_matches(db_conn, nct_id): + sql = ''' + select * + from "DiseaseBurden".trial_to_icd10 tti + where nct_id = %s + ''' + with db_conn.cursor() as cursor: + cursor.execute(sql,[nct_id]) + return cursor.fetchall() + + +def store_validation(db_conn, list_of_insert_data): + sql = """ + update "DiseaseBurden".trial_to_icd10 + set approved=%s, approval_timestamp=%s + where id=%s + ; + """ + with db_conn.cursor() as cursor: + for l in list_of_insert_data: + cursor.execute(sql, l) + db_conn.commit() + +def get_trial_summary(db_conn,nct_id): + sql_summary =""" +select + s.nct_id, + brief_title , + official_title , + bs.description as brief_description, + dd.description as detailed_description +from ctgov.studies s + left join ctgov.brief_summaries bs + on bs.nct_id = s.nct_id + left join ctgov.detailed_descriptions dd + on dd.nct_id = s.nct_id +where s.nct_id = %s +; +""" + sql_conditions=""" +--conditions mentioned +select * from ctgov.conditions c +where c.nct_id = %s +; +""" + sql_keywords=""" +select nct_id ,downcase_name +from ctgov.keywords k +where k.nct_id = %s +; +""" + with db_conn.cursor() as curse: + curse.execute(sql_summary,[nct_id]) + summary = curse.fetchall() + + curse.execute(sql_keywords,[nct_id]) + keywords = curse.fetchall() + + curse.execute(sql_conditions,[nct_id]) + conditions = curse.fetchall() + + return {"summary":summary, "keywords":keywords, "conditions":conditions} + +def get_list_icd10_codes(db_conn): + sql = """ + select distinct code + from "DiseaseBurden".icd10_to_cause itc + order by code; + """ + with db_conn.cursor() as curse: + curse.execute(sql) + codes = curse.fetchall() + + return [ x[0] for x in codes ] + +def record_suggested_matches(db_conn, nct_id,condition,icd10_code): + sql1 = """ + INSERT INTO "DiseaseBurden".trial_to_icd10 + (nct_id,"condition",ui,"source",approved,approval_timestamp) + VALUES (%s,%s,%s,'hand matched','accepted',%s) + ; + """ + + + with db_conn.cursor() as curse: + curse.execute(sql1,[nct_id,condition,icd10_code,datetime.now()]) + db_conn.commit() diff --git a/FormulariesMatching/FormulariesMatching/templates/.formulary_index.html.swp b/FormulariesMatching/FormulariesMatching/templates/.formulary_index.html.swp new file mode 100644 index 0000000000000000000000000000000000000000..4543b2541a4d98bacf577af7ac2246c1ce92b200 GIT binary patch literal 12288 zcmeI2Pixdb7>B2V2-;TBvp81DwrHE(_M({P(o#^YUfP06iL=RUGjujtlG(bYu7c2u z7cb&PJP6+W3f??<^$U3N&!-T7GubV)l)8c_mN)QhHZyNB@B3utuUt2dVTwdpFu zafY!M--hM0kLTG-!`MTmwQc^@VdJKfDDiUGBAJv(;8w=5O=H> zTkD;N_MLl|bRq*}fDDiUGC&5%02v?yWPl8i0Wv@a{zC&IW-L9$*v(-ikN^MszyH6U zX6y_21a`nCXafNrfCVrQu7k_q1o$z^*n996yaLa`Q?La(;10MA7C{w^g5%)B2xITS z8_)q7gkTk{fC*3m7r;sI9l3r6ufYy@1X>^h_rP6X>sbJFkO4A42FL&zAOmE843Ggb z@P`d-k8_#2kGQV0XoHV;8S_0O>RJY@?7#+H+l)L(6lUz3D#?aZ$b%^IoRHiqDRxvjIhAzHxg{P4TRXpb|?cI~Oe=J}=2J$hY zO~PhSK3=7~oi^>pVi|{86~<*|l3>}5Zg(LU$T6|^mo+dynJIeHjw!U*WEiv&j-rqCdYufu9j97Csy&N!+B9?B^xh2@a=yAa-bx}ySJ-%X7 F`3ZQOgYW +{% block title %}{% endblock %} - ClinicalTrialsProject + + + + +
+
+ {% block header %}{% endblock %} +
+ {% block content %}{% endblock %} +
diff --git a/FormulariesMatching/FormulariesMatching/templates/formulary_index.html b/FormulariesMatching/FormulariesMatching/templates/formulary_index.html new file mode 100644 index 0000000..a141319 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/templates/formulary_index.html @@ -0,0 +1,49 @@ +{% extends 'base.html' %} + +{% block header %} +

{% block title %} ICD-10 to Trial Conditions Validation {% endblock %}

+{% endblock %} + +{% block content %} + +

Trials to Validate

+ + + +{% for trial in list_to_validate %} + +{% endfor %} +
Trials
+ + {{ trial [0] }} + +
+ +

Trials that have been Validated

+ + + +{% for trial in validated_list %} + +{% endfor %} +
Trials Links
+ + {{ trial [0] }} + + (Most recently updated {{trial[1]}}) +
+ +

Trials that don't have a good match

+ + + +{% for trial in unmatched_list %} + +{% endfor %} +
Trial Links
+ + {{ trial [0] }} + +
+ +{% endblock %} diff --git a/FormulariesMatching/FormulariesMatching/templates/icd10_index.html b/FormulariesMatching/FormulariesMatching/templates/icd10_index.html new file mode 100644 index 0000000..a141319 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/templates/icd10_index.html @@ -0,0 +1,49 @@ +{% extends 'base.html' %} + +{% block header %} +

{% block title %} ICD-10 to Trial Conditions Validation {% endblock %}

+{% endblock %} + +{% block content %} + +

Trials to Validate

+ + + +{% for trial in list_to_validate %} + +{% endfor %} +
Trials
+ + {{ trial [0] }} + +
+ +

Trials that have been Validated

+ + + +{% for trial in validated_list %} + +{% endfor %} +
Trials Links
+ + {{ trial [0] }} + + (Most recently updated {{trial[1]}}) +
+ +

Trials that don't have a good match

+ + + +{% for trial in unmatched_list %} + +{% endfor %} +
Trial Links
+ + {{ trial [0] }} + +
+ +{% endblock %} diff --git a/FormulariesMatching/FormulariesMatching/templates/index.html b/FormulariesMatching/FormulariesMatching/templates/index.html new file mode 100644 index 0000000..bde5a58 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/templates/index.html @@ -0,0 +1,24 @@ +{% extends 'base.html' %} + +{% block header %} +

+{% block title %} +Formulary Matching +{% endblock %} +

+{% endblock %} + + + + +{% block content %} + +This is the home page for matching things to clinical trials + +There are a few major efforts related to matching. + +The first is to match trials to IDC10 codes + +The second is to link trials to formulary groups. + +{% endblock %} diff --git a/FormulariesMatching/requirements.txt b/FormulariesMatching/requirements.txt new file mode 100644 index 0000000..3b12535 --- /dev/null +++ b/FormulariesMatching/requirements.txt @@ -0,0 +1,6 @@ +flask +psycopg[binary] +datetime +watchdog +waitress +python-dotenv diff --git a/FormulariesMatching/setup.py b/FormulariesMatching/setup.py new file mode 100644 index 0000000..08e5e24 --- /dev/null +++ b/FormulariesMatching/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup + +setup( + name='FormulariesMatching', + packages=['FormulariesMatching'], + include_package_data=True, + install_requires=[ + 'flask', + 'psycopg[binary]', + 'datetime', + ], +) diff --git a/FormulariesMatching/start.sh b/FormulariesMatching/start.sh new file mode 100755 index 0000000..6868a58 --- /dev/null +++ b/FormulariesMatching/start.sh @@ -0,0 +1 @@ +waitress-serve --port=5000 --call 'Icd10ConditionsMatching:create_app' diff --git a/sample.env b/sample.env index 6e2f542..d708fb6 100644 --- a/sample.env +++ b/sample.env @@ -1,6 +1,6 @@ #Postgres Info -POSTGRES_HOST= +POSTGRES_HOST=will-office POSTGRES_DB=aact_db POSTGRES_PORT=5432 POSTGRES_USER=root From dd90715400a587cde1c5f73e43b04f6357f5e392 Mon Sep 17 00:00:00 2001 From: Will King Date: Fri, 22 Sep 2023 13:53:13 -0700 Subject: [PATCH 2/4] got uspdc displaying --- .../FormulariesMatching/__init__.py | 7 +- .../formularies_matching.py | 47 ----------- .../FormulariesMatching/matching.py | 70 ++++++++++++++++ .../FormulariesMatching/static/styles.css | 5 ++ .../templates/.formulary_index.html.swp | Bin 12288 -> 0 bytes .../FormulariesMatching/templates/base.html | 16 ++-- .../templates/formulary_index.html | 10 +-- .../templates/trial_formularies.html | 76 ++++++++++++++++++ .../FormulariesMatching/uspdc_db.py | 47 +++++++++++ .../FormulariesMatching/uspmmg_db.py | 35 ++++++++ .../FormulariesMatching/vaform_db.py | 35 ++++++++ 11 files changed, 285 insertions(+), 63 deletions(-) delete mode 100644 FormulariesMatching/FormulariesMatching/formularies_matching.py create mode 100644 FormulariesMatching/FormulariesMatching/matching.py create mode 100644 FormulariesMatching/FormulariesMatching/static/styles.css delete mode 100644 FormulariesMatching/FormulariesMatching/templates/.formulary_index.html.swp create mode 100644 FormulariesMatching/FormulariesMatching/templates/trial_formularies.html create mode 100644 FormulariesMatching/FormulariesMatching/uspdc_db.py create mode 100644 FormulariesMatching/FormulariesMatching/uspmmg_db.py create mode 100644 FormulariesMatching/FormulariesMatching/vaform_db.py diff --git a/FormulariesMatching/FormulariesMatching/__init__.py b/FormulariesMatching/FormulariesMatching/__init__.py index 53f2575..57530ee 100644 --- a/FormulariesMatching/FormulariesMatching/__init__.py +++ b/FormulariesMatching/FormulariesMatching/__init__.py @@ -40,11 +40,8 @@ def create_app(test_config=None): - from . import db_interface - #db_interface.check_connection(app) - - from . import formularies_matching - app.register_blueprint(formularies_matching.bp) + from . import matching + app.register_blueprint(matching.bp) return app diff --git a/FormulariesMatching/FormulariesMatching/formularies_matching.py b/FormulariesMatching/FormulariesMatching/formularies_matching.py deleted file mode 100644 index 8c9d3fe..0000000 --- a/FormulariesMatching/FormulariesMatching/formularies_matching.py +++ /dev/null @@ -1,47 +0,0 @@ -import functools -from flask import (Blueprint, flash, g, redirect, render_template, request, session, url_for) -from datetime import datetime - -from FormulariesMatching.db_interface import ( - get_db, - get_connection_details, - get_trial_summary, - ) - - - -#setup blueprint -bp = Blueprint("formularies validation", __name__, url_prefix='/link/formularies') - -@bp.route("/", methods=['GET']) -def get_remaining(): - #get db connection - #db_conn = get_db() - - #get required data - connection_valid = get_connection_details() - - #return html - return connection_valid - - -@bp.route("/", methods=['GET',"POST"]) -def match_trial_to_formulary_groups(nct_id): - #get db connection - db_conn = get_db() - - if request.method == "GET": - pass - #get list of potential matches for each of the formularies - #get trial summary - summary = get_trial_summary(db_conn,nct_id) - - #render template - return summary - - elif request.method == "POST": - pass - #build array of data to insert - - else: - pass diff --git a/FormulariesMatching/FormulariesMatching/matching.py b/FormulariesMatching/FormulariesMatching/matching.py new file mode 100644 index 0000000..4326301 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/matching.py @@ -0,0 +1,70 @@ +import functools +from flask import (Blueprint, flash, g, redirect, render_template, request, session, url_for) +from datetime import datetime + +from FormulariesMatching.db_interface import ( + get_db, + get_connection_details, + get_trial_summary, + ) +import FormulariesMatching.uspdc_db as uspdc +import FormulariesMatching.uspmmg_db as uspmmg +import FormulariesMatching.vaform_db as vaform + +FORMULARIES = { + "USP DC":uspdc, +# "USP MMG":uspmmg, +# "VA Formulary":vaform, + } + + +#setup blueprint +bp = Blueprint("formularies", __name__, url_prefix='/link/formularies') + +@bp.route("/", methods=['GET']) +def get_remaining_trials(): + #get db connection + #db_conn = get_db() + + #get required data + connection_valid = get_connection_details() + + #return html + return connection_valid + + +@bp.route("/", methods=['GET',"POST"]) +def match_trial_to_formulary_groups(nct_id): + #get db connection + db_conn = get_db() + + if request.method == "GET": + + #get list of potential matches for each of the formularies + potential_matches = {} + class_lists = {} + + for formulary in FORMULARIES: + potential_matches[formulary] = FORMULARIES[formulary].get_groups_per_nctid(db_conn,nct_id) + class_lists[formulary] = FORMULARIES[formulary].get_all_formulary_groups(db_conn) + + #get trial summary + summary = get_trial_summary(db_conn,nct_id) + + #render template +# return [potential_matches,class_lists,summary] + return render_template('trial_formularies.html', + nct_id=nct_id, + potential_matches=potential_matches, + class_lists=class_lists, + summary=summary, + ) + + elif request.method == "POST": + pass + #build array of data to insert + + else: + raise Exception("HTTP method <{}> not implemented".format(request.method)) + + diff --git a/FormulariesMatching/FormulariesMatching/static/styles.css b/FormulariesMatching/FormulariesMatching/static/styles.css new file mode 100644 index 0000000..58ca343 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/static/styles.css @@ -0,0 +1,5 @@ +.table { + width: 100%; + border-collapse: collapse; + border: 1px solid; +} diff --git a/FormulariesMatching/FormulariesMatching/templates/.formulary_index.html.swp b/FormulariesMatching/FormulariesMatching/templates/.formulary_index.html.swp deleted file mode 100644 index 4543b2541a4d98bacf577af7ac2246c1ce92b200..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2Pixdb7>B2V2-;TBvp81DwrHE(_M({P(o#^YUfP06iL=RUGjujtlG(bYu7c2u z7cb&PJP6+W3f??<^$U3N&!-T7GubV)l)8c_mN)QhHZyNB@B3utuUt2dVTwdpFu zafY!M--hM0kLTG-!`MTmwQc^@VdJKfDDiUGBAJv(;8w=5O=H> zTkD;N_MLl|bRq*}fDDiUGC&5%02v?yWPl8i0Wv@a{zC&IW-L9$*v(-ikN^MszyH6U zX6y_21a`nCXafNrfCVrQu7k_q1o$z^*n996yaLa`Q?La(;10MA7C{w^g5%)B2xITS z8_)q7gkTk{fC*3m7r;sI9l3r6ufYy@1X>^h_rP6X>sbJFkO4A42FL&zAOmE843Ggb z@P`d-k8_#2kGQV0XoHV;8S_0O>RJY@?7#+H+l)L(6lUz3D#?aZ$b%^IoRHiqDRxvjIhAzHxg{P4TRXpb|?cI~Oe=J}=2J$hY zO~PhSK3=7~oi^>pVi|{86~<*|l3>}5Zg(LU$T6|^mo+dynJIeHjw!U*WEiv&j-rqCdYufu9j97Csy&N!+B9?B^xh2@a=yAa-bx}ySJ-%X7 F`3ZQOgYW + + + + {% block title %}{% endblock %} - ClinicalTrialsProject - + + -
-
+ {% block header %}{% endblock %} -
{% block content %}{% endblock %} -
+ + diff --git a/FormulariesMatching/FormulariesMatching/templates/formulary_index.html b/FormulariesMatching/FormulariesMatching/templates/formulary_index.html index a141319..459d4bb 100644 --- a/FormulariesMatching/FormulariesMatching/templates/formulary_index.html +++ b/FormulariesMatching/FormulariesMatching/templates/formulary_index.html @@ -1,16 +1,16 @@ {% extends 'base.html' %} {% block header %} -

{% block title %} ICD-10 to Trial Conditions Validation {% endblock %}

+

{% block title %} Linking Trials to Formularies{% endblock %}

{% endblock %} {% block content %} -

Trials to Validate

+

Unlinked Trials

-{% for trial in list_to_validate %} +{% for trial in unlinked_trials %}
Trials
{{ trial [0] }} @@ -19,7 +19,7 @@ {% endfor %}
-

Trials that have been Validated

+

Linked Trials

@@ -33,7 +33,7 @@ {% endfor %}
Trials Links
-

Trials that don't have a good match

+

Flagged for later Trials

diff --git a/FormulariesMatching/FormulariesMatching/templates/trial_formularies.html b/FormulariesMatching/FormulariesMatching/templates/trial_formularies.html new file mode 100644 index 0000000..a14158b --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/templates/trial_formularies.html @@ -0,0 +1,76 @@ +{% extends 'base.html' %} + +{% block header %} +

+ {% block title %} + Match Trial {{nct_id}} to Formularies + {% endblock %} +

+{% endblock %} + +{% block content %} + +Based on the drugs associated with a trial, +the following are the formulary group suggestions. + + +{% for formulary in potential_matches %} +
+

+{{ formulary }} +

+ +

+{% for formulary in potential_matches %} + +

Trial Links
+ + + + + + {% for row in potential_matches[formulary] %} + + + + + + + {% endfor %} + +
CategoryClassSelect
{{ row[1] }} {{ row[2] }} +
+

+

+If you've determined it belongs to another class + +{% endfor %} + +{% endfor %} + +

+

+ +

+ + + +{% endblock %} diff --git a/FormulariesMatching/FormulariesMatching/uspdc_db.py b/FormulariesMatching/FormulariesMatching/uspdc_db.py new file mode 100644 index 0000000..60b9688 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/uspdc_db.py @@ -0,0 +1,47 @@ +import psycopg as psyco +from datetime import datetime + +from flask import current_app, g + + + + +def get_all_formulary_groups(db_conn): + ''' + Get the list of active formulary groups + TODO: IMplement for the given formulary + ''' + sql = '''\ +select distinct "USP Category", "USP Class" +from "Formularies".usp_dc ud +order by "USP Category", "USP Class" +; +''' + #query + with db_conn.cursor() as curse: + curse.execute(sql) + return curse.fetchall() + + + + +def get_groups_per_nctid(db_conn, nct_id): + ''' + Get the list of formulary groups associated with + the drugs found in a trial identified by NCTID + ''' + pass + sql = '''\ +select * from "Formularies".trial_to_uspdc_category_class ttucc +where nct_id = %(nctid)s +; +''' + + #query + with db_conn.cursor() as curse: + curse.execute(sql, {"nctid":nct_id}) + return curse.fetchall() + + +def store_trial_to_formulary_group_matches(): + pass diff --git a/FormulariesMatching/FormulariesMatching/uspmmg_db.py b/FormulariesMatching/FormulariesMatching/uspmmg_db.py new file mode 100644 index 0000000..fc9e5e3 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/uspmmg_db.py @@ -0,0 +1,35 @@ +import psycopg as psyco +from datetime import datetime + +from flask import current_app, g + + + + +def get_all_formulary_groups(db_conn): + ''' + Get the list of active formulary groups + TODO: IMplement for the given formulary + ''' + sql = '''\ +select distinct "USP Category", "USP Class" +from "Formularies".usp_dc ud +order by "USP Category", "USP Class" +; +''' + + #query + with db_conn.cursor() as curse: + curse.execute(sql) + return curse.fetchall() + +def get_formulary_groups_per_NCTID(db_conn, nct_id): + ''' + Get the list of formulary groups associated with + the drugs found in a trial identified by NCTID + ''' + pass + + +def store_trial_to_formulary_group_matches(): + pass diff --git a/FormulariesMatching/FormulariesMatching/vaform_db.py b/FormulariesMatching/FormulariesMatching/vaform_db.py new file mode 100644 index 0000000..fc9e5e3 --- /dev/null +++ b/FormulariesMatching/FormulariesMatching/vaform_db.py @@ -0,0 +1,35 @@ +import psycopg as psyco +from datetime import datetime + +from flask import current_app, g + + + + +def get_all_formulary_groups(db_conn): + ''' + Get the list of active formulary groups + TODO: IMplement for the given formulary + ''' + sql = '''\ +select distinct "USP Category", "USP Class" +from "Formularies".usp_dc ud +order by "USP Category", "USP Class" +; +''' + + #query + with db_conn.cursor() as curse: + curse.execute(sql) + return curse.fetchall() + +def get_formulary_groups_per_NCTID(db_conn, nct_id): + ''' + Get the list of formulary groups associated with + the drugs found in a trial identified by NCTID + ''' + pass + + +def store_trial_to_formulary_group_matches(): + pass From 692559ee5a918e5b608be4b172f164a7955d3d72 Mon Sep 17 00:00:00 2001 From: will king Date: Mon, 25 Sep 2023 10:36:39 -0700 Subject: [PATCH 3/4] began work to get list of unmatched trials --- .../FormulariesMatching/db_interface.py | 42 +++++++++++++++++++ .../FormulariesMatching/matching.py | 7 ++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/FormulariesMatching/FormulariesMatching/db_interface.py b/FormulariesMatching/FormulariesMatching/db_interface.py index 93f5fe9..19c60de 100644 --- a/FormulariesMatching/FormulariesMatching/db_interface.py +++ b/FormulariesMatching/FormulariesMatching/db_interface.py @@ -81,3 +81,45 @@ where k.nct_id = %s return {"summary":summary, "keywords":keywords, "conditions":conditions} + +def get_trials_unmatched_to_formularies(db_conn): + """ + Get the NCT_IDs of trials not yet matchec to formularies + + For each formulary + get list + add to dsp + """ + + #setup sql for each formulary + uspdc_sql = '''\ +/* +Get the trials that have not been proceesed. +First: get most recent matched status +Second: only include those who have non-null status +Third: check list of trials against this and remove any of them. +This leaves unmatched trials. +*/ +select distinct(nct_id) +from "DiseaseBurden".trial_to_icd10 tti +where nct_id not in ( + select nct_id from "Formularies".uspdc_most_recent_matched_status umrms + where status is not null + ) +; ''' + uspmmg_sql = ''' Null; ''' + vaform_sql = ''' Null; ''' + + #query each formulary, adding data to dict + formulary_list = dict() + + with db_conn.cursor as curse: + + # uspdc + curse.execute(uspdc_sql) + formulary_list["uspdc"] = curse.fetchall() + + # uspmm + # vaform + + return formulary_list diff --git a/FormulariesMatching/FormulariesMatching/matching.py b/FormulariesMatching/FormulariesMatching/matching.py index 4326301..b5df1ff 100644 --- a/FormulariesMatching/FormulariesMatching/matching.py +++ b/FormulariesMatching/FormulariesMatching/matching.py @@ -6,6 +6,7 @@ from FormulariesMatching.db_interface import ( get_db, get_connection_details, get_trial_summary, + get_trials_unmatched_to_formularies, ) import FormulariesMatching.uspdc_db as uspdc import FormulariesMatching.uspmmg_db as uspmmg @@ -24,10 +25,10 @@ bp = Blueprint("formularies", __name__, url_prefix='/link/formularies') @bp.route("/", methods=['GET']) def get_remaining_trials(): #get db connection - #db_conn = get_db() + db_conn = get_db() - #get required data - connection_valid = get_connection_details() + #get list of trials + connection_valid = get_trials_unmatched_to_formularies(db_conn) #return html return connection_valid From d4527b6b7cd8f7c9dd4c90c73d0752d7674d763c Mon Sep 17 00:00:00 2001 From: Will King Date: Wed, 4 Oct 2023 09:23:17 -0700 Subject: [PATCH 4/4] current version of formulary linker --- .../FormulariesMatching/db_interface.py | 50 ++++++++++-- .../FormulariesMatching/matching.py | 44 ++++++++-- .../templates/formulary_index.html | 18 ++--- .../templates/trial_formularies.html | 80 ++++++++++++++++--- .../FormulariesMatching/uspdc_db.py | 20 ++++- .../FormulariesMatching/uspmmg_db.py | 37 +++++++-- FormulariesMatching/start.sh | 3 +- 7 files changed, 210 insertions(+), 42 deletions(-) diff --git a/FormulariesMatching/FormulariesMatching/db_interface.py b/FormulariesMatching/FormulariesMatching/db_interface.py index 19c60de..e7c0bf2 100644 --- a/FormulariesMatching/FormulariesMatching/db_interface.py +++ b/FormulariesMatching/FormulariesMatching/db_interface.py @@ -43,6 +43,7 @@ def check_connection(app): def get_trial_summary(db_conn,nct_id): sql_summary =""" +/*get brief and detailed descriptions*/ select s.nct_id, brief_title , @@ -64,22 +65,39 @@ where c.nct_id = %s ; """ sql_keywords=""" +/*get keywords*/ select nct_id ,downcase_name from ctgov.keywords k where k.nct_id = %s ; """ + + sql_indications=''' +select downcase_mesh_term +from ctgov.browse_interventions bi +where bi.nct_id = %s and mesh_type = 'mesh-list' +''' + + with db_conn.cursor() as curse: curse.execute(sql_summary,[nct_id]) summary = curse.fetchall() curse.execute(sql_keywords,[nct_id]) - keywords = curse.fetchall() + keywords = [ x[1] for x in curse.fetchall()] curse.execute(sql_conditions,[nct_id]) - conditions = curse.fetchall() + conditions = [ x[2] for x in curse.fetchall()] + + curse.execute(sql_indications,[nct_id]) + indications = [ x[0] for x in curse.fetchall()] - return {"summary":summary, "keywords":keywords, "conditions":conditions} + return { + "summary":summary, + "keywords":keywords, + "conditions":conditions, + "indications":indications + } def get_trials_unmatched_to_formularies(db_conn): @@ -106,20 +124,40 @@ where nct_id not in ( select nct_id from "Formularies".uspdc_most_recent_matched_status umrms where status is not null ) -; ''' - uspmmg_sql = ''' Null; ''' + and nct_id in (select distinct nct_id from public.formatted_data_mat fd ) +; +''' + uspmmg_sql = '''\ +/* +Get the trials that have not been proceesed. +First: get most recent matched status +Second: only include those who have non-null status +Third: check list of trials against this and remove any of them. +This leaves unmatched trials. +*/ +select distinct(nct_id) +from "DiseaseBurden".trial_to_icd10 tti +where nct_id not in ( + select nct_id from "Formularies".uspmmg_most_recent_matched_status umrms + where status is not null + ) + and nct_id in (select distinct nct_id from public.formatted_data_mat fd ) +; +''' vaform_sql = ''' Null; ''' #query each formulary, adding data to dict formulary_list = dict() - with db_conn.cursor as curse: + with db_conn.cursor() as curse: # uspdc curse.execute(uspdc_sql) formulary_list["uspdc"] = curse.fetchall() # uspmm + curse.execute(uspmmg_sql) + formulary_list["uspmmg"] = curse.fetchall() # vaform return formulary_list diff --git a/FormulariesMatching/FormulariesMatching/matching.py b/FormulariesMatching/FormulariesMatching/matching.py index b5df1ff..e1ea4b3 100644 --- a/FormulariesMatching/FormulariesMatching/matching.py +++ b/FormulariesMatching/FormulariesMatching/matching.py @@ -12,9 +12,11 @@ import FormulariesMatching.uspdc_db as uspdc import FormulariesMatching.uspmmg_db as uspmmg import FormulariesMatching.vaform_db as vaform +import re + FORMULARIES = { "USP DC":uspdc, -# "USP MMG":uspmmg, + "USP MMG":uspmmg, # "VA Formulary":vaform, } @@ -28,10 +30,14 @@ def get_remaining_trials(): db_conn = get_db() #get list of trials - connection_valid = get_trials_unmatched_to_formularies(db_conn) + unmatched_trials = get_trials_unmatched_to_formularies(db_conn) + unmatched_trials_list = list(set([ x[0] for y in unmatched_trials for x in unmatched_trials[y]] )) #return html - return connection_valid +# return {"formularies":unmatched_trials,"nctids":nctid_list} + return render_template('formulary_index.html', + unmatched_trials=unmatched_trials_list, + ) @bp.route("/", methods=['GET',"POST"]) @@ -62,8 +68,36 @@ def match_trial_to_formulary_groups(nct_id): ) elif request.method == "POST": - pass - #build array of data to insert + + #For each Formulary + translation={"('":None,"')":None} + + for key in request.form.keys(): + match key.split("|"): + case ["select_box", formulary]: + #parse data + result = request.form["select_box|{}".format(formulary)] + if result == 'difficult': + FORMULARIES[formulary].insert_match(db_conn,nct_id,None,None,'difficult') + else: + category,uspclass = [re.sub("['\(\)]","",x).strip() for x in result.split("', '")] + + FORMULARIES[formulary].insert_match(db_conn,nct_id,category,uspclass,'accepted') + + + case ["check_box", data]: + formulary_trial,category,uspclass = [re.sub("['\(\)]","",x).strip() for x in data.split("', '")] + formulary,_ = formulary_trial.split(",") + #Insert data + + FORMULARIES[formulary].insert_match(db_conn,nct_id,category,uspclass,'accepted') + + case _: + return 400 + + return redirect(url_for("formularies.get_remaining_trials")) + + else: raise Exception("HTTP method <{}> not implemented".format(request.method)) diff --git a/FormulariesMatching/FormulariesMatching/templates/formulary_index.html b/FormulariesMatching/FormulariesMatching/templates/formulary_index.html index 459d4bb..fe330b5 100644 --- a/FormulariesMatching/FormulariesMatching/templates/formulary_index.html +++ b/FormulariesMatching/FormulariesMatching/templates/formulary_index.html @@ -8,16 +8,14 @@

Unlinked Trials

- - -{% for trial in unlinked_trials %} - -{% endfor %} -
Trials
- - {{ trial [0] }} - -
+

Linked Trials

diff --git a/FormulariesMatching/FormulariesMatching/templates/trial_formularies.html b/FormulariesMatching/FormulariesMatching/templates/trial_formularies.html index a14158b..be5d855 100644 --- a/FormulariesMatching/FormulariesMatching/templates/trial_formularies.html +++ b/FormulariesMatching/FormulariesMatching/templates/trial_formularies.html @@ -10,18 +10,75 @@ {% block content %} -Based on the drugs associated with a trial, -the following are the formulary group suggestions. +
+

Trial Summary

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Short Title{{ summary['summary'][0][1] }}
Complete Title{{ summary['summary'][0][2] }}
Summary{{ summary['summary'][0][3] }}
Summary 2{{ summary['summary'][0][4] }}
Indicated Drugs +
    + {% for drug in summary['indications'] %} +
  • + + {{drug}} + +
    • + {% endfor %} +
+
Conditions + +
keywords{{ summary['keywords'] }}
+
-
+ +

Matching Classes

{% for formulary in potential_matches %}
-

+

{{ formulary }} -

+

-{% for formulary in potential_matches %} @@ -34,9 +91,9 @@ the following are the formulary group suggestions. @@ -47,22 +104,21 @@ the following are the formulary group suggestions.

If you've determined it belongs to another class - {% for option in class_lists[formulary] %} - {% endfor %} -{% endfor %} {% endfor %} diff --git a/FormulariesMatching/FormulariesMatching/uspdc_db.py b/FormulariesMatching/FormulariesMatching/uspdc_db.py index 60b9688..47e9c7b 100644 --- a/FormulariesMatching/FormulariesMatching/uspdc_db.py +++ b/FormulariesMatching/FormulariesMatching/uspdc_db.py @@ -32,7 +32,7 @@ def get_groups_per_nctid(db_conn, nct_id): ''' pass sql = '''\ -select * from "Formularies".trial_to_uspdc_category_class ttucc +select * from "Formularies".uspdc_trial_to_category_class ttucc where nct_id = %(nctid)s ; ''' @@ -43,5 +43,19 @@ where nct_id = %(nctid)s return curse.fetchall() -def store_trial_to_formulary_group_matches(): - pass +def insert_match(db_conn, nct_id, category,uspclass,status): + sql = '''\ +INSERT INTO "Formularies".uspdc_matching +VALUES +( + %(nct_id)s + ,%(category)s + ,%(uspclass)s + ,%(status)s + ,NOW() +) +; +''' + with db_conn.cursor() as curse: + curse.execute(sql, {"nct_id":nct_id, "category":category, "uspclass":uspclass, "status":status} ) + db_conn.commit() diff --git a/FormulariesMatching/FormulariesMatching/uspmmg_db.py b/FormulariesMatching/FormulariesMatching/uspmmg_db.py index fc9e5e3..02edcaa 100644 --- a/FormulariesMatching/FormulariesMatching/uspmmg_db.py +++ b/FormulariesMatching/FormulariesMatching/uspmmg_db.py @@ -13,23 +13,50 @@ def get_all_formulary_groups(db_conn): ''' sql = '''\ select distinct "USP Category", "USP Class" -from "Formularies".usp_dc ud +from "Formularies".usp_mmg ud order by "USP Category", "USP Class" ; ''' - #query with db_conn.cursor() as curse: curse.execute(sql) return curse.fetchall() + -def get_formulary_groups_per_NCTID(db_conn, nct_id): + + +def get_groups_per_nctid(db_conn, nct_id): ''' Get the list of formulary groups associated with the drugs found in a trial identified by NCTID ''' pass + sql = '''\ +select * from "Formularies".uspmmg_trial_to_category_class ttucc +where nct_id = %(nctid)s +; +''' + #query + with db_conn.cursor() as curse: + curse.execute(sql, {"nctid":nct_id}) + return curse.fetchall() -def store_trial_to_formulary_group_matches(): - pass + + +def insert_match(db_conn, nct_id, category,uspclass,status): + sql = '''\ +INSERT INTO "Formularies".uspmmg_matching +VALUES +( + %(nct_id)s + ,%(category)s + ,%(uspclass)s + ,%(status)s + ,NOW() +) +; +''' + with db_conn.cursor() as curse: + curse.execute(sql, {"nct_id":nct_id, "category":category, "uspclass":uspclass, "status":status} ) + db_conn.commit() diff --git a/FormulariesMatching/start.sh b/FormulariesMatching/start.sh index 6868a58..630b60f 100755 --- a/FormulariesMatching/start.sh +++ b/FormulariesMatching/start.sh @@ -1 +1,2 @@ -waitress-serve --port=5000 --call 'Icd10ConditionsMatching:create_app' +waitress-serve --port=5000 --call 'FormulariesMatching:create_app' +#flask --app FormulariesMatching run --debug

{{ row[1] }} {{ row[2] }}