diff --git a/Scripts/DevelopingLinks.sql b/Scripts/DevelopingLinks.sql index 37e6367..1aed12f 100644 --- a/Scripts/DevelopingLinks.sql +++ b/Scripts/DevelopingLinks.sql @@ -1,4 +1,9 @@ drop table if exists "DiseaseBurden".trial_to_icd10; +drop type if exists "DiseaseBurden".validation_type; + +create type "DiseaseBurden".validation_type as enum ('accepted', 'rejected', 'unmatched'); +comment on type "DiseaseBurden".validation_type is 'This is used to record interactions with each type. It can be accepted (yes this should be used), rejected (no this doesn`t match), or unmatched (where non of the proposed options match)'; + CREATE TABLE "DiseaseBurden".trial_to_icd10 ( id integer NOT NULL GENERATED ALWAYS AS IDENTITY, @@ -9,7 +14,9 @@ CREATE TABLE "DiseaseBurden".trial_to_icd10 ( rootsource varchar NULL, "name" varchar NULL, "source" varchar null, - CONSTRAINT trial_to_icd10_pk PRIMARY KEY (id) + approved "DiseaseBurden".validation_type, + approval_timestamp timestamp, + CONSTRAINT trial_to_icd10_pk PRIMARY KEY (id) ); comment on type "DiseaseBurden".trial_to_icd10 is 'This represents potential links between trials and icd10 codes. Most of the links are both automatic and wrong.'; @@ -27,17 +34,3 @@ CREATE TABLE "DiseaseBurden".icd10_to_cause ( -drop table if exists "DiseaseBurden".match_status; -drop type if exists "DiseaseBurden".validation_type; - -create type "DiseaseBurden".validation_type as enum ('accepted', 'rejected', 'unmatched'); -comment on type "DiseaseBurden".validation_type is 'This is used to record interactions with each type. It can be accepted (yes this should be used), rejected (no this doesn`t match), or unmatched (where non of the proposed options match)'; - -CREATE TABLE "DiseaseBurden".match_status ( - id serial4 NOT NULL, - approved "DiseaseBurden".validation_type NOT NULL, - approval_timestamp timestamp NOT NULL, - CONSTRAINT match_status_pk PRIMARY KEY (id, approval_timestamp) -); -COMMENT ON TABLE "DiseaseBurden".match_status IS 'This allows me to record if a particular proposed match is approved or not.'; - diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/db_interface.py b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/db_interface.py index 6c8a897..523ee52 100644 --- a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/db_interface.py +++ b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/db_interface.py @@ -45,11 +45,7 @@ def select_remaing_trials_to_analyze(db_conn): sql = ''' select distinct nct_id from "DiseaseBurden".trial_to_icd10 tti - where tti.id not in - ( - select distinct id - from "DiseaseBurden".match_status - ) + where tti.approved is null order by nct_id ; ''' @@ -63,15 +59,11 @@ def select_analyzed_trials(db_conn): This will get the set of trials that have been analyzed. ''' sql = ''' - select distinct nct_id + select distinct nct_id, max(approval_timestamp) from "DiseaseBurden".trial_to_icd10 tti - where tti.id in - ( - select distinct id - from "DiseaseBurden".match_status - where approved in ('accepted','rejected') - ) - order by nct_id + where tti.approved in ('accepted','rejected') + group by nct_id + order by max(approval_timestamp) desc ; ''' with db_conn.cursor() as cursor: @@ -85,12 +77,7 @@ def select_unmatched_trials(db_conn): sql = ''' select distinct nct_id from "DiseaseBurden".trial_to_icd10 tti - where tti.id in - ( - select distinct id - from "DiseaseBurden".match_status - where approved = 'unmatched' - ) + where tti.approved = 'unmatched' order by nct_id ; ''' @@ -110,14 +97,16 @@ def get_trial_conditions_and_proposed_matches(db_conn, nct_id): return cursor.fetchall() -def store_validation(db_conn, list_of_inserts): +def store_validation(db_conn, list_of_insert_data): sql = """ - insert into "DiseaseBurden".match_status (id, approved, approval_timestamp) - values %s + update "DiseaseBurden".trial_to_icd10 + set approved=%s, approval_timestamp=%s + where id=%s ; """ with db_conn.cursor() as cursor: - extras.execute_values(cursor, sql, list_of_inserts) + for l in list_of_insert_data: + cursor.execute(sql, l) db_conn.commit() def get_trial_summary(db_conn,nct_id): @@ -175,18 +164,12 @@ def get_list_icd10_codes(db_conn): def record_suggested_matches(db_conn, nct_id,condition,icd10_code): sql1 = """ INSERT INTO "DiseaseBurden".trial_to_icd10 - (nct_id,"condition",ui,"source") - VALUES (%s,%s,%s,'hand matched') - returning id + (nct_id,"condition",ui,"source",approved,approval_timestamp) + VALUES (%s,%s,%s,'hand matched','accepted',%s) ; """ - sql2 = ''' - INSERT INTO "DiseaseBurden".match_status (id,approved,approval_timestamp) - VALUES (%s,%s,%s) - ''' + with db_conn.cursor() as curse: - curse.execute(sql1,[nct_id,condition,icd10_code]) - id = curse.fetchone()[0] - curse.execute(sql2,[id,"accepted",datetime.now()]) - db_conn.commit() \ No newline at end of file + curse.execute(sql1,[nct_id,condition,icd10_code,datetime.now()]) + db_conn.commit() diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/base.html b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/base.html index cfbf613..75ecb98 100644 --- a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/base.html +++ b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/base.html @@ -5,7 +5,15 @@ diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_index.html b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_index.html index 6de2145..761bc31 100644 --- a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_index.html +++ b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_index.html @@ -27,7 +27,8 @@ {{ trial [0] }} - + + (Most recently updated {{trial[1]}}) {% endfor %} diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_of_trial.html b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_of_trial.html index 9d2e0c2..be256fd 100644 --- a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_of_trial.html +++ b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/templates/validation_of_trial.html @@ -55,7 +55,7 @@ {% for condition in condition_list %} - + {{condition[2]}} {{condition[3]}} {{condition[5]}} diff --git a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/validation.py b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/validation.py index d742a53..cc50604 100644 --- a/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/validation.py +++ b/scripts/Icd10ConditionsMatching/Icd10ConditionsMatching/validation.py @@ -67,29 +67,31 @@ def validate_trial(nct_id): #if match id in submitted form, mark as approved, otherwise mark as rejected for condition in condition_list: id = condition[0] - list_of_insert_data.append((id, request.form.get(str(id),"rejected"),datetime.now())) - + list_of_insert_data.append((request.form.get(str(id),"rejected"), datetime.now(),id)) + print(list_of_insert_data) store_validation(db_conn, list_of_insert_data) return redirect(url_for("validation.remaining")) elif "marked_unmatched" in request.form: #if this was marked as "unmatched", store that for each entry. for condition in condition_list: id = condition[0] - list_of_insert_data.append((id, "unmatched",datetime.now())) + list_of_insert_data.append(( "unmatched", datetime.now(), id)) store_validation(db_conn, list_of_insert_data) return redirect(url_for("validation.remaining")) elif "alternate_submission" in request.form: code = request.form["alt_sub"] - code = code.strip().replace("\.",'').ljust(7,"-") + code = code.strip().replace(".",'').ljust(7,"-") condition = request.form["condition"].strip() codelist = get_list_icd10_codes(db_conn) if code in codelist: record_suggested_matches(db_conn, nct_id, condition, code) - return redirect(url_for("validation.remaining")) + return redirect(request.path) else: - return "Entered `{}`, which is not in the list of available ICD-10 codes".format(code.strip("-")), 422 + return """ + Entered `{}`, which is not in the list of available ICD-10 codes. Return to trial summary + """.format(code.strip("-"),request.path), 422 diff --git a/scripts/umls_requests.py b/scripts/umls_requests.py index ccd7662..8652a83 100644 --- a/scripts/umls_requests.py +++ b/scripts/umls_requests.py @@ -61,9 +61,9 @@ with postgres_conn() as pconn, pconn.cursor(cursor_factory=extras.DictCursor) as sql_insert = """ INSERT INTO "DiseaseBurden".trial_to_icd10 - (nct_id, "condition", ui,uri,rootsource,"name","source") + (nct_id, "condition", ui,uri,rootsource,"name","source",approved,approval_timestamp) VALUES - (%(nct_id)s, %(condition)s, %(ui)s, %(uri)s, %(rootSource)s, %(name)s, 'UMLS API search') + (%(nct_id)s, %(condition)s, %(ui)s, %(uri)s, %(rootSource)s, %(name)s, 'UMLS API search', null,null) """ for entry in tqdm(entries,desc="Inserting entries to DB"): pcurse.execute(sql_insert,entry._asdict())