From 47996ba60750894952154ffa4800ee726c4d7b49 Mon Sep 17 00:00:00 2001 From: will king Date: Mon, 27 Mar 2023 16:14:44 -0700 Subject: [PATCH] updated import-icd10... to use execute_values to speed up insert. --- scripts/import-icd10_to_cause.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts/import-icd10_to_cause.py b/scripts/import-icd10_to_cause.py index fea46e3..0eb7f08 100644 --- a/scripts/import-icd10_to_cause.py +++ b/scripts/import-icd10_to_cause.py @@ -1,6 +1,7 @@ from drugtools.env_setup import ENV,postgres_conn from psycopg2 import extras from collections import namedtuple +from tqdm import tqdm FILES=[ @@ -12,21 +13,24 @@ SEP="|" sql = """ INSERT INTO "DiseaseBurden".icd10_to_cause (code,cause_text) -VALUES - (%(code)s,%(cause)s) +VALUES %s """ -#read in files -#adjust codes? -#load files to table with postgres_conn() as pconn, pconn.cursor(cursor_factory=extras.DictCursor) as pcurse: + entries = [] for fpath in FILES: + print(fpath) + with open(fpath,"r") as fh: - for line in fh.readlines(): + for line in tqdm(fh.readlines(),desc=fpath): code,cause = line.split(SEP) code = code.strip() cause = cause.strip() - pcurse.execute(sql,{"code":code,"cause":cause}) \ No newline at end of file + entries.append((code,cause)) + + extras.execute_values(pcurse, sql , entries) + +