From 2aa8fdee4e1849e91484b327174c0c51ecf7210f Mon Sep 17 00:00:00 2001 From: youainti Date: Wed, 1 Jun 2022 19:53:23 -0700 Subject: [PATCH] reservations working properly. Still missing rate control --- history_downloader/downloader.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/history_downloader/downloader.py b/history_downloader/downloader.py index 0a0c2d3..cbe0855 100644 --- a/history_downloader/downloader.py +++ b/history_downloader/downloader.py @@ -189,8 +189,7 @@ def reserve_trials(db_connection, limit=10): query = """ WITH OF_INTEREST AS (SELECT NCT_ID - FROM HTTP.MOST_RECENT_DOWNLOAD_STATUS - WHERE HTTP.MOST_RECENT_DOWNLOAD_STATUS.STATUS = 'Of Interest'::HTTP.HISTORY_DOWNLOAD_STATUS + FROM HTTP.TRIALS_TO_DOWNLOAD LIMIT %s ) INSERT INTO HTTP.DOWNLOAD_STATUS (NCT_ID,STATUS) @@ -229,11 +228,11 @@ if __name__ == "__main__": #get list of nct_ids - nctids = reserve_trials(con, 4) + nctids = reserve_trials(con, 100) print(nctids) #start analyzing them - with Pool(processes=3) as process_pool: + with Pool(processes=4) as process_pool: process_pool.map(downloader, nctids)