From 804a90c24768fc2139143712b7fca3af58335cc5 Mon Sep 17 00:00:00 2001 From: youainti Date: Tue, 7 Feb 2023 15:45:08 -0800 Subject: [PATCH] lots of updates while integrating rxnav and aact into one dockerfile --- .gitignore | 4 +- ...ING NCTIDs to NDCs and Marketing dates.sql | 0 {RxMix => RxNav}/migrate_mysql2pgsql.py | 0 {RxMix => RxNav}/views to history.sql | 0 .../AACT_downloader/db_store/readme.txt | 1 - .../AACT_downloader/docker-compose.yaml | 2 + .../040_StructuredProductLabels_Schema.sql | 1 + containers/RxNav-In-a-box/docker-compose.yml | 15 ------ containers/docker-compose.yaml | 20 +++++--- history_downloader/downloader.py | 3 ++ justfile | 47 +++++++++++-------- 11 files changed, 50 insertions(+), 43 deletions(-) rename {RxMix => RxNav}/ASSOICATING NCTIDs to NDCs and Marketing dates.sql (100%) rename {RxMix => RxNav}/migrate_mysql2pgsql.py (100%) rename {RxMix => RxNav}/views to history.sql (100%) delete mode 100644 containers/AACT_downloader/db_store/readme.txt delete mode 100644 containers/RxNav-In-a-box/docker-compose.yml diff --git a/.gitignore b/.gitignore index cacea38..9da644c 100644 --- a/.gitignore +++ b/.gitignore @@ -187,5 +187,7 @@ NCT*.html containers/RxNav-In-a-box/rxnav_data/* containers/RxNav-In-a-box/rxnav-in-a-box-* #Ignore stuff from AACT_downlaoder -containers/db_store/* +containers/AACT_downloader/postgresql/* +containers/AACT_downloader/aact_downloads/* + diff --git a/RxMix/ASSOICATING NCTIDs to NDCs and Marketing dates.sql b/RxNav/ASSOICATING NCTIDs to NDCs and Marketing dates.sql similarity index 100% rename from RxMix/ASSOICATING NCTIDs to NDCs and Marketing dates.sql rename to RxNav/ASSOICATING NCTIDs to NDCs and Marketing dates.sql diff --git a/RxMix/migrate_mysql2pgsql.py b/RxNav/migrate_mysql2pgsql.py similarity index 100% rename from RxMix/migrate_mysql2pgsql.py rename to RxNav/migrate_mysql2pgsql.py diff --git a/RxMix/views to history.sql b/RxNav/views to history.sql similarity index 100% rename from RxMix/views to history.sql rename to RxNav/views to history.sql diff --git a/containers/AACT_downloader/db_store/readme.txt b/containers/AACT_downloader/db_store/readme.txt deleted file mode 100644 index 43ae271..0000000 --- a/containers/AACT_downloader/db_store/readme.txt +++ /dev/null @@ -1 +0,0 @@ -This is where the db is stored. diff --git a/containers/AACT_downloader/docker-compose.yaml b/containers/AACT_downloader/docker-compose.yaml index dffa8d0..ae44db9 100644 --- a/containers/AACT_downloader/docker-compose.yaml +++ b/containers/AACT_downloader/docker-compose.yaml @@ -23,5 +23,7 @@ services: - ./aact_downloads/postgres_data.dmp:/mnt/host_data/postgres_data.dmp # this is the folder containing entrypoint info. - ./docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/ + env_file: + ../.env diff --git a/containers/AACT_downloader/docker-entrypoint-initdb.d/040_StructuredProductLabels_Schema.sql b/containers/AACT_downloader/docker-entrypoint-initdb.d/040_StructuredProductLabels_Schema.sql index 7aaa9bf..c072b54 100644 --- a/containers/AACT_downloader/docker-entrypoint-initdb.d/040_StructuredProductLabels_Schema.sql +++ b/containers/AACT_downloader/docker-entrypoint-initdb.d/040_StructuredProductLabels_Schema.sql @@ -1,3 +1,4 @@ +CREATE SCHEMA spl AUTHORIZATION root; DROP TABLE IF EXISTS spl.nsde; diff --git a/containers/RxNav-In-a-box/docker-compose.yml b/containers/RxNav-In-a-box/docker-compose.yml deleted file mode 100644 index 350e795..0000000 --- a/containers/RxNav-In-a-box/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '3.1' - -services: - rxnav-db: - image: mariadb:10.4 - restart: "no" - expose: - - "3306" - volumes: - - ./RxNav-In-a-box/rxnav-in-a-box/mysql:/docker-entrypoint-initdb.d:ro - - ./RxNav-In-a-box/rxnav_data:/ - environment: - MYSQL_RANDOM_ROOT_PASSWORD: "yes" - MYSQL_USER: ${MYSQL_USER} - MYSQL_PASSWORD: ${MYSQL_PASSWORD} diff --git a/containers/docker-compose.yaml b/containers/docker-compose.yaml index 167c877..d878c12 100644 --- a/containers/docker-compose.yaml +++ b/containers/docker-compose.yaml @@ -2,10 +2,12 @@ version: '3' networks: pharmaceutical_research: #because it helps to have a way to link specifically to this. + external: true services: aact_db: image: postgres:14-alpine + restart: "no" networks: - pharmaceutical_research container_name: aact_db @@ -14,26 +16,32 @@ services: POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: ${POSTGRES_DB} + VERSION: podman ports: - "5432:5432" #host:container volumes: #host:container is the format. # this is persistant storage for the database - - ./db_store/:/var/lib/postgresql/ + - ./AACT_downloader/postgresql/:/var/lib/postgresql/ # this is the database dump to restore from - - ./aact_downloads/postgres_data.dmp:/mnt/host_data/postgres_data.dmp + - ./AACT_downloader/aact_downloads/postgres_data.dmp:/mnt/host_data/postgres_data.dmp # this is the folder containing entrypoint info. - - ./docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/ + - ./AACT_downloader/docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/ rxnav-db: image: mariadb:10.4 restart: "no" ports: - - "3306":"3306" + - "3306:3306" volumes: - - ./RxNav-In-a-box/rxnav-in-a-box/mysql:/docker-entrypoint-initdb.d:ro - - ./RxNav-In-a-box/rxnav_data:/ + - ./RxNav-In-a-box/rxnav-in-a-box-20230103/mysql:/docker-entrypoint-initdb.d:ro + - ./RxNav-In-a-box/rxnav_data:/var/lib/mysql environment: MYSQL_RANDOM_ROOT_PASSWORD: "yes" MYSQL_USER: ${MYSQL_USER} MYSQL_PASSWORD: ${MYSQL_PASSWORD} + VERSION: podman + env_file: + .env + networks: + - pharmaceutical_research diff --git a/history_downloader/downloader.py b/history_downloader/downloader.py index 10b878b..81d52bc 100644 --- a/history_downloader/downloader.py +++ b/history_downloader/downloader.py @@ -6,6 +6,7 @@ from multiprocessing import Pool, Value import math import time import argparse +from dotenv import dotenv_values def get_highest_version_number(response): """ @@ -222,6 +223,8 @@ def reserve_trials(db_connection, limit=10): return nctids_list if __name__ == "__main__": + env_path = '' + config = dotenv_values(env_path) """ Main! """ diff --git a/justfile b/justfile index 11f779a..a3691ed 100644 --- a/justfile +++ b/justfile @@ -5,19 +5,23 @@ # - move postgress login credentials (allow them to be printed from just while setting up) -data_link := "https://ctti-aact.nyc3.digitaloceanspaces.com/27grtsnhtccplxapj2o8ak9aotvv" -data_file := "2022-12-23_postgres_data.zip" -data_path := "./AACT_downloader/aact_downloads" -data_filepath := data_path / data_file +#paths for aact_db (postgres) +aact_download_link := "https://ctti-aact.nyc3.digitaloceanspaces.com/27grtsnhtccplxapj2o8ak9aotvv" +aact_download_file := "2022-12-23_postgres_data.zip" +aact_download_path := "./containers/AACT_downloader/aact_downloads" +aact_zipped_data_filepath := aact_download_path / aact_download_file #must match the 'container name: aact_db' in the docker-compose.yaml -docker_container := `docker container ls -a | grep aact_db | cut -f 1 -d " " | tr "\n" " "` +docker_container := `docker container ls -a | grep "aact_db|rxnav_db" | cut -f 1 -d " " | tr "\n" " "` + +#paths for rxnavinabox +rxnav_path := "./containers/RxNav-In-a-box" +rxnav_version := "rxnav-in-a-box-20230103" +rxnav_data_path := rxnav_path / rxnav_version / "mysql" / "02_data.sql" #Various paths for docker stuff -docker-compose_path := "./AACT_downloader/docker-compose.yaml" +docker-compose_path := "./containers/docker-compose.yaml" -#rxnorm_mappings -rxnorm_mappings_url := "https://dailymed-data.nlm.nih.gov/public-release-files/rxnorm_mappings.zip" #Number of historical trials to download. count := "100" @@ -28,18 +32,23 @@ check-status: docker --version #check if python version > 3.10. python --version - python -c 'import sys; exit(sys.hexversion >= 50859504)' + #python -c 'import sys; exit(sys.hexversion >= 50859504)' curl --version echo "current docker containers:{{docker_container}}" -#Setup the AACT container + setup-containers: + echo "todo" @echo "Check for downloaded data" - [ -s {{data_path}}/postgres_data.dmp ] + #aact + [ -s {{aact_download_path}}/postgres_data.dmp ] + #rxnav + [ -s {{rxnav_data_path}} ] #run docker compose - @echo "Setting up AACT container" + @echo "Setting up AACT_db & RxNav_db container" docker-compose -f {{docker-compose_path}} up -d + #Stop the appropriate docker container stop-containers: @@ -60,10 +69,13 @@ clean-docker: stop-containers #Download the AACT data download-aact-data: - curl {{data_link}} > ./AACT_downloader/aact_downloads/{{data_file}} - unzip {{data_filepath}} -d {{data_path}} - rm {{data_filepath}} + #download + curl {{aact_download_link}} > {{aact_zipped_data_filepath}} + unzip {{aact_zipped_data_filepath}} -d {{aact_download_path}} + rm {{aact_zipped_data_filepath}} +download-rxnav-data: + echo "Currently manually downloaded." #build based on previously downloaded data build: check-status setup-containers @@ -105,8 +117,3 @@ get-nsde: cd market_data && bash download_nsde.sh cd market_data && python extract_nsde.py -get-rxnorm-mappings: - #this may not be needed, all it does is match spls to rxcuis and I think I already have that. - curl {{rxnorm_mappings_url}} > ./market_data/rxnorm_mappings.zip - cd ./market_data && unzip ./rxnorm_mappings.zip - rm ./market_data/rxnorm_mappings.zip