lots of updates while integrating rxnav and aact into one dockerfile

.gitignore

@@ -187,5 +187,7 @@ NCT*.html
 containers/RxNav-In-a-box/rxnav_data/*
 containers/RxNav-In-a-box/rxnav-in-a-box-*
 #Ignore stuff from AACT_downlaoder
-containers/db_store/*
+containers/AACT_downloader/postgresql/*
+containers/AACT_downloader/aact_downloads/*
+
 

containers/AACT_downloader/db_store/readme.txt

@@ -1 +0,0 @@
-This is where the db is stored.

containers/AACT_downloader/docker-compose.yaml

@@ -23,5 +23,7 @@ services:
       - ./aact_downloads/postgres_data.dmp:/mnt/host_data/postgres_data.dmp
       # this is the folder containing entrypoint info.
       - ./docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/
+    env_file:
+      ../.env
 
 

containers/AACT_downloader/docker-entrypoint-initdb.d/040_StructuredProductLabels_Schema.sql

@@ -1,3 +1,4 @@
+CREATE SCHEMA spl AUTHORIZATION root;
 
 DROP TABLE IF EXISTS spl.nsde;
 

containers/RxNav-In-a-box/docker-compose.yml

@@ -1,15 +0,0 @@
-version: '3.1'
-
-services:
-  rxnav-db:
-    image: mariadb:10.4
-    restart: "no"
-    expose:
-    - "3306"
-    volumes:
-      - ./RxNav-In-a-box/rxnav-in-a-box/mysql:/docker-entrypoint-initdb.d:ro
-      - ./RxNav-In-a-box/rxnav_data:/
-    environment:
-      MYSQL_RANDOM_ROOT_PASSWORD: "yes"
-      MYSQL_USER: ${MYSQL_USER}
-      MYSQL_PASSWORD: ${MYSQL_PASSWORD}

containers/docker-compose.yaml

@@ -2,10 +2,12 @@ version: '3'
 
 networks:
   pharmaceutical_research: #because it helps to have a way to link specifically to this. 
+    external: true
 
 services:
   aact_db:
     image: postgres:14-alpine
+    restart: "no"
     networks:
       - pharmaceutical_research
     container_name: aact_db
@@ -14,26 +16,32 @@ services:
       POSTGRES_USER: ${POSTGRES_USER}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
       POSTGRES_DB: ${POSTGRES_DB}
+      VERSION: podman
     ports:
       - "5432:5432" #host:container
     volumes: #host:container is the format.
       # this is persistant storage for the database  
-      - ./db_store/:/var/lib/postgresql/
+      - ./AACT_downloader/postgresql/:/var/lib/postgresql/
       # this is the database dump to restore from
-      - ./aact_downloads/postgres_data.dmp:/mnt/host_data/postgres_data.dmp
+      - ./AACT_downloader/aact_downloads/postgres_data.dmp:/mnt/host_data/postgres_data.dmp
       # this is the folder containing entrypoint info.
-      - ./docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/
+      - ./AACT_downloader/docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/
 
 
   rxnav-db:
     image: mariadb:10.4
     restart: "no"
     ports:
-      - "3306":"3306"
+      - "3306:3306"
     volumes:
-      - ./RxNav-In-a-box/rxnav-in-a-box/mysql:/docker-entrypoint-initdb.d:ro
+      - ./RxNav-In-a-box/rxnav-in-a-box-20230103/mysql:/docker-entrypoint-initdb.d:ro
-      - ./RxNav-In-a-box/rxnav_data:/
+      - ./RxNav-In-a-box/rxnav_data:/var/lib/mysql
     environment:
       MYSQL_RANDOM_ROOT_PASSWORD: "yes"
       MYSQL_USER: ${MYSQL_USER}
       MYSQL_PASSWORD: ${MYSQL_PASSWORD}
+      VERSION: podman
+    env_file:
+      .env
+    networks:
+      - pharmaceutical_research

history_downloader/downloader.py

@@ -6,6 +6,7 @@ from multiprocessing import Pool, Value
 import math
 import time
 import argparse
+from dotenv import dotenv_values
 
 def get_highest_version_number(response):
     """
@@ -222,6 +223,8 @@ def reserve_trials(db_connection, limit=10):
     return nctids_list
 
 if __name__ == "__main__":
+    env_path = ''
+    config = dotenv_values(env_path)
     """
     Main!
     """

justfile

@@ -5,19 +5,23 @@
 #   - move postgress login credentials (allow them to be printed from just while setting up)
 
 
-data_link := "https://ctti-aact.nyc3.digitaloceanspaces.com/27grtsnhtccplxapj2o8ak9aotvv"
+#paths for aact_db (postgres)
-data_file := "2022-12-23_postgres_data.zip"
+aact_download_link := "https://ctti-aact.nyc3.digitaloceanspaces.com/27grtsnhtccplxapj2o8ak9aotvv"
-data_path := "./AACT_downloader/aact_downloads"
+aact_download_file := "2022-12-23_postgres_data.zip"
-data_filepath := data_path / data_file
+aact_download_path := "./containers/AACT_downloader/aact_downloads"
+aact_zipped_data_filepath := aact_download_path / aact_download_file
 
 #must match the 'container name: aact_db' in the docker-compose.yaml
-docker_container := `docker container ls -a | grep aact_db | cut -f 1 -d " " | tr "\n" " "`
+docker_container := `docker container ls -a | grep "aact_db|rxnav_db" | cut -f 1 -d " " | tr "\n" " "`
+
+#paths for rxnavinabox
+rxnav_path := "./containers/RxNav-In-a-box"
+rxnav_version := "rxnav-in-a-box-20230103"
+rxnav_data_path := rxnav_path / rxnav_version / "mysql" / "02_data.sql"
 
 #Various paths for docker stuff
-docker-compose_path := "./AACT_downloader/docker-compose.yaml"
+docker-compose_path := "./containers/docker-compose.yaml"
 
-#rxnorm_mappings
-rxnorm_mappings_url := "https://dailymed-data.nlm.nih.gov/public-release-files/rxnorm_mappings.zip"
 
 #Number of historical trials to download.
 count := "100"
@@ -28,19 +32,24 @@ check-status:
     docker --version
     #check if python version > 3.10. 
     python --version
-    python -c 'import sys; exit(sys.hexversion >= 50859504)'
+    #python -c 'import sys; exit(sys.hexversion >= 50859504)'
     curl --version
     echo "current docker containers:{{docker_container}}"
 
-#Setup the AACT container
+
 setup-containers: 
+    echo "todo"
     @echo "Check for downloaded data"
-    [ -s {{data_path}}/postgres_data.dmp ]
+    #aact
+    [ -s {{aact_download_path}}/postgres_data.dmp ]
+    #rxnav
+    [ -s {{rxnav_data_path}} ]
 
     #run docker compose
-    @echo "Setting up AACT container"
+    @echo "Setting up AACT_db & RxNav_db container"
     docker-compose -f {{docker-compose_path}} up -d
     
+
 #Stop the appropriate docker container
 stop-containers:
     #stop all docker containers if they are currently running.
@@ -60,10 +69,13 @@ clean-docker: stop-containers
 
 #Download the AACT data
 download-aact-data:
-    curl {{data_link}} > ./AACT_downloader/aact_downloads/{{data_file}}
+    #download
-    unzip {{data_filepath}} -d {{data_path}}
+    curl {{aact_download_link}} > {{aact_zipped_data_filepath}}
-    rm {{data_filepath}}
+    unzip {{aact_zipped_data_filepath}} -d {{aact_download_path}}
+    rm {{aact_zipped_data_filepath}}
 
+download-rxnav-data:
+    echo "Currently manually downloaded."
 
 #build based on previously downloaded data
 build: check-status setup-containers
@@ -105,8 +117,3 @@ get-nsde:
     cd market_data && bash download_nsde.sh
     cd market_data && python extract_nsde.py
 
-get-rxnorm-mappings:
-    #this may not be needed, all it does is match spls to rxcuis and I think I already have that.
-    curl {{rxnorm_mappings_url}} > ./market_data/rxnorm_mappings.zip
-    cd ./market_data && unzip ./rxnorm_mappings.zip
-    rm ./market_data/rxnorm_mappings.zip