From 66becd07716f98e0e35521fe9c057d83993922f7 Mon Sep 17 00:00:00 2001
From: Will King <will@youainti.com>
Date: Fri, 27 Sep 2024 18:55:29 -0700
Subject: [PATCH] Created basic outline for data processing

---
 .gitignore           |  9 +++++++++
 README.md            | 27 ++++++++++++++++++++++++++-
 codebooks/README.md  |  4 ++++
 journals/Template.md | 11 +++++++++++
 raw/README.md        | 10 ++++++++++
 releases/README.md   | 12 ++++++++++++
 tmp/readme.md        |  1 +
 7 files changed, 73 insertions(+), 1 deletion(-)
 create mode 100644 codebooks/README.md
 create mode 100644 journals/Template.md
 create mode 100644 raw/README.md
 create mode 100644 releases/README.md
 create mode 100644 tmp/readme.md

diff --git a/.gitignore b/.gitignore
index 0f30474..5dea3f8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,12 @@
+### Custom
+
+#ignore any temporary outputs.
+tmp/ 
+
+
+
+
+### Defaults
 # ---> Python
 # Byte-compiled / optimized / DLL files
 __pycache__/
diff --git a/README.md b/README.md
index baa387b..ada0a75 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,27 @@
-# DataMarshallingTemplate
+# Data Marshalling Template
+
+This is where you will begin "marshalling" the data in the analysis. 
+
+The goal is to take some raw data and begin to turn it into a product that can be ingested in later analyses.
+The directory structure below is designed to help encourage this process.
+
+./raw/              : This will contain your Raw data.
+./releases/         : This will contain your data ouptuts.
+./scripts/          : These are the tools you will use to process the raw data and turn it into the releases.
+./tmp/              : This is where you keep intermediats. It will not be stored in the git repository unless you specifically include it.
+./codebooks/        : These are the documentation you will find and write for your data sources.
+./journals/         : This is where you keep your notebooks to track what you are doing. Just keep a text file open as you are reading documentation, coding, etc, and note things down.
+./references.bib    : Every time you find a potential datasource, write down a citation for it here **OR IN THE REFERENCES MANAGER YOU ARE ALREADY USING**. I recommend Zotero.
+./README.md         : This file is to keep track of what you are doing. You should change it as you go, to make sure people who read this know where to get started.
+
+
+## Getting Started
+
+Information about how to use this.
+
+### Overview
+
+## Disclaimers
+
+
 
diff --git a/codebooks/README.md b/codebooks/README.md
new file mode 100644
index 0000000..ff31d4b
--- /dev/null
+++ b/codebooks/README.md
@@ -0,0 +1,4 @@
+# Output Codebooks
+
+This is where you draft your codebooks for your output.
+Part of your release process should include adding the current version of the codebook to the release folder.
diff --git a/journals/Template.md b/journals/Template.md
new file mode 100644
index 0000000..bb54020
--- /dev/null
+++ b/journals/Template.md
@@ -0,0 +1,11 @@
+# Work Journal - NAME
+
+Instructions:
+
+- Each day, create a H2 heading
+- Use H3 subheadings to track of different things you work on.
+
+## Date 
+
+### topic or branch
+this is where I keep track as I am working on that 
diff --git a/raw/README.md b/raw/README.md
new file mode 100644
index 0000000..a6272b1
--- /dev/null
+++ b/raw/README.md
@@ -0,0 +1,10 @@
+# Instructions
+
+Each data source should go in it's own folder. 
+Each version of the datasource should be within its own folder within that folder, i.e.:
+
+./source1/version1_date/
+./source1/version2_date/
+./source2/version1_date/
+
+Codebooks should be associated with versions.
diff --git a/releases/README.md b/releases/README.md
new file mode 100644
index 0000000..0c62529
--- /dev/null
+++ b/releases/README.md
@@ -0,0 +1,12 @@
+This is where you will track the data you release. 
+
+Here are a couple of thoughts on how to do so.
+
+- have your scripts output to tmp, then bundle it as appropriate
+- Dump your database backup here.
+
+
+Just a couple of other thoughts:
+
+- Copy the codebooks you write over to each release.
+- Try to have each release be a single file, e.g. a zip file.
diff --git a/tmp/readme.md b/tmp/readme.md
new file mode 100644
index 0000000..0c4d9ee
--- /dev/null
+++ b/tmp/readme.md
@@ -0,0 +1 @@
+Ignored in Git