begin refactor from notebooks to commandline script

harry-gibson · Sep 9, 2020 · 46969d4 · 46969d4
1 parent 9243226
commit 46969d4
Show file tree

Hide file tree

Showing 16 changed files with 1,409 additions and 450 deletions.
diff --git a/DHS_Preprocess_And_Ingestion/03_Unzip_Files/Unzip_Rename_Surveys.ipynb b/DHS_Preprocess_And_Ingestion/03_Unzip_Files/Unzip_Rename_Surveys.ipynb
@@ -192,13 +192,7 @@
       "\\\\map-fs1.ndph.ox.ac.uk\\map_data\\DHS_Automation\\Acquisition\\Update_Staging\\download\\390\\ZAGE71FL.shp\n",
       "ZAGE71FL.shp.xml\n",
       "\\\\map-fs1.ndph.ox.ac.uk\\map_data\\DHS_Automation\\Acquisition\\Update_Staging\\download\\390\\ZAGE71FL.shp.xml\n",
-      "ZAGE71FL.shx\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
+      "ZAGE71FL.shx\n",
       "\\\\map-fs1.ndph.ox.ac.uk\\map_data\\DHS_Automation\\Acquisition\\Update_Staging\\download\\390\\ZAGE71FL.shx\n",
       "ZAMR71.DAT\n",
       "\\\\map-fs1.ndph.ox.ac.uk\\map_data\\DHS_Automation\\Acquisition\\Update_Staging\\download\\390\\ZAMR71.DAT\n",
@@ -383,7 +377,10 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": true
+    "collapsed": true,
+    "jupyter": {
+     "outputs_hidden": true
+    }
    },
    "outputs": [],
    "source": []
@@ -405,9 +402,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.7"
+   "version": "3.7.6"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }
diff --git a/DHS_Preprocess_And_Ingestion/05_Parse_DCF_Metadata/DCF_Parser.ipynb b/DHS_Preprocess_And_Ingestion/05_Parse_DCF_Metadata/DCF_Parser.ipynb
@@ -192,7 +192,10 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": true
+    "collapsed": true,
+    "jupyter": {
+     "outputs_hidden": true
+    }
    },
    "outputs": [],
    "source": [
@@ -231,7 +234,10 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": true
+    "collapsed": true,
+    "jupyter": {
+     "outputs_hidden": true
+    }
    },
    "outputs": [],
    "source": [
@@ -307,9 +313,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.7"
+   "version": "3.7.6"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 1
+ "nbformat_minor": 4
 }
diff --git a/DHS_Preprocess_And_Ingestion/06_Parse_DAT_Data/DHS_DAT_Splitter.ipynb b/DHS_Preprocess_And_Ingestion/06_Parse_DAT_Data/DHS_DAT_Splitter.ipynb
@@ -36,9 +36,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "inSpecDir = r'\\\\map-fs1.ndph.ox.ac.uk\\map_data\\DHS_Automation\\Acquisition\\Update_Staging\\parsed'\n",
-    "inDataDir = r'\\\\map-fs1.ndph.ox.ac.uk\\map_data\\DHS_Automation\\Acquisition\\Update_Staging\\download'\n",
-    "outputCSVTableDir = r'\\\\map-fs1.ndph.ox.ac.uk\\map_data\\DHS_Automation\\Acquisition\\Update_Staging\\tables'"
+    "inSpecDir = r'C:\\Users\\zool1301\\Documents\\02_Update_Staging_Template\\parsed'\n",
+    "inDataDir = r'C:\\Users\\zool1301\\Documents\\02_Update_Staging_Template\\download'\n",
+    "outputCSVTableDir = r'C:\\Users\\zool1301\\Documents\\02_Update_Staging_Template\\tables'"
    ]
   },
   {
@@ -61,15 +61,10 @@
     {
      "data": {
       "text/plain": [
-       "['\\\\\\\\map-fs1.ndph.ox.ac.uk\\\\map_data\\\\DHS_Automation\\\\Acquisition\\\\Update_Staging\\\\download\\\\500\\\\500.JOMR71.DAT',\n",
-       " '\\\\\\\\map-fs1.ndph.ox.ac.uk\\\\map_data\\\\DHS_Automation\\\\Acquisition\\\\Update_Staging\\\\download\\\\500\\\\500.JOIR71.DAT',\n",
-       " '\\\\\\\\map-fs1.ndph.ox.ac.uk\\\\map_data\\\\DHS_Automation\\\\Acquisition\\\\Update_Staging\\\\download\\\\523\\\\523.PKMR71.DAT',\n",
-       " '\\\\\\\\map-fs1.ndph.ox.ac.uk\\\\map_data\\\\DHS_Automation\\\\Acquisition\\\\Update_Staging\\\\download\\\\523\\\\523.PKIR71.DAT',\n",
-       " '\\\\\\\\map-fs1.ndph.ox.ac.uk\\\\map_data\\\\DHS_Automation\\\\Acquisition\\\\Update_Staging\\\\download\\\\494\\\\494.MVIR71.DAT',\n",
-       " '\\\\\\\\map-fs1.ndph.ox.ac.uk\\\\map_data\\\\DHS_Automation\\\\Acquisition\\\\Update_Staging\\\\download\\\\494\\\\494.MVMR71.DAT',\n",
-       " '\\\\\\\\map-fs1.ndph.ox.ac.uk\\\\map_data\\\\DHS_Automation\\\\Acquisition\\\\Update_Staging\\\\download\\\\390\\\\390.ZADV71FL.DAT',\n",
-       " '\\\\\\\\map-fs1.ndph.ox.ac.uk\\\\map_data\\\\DHS_Automation\\\\Acquisition\\\\Update_Staging\\\\download\\\\390\\\\390.ZAIR71.DAT',\n",
-       " '\\\\\\\\map-fs1.ndph.ox.ac.uk\\\\map_data\\\\DHS_Automation\\\\Acquisition\\\\Update_Staging\\\\download\\\\390\\\\390.ZAMR71.DAT']"
+       "['C:\\\\Users\\\\zool1301\\\\Documents\\\\02_Update_Staging_Template\\\\download\\\\517\\\\517.MLIR7A.DAT',\n",
+       " 'C:\\\\Users\\\\zool1301\\\\Documents\\\\02_Update_Staging_Template\\\\download\\\\517\\\\517.MLMR7A.DAT',\n",
+       " 'C:\\\\Users\\\\zool1301\\\\Documents\\\\02_Update_Staging_Template\\\\download\\\\542\\\\542.ZMIR71.DAT',\n",
+       " 'C:\\\\Users\\\\zool1301\\\\Documents\\\\02_Update_Staging_Template\\\\download\\\\542\\\\542.ZMMR71.DAT']"
       ]
      },
      "execution_count": 4,
@@ -111,15 +106,10 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Parsing 500.JOMR71\n",
-      "Parsing 500.JOIR71\n",
-      "Parsing 523.PKMR71\n",
-      "Parsing 523.PKIR71\n",
-      "Parsing 494.MVIR71\n",
-      "Parsing 494.MVMR71\n",
-      "Parsing 390.ZADV71FL\n",
-      "Parsing 390.ZAIR71\n",
-      "Parsing 390.ZAMR71\n"
+      "Parsing 517.MLIR7A\n",
+      "Parsing 517.MLMR7A\n",
+      "Parsing 542.ZMIR71\n",
+      "Parsing 542.ZMMR71\n"
      ]
     }
    ],
@@ -312,7 +302,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.7"
+   "version": "3.7.6"
   }
  },
  "nbformat": 4,