[u p] Explicitly configure source type in environment files (partial #…

…6355)
DataBiosphere · Aug 30, 2024 · 5948841 · 5948841
1 parent 098f814
commit 5948841
Show file tree

Hide file tree

Showing 9 changed files with 3,463 additions and 2,469 deletions.
diff --git a/UPGRADING.rst b/UPGRADING.rst
@@ -20,6 +20,15 @@ reverted. This is all fairly informal and loosely defined. Hopefully we won't
 have too many entries in this file.
 
 
+#6355 Explicitly configure source type in environment files
+===========================================================
+
+The ``mksrc`` function has been updated to include a parameter specifying the
+type of each source (either "parquet" or "bigquery"). For each of your personal
+deployments, specify "bigquery" for all sources. As always, use the sandbox
+deployment's ``environment.py`` as a model when upgrading personal deployments.
+
+
 #6446 Base image of Azul image is not pinned to digest
 ======================================================
 

diff --git a/deployments/anvilbox/environment.py b/deployments/anvilbox/environment.py
@@ -3,6 +3,7 @@
 )
 import json
 from typing import (
+    Literal,
     Optional,
 )
 
@@ -27,7 +28,8 @@ def common_prefix(n: int) -> str:
 pop = 2  # remove snapshot
 
 
-def mksrc(google_project,
+def mksrc(source_type: Literal['bigquery', 'parquet'],
+          google_project,
           snapshot,
           subgraphs,
           flags: int = 0,
@@ -40,7 +42,7 @@ def mksrc(google_project,
         prefix = common_prefix(subgraphs)
     source = None if flags & pop else ':'.join([
         'tdr',
-        'bigquery',
+        source_type,
         'gcp',
         google_project,
         snapshot,
@@ -71,9 +73,9 @@ def mkdict(previous_catalog: dict[str, str],
 
 
 anvil_sources = mkdict({}, 3, mkdelta([
-    mksrc('datarepo-dev-e53e74aa', 'ANVIL_1000G_2019_Dev_20230609_ANV5_202306121732', 6804),
-    mksrc('datarepo-dev-42c70e6a', 'ANVIL_CCDG_Sample_1_20230228_ANV5_202302281520', 28),
-    mksrc('datarepo-dev-97ad270b', 'ANVIL_CMG_Sample_1_20230225_ANV5_202302281509', 25)
+    mksrc('bigquery', 'datarepo-dev-e53e74aa', 'ANVIL_1000G_2019_Dev_20230609_ANV5_202306121732', 6804),
+    mksrc('bigquery', 'datarepo-dev-42c70e6a', 'ANVIL_CCDG_Sample_1_20230228_ANV5_202302281520', 28),
+    mksrc('bigquery', 'datarepo-dev-97ad270b', 'ANVIL_CMG_Sample_1_20230225_ANV5_202302281509', 25)
 ]))
 
 

diff --git a/deployments/anvildev/environment.py b/deployments/anvildev/environment.py
@@ -3,6 +3,7 @@
 )
 import json
 from typing import (
+    Literal,
     Optional,
 )
 
@@ -22,7 +23,8 @@ def partition_prefix_length(n: int) -> int:
 pop = 2  # remove snapshot
 
 
-def mksrc(google_project,
+def mksrc(source_type: Literal['bigquery', 'parquet'],
+          google_project,
           snapshot,
           subgraphs,
           flags: int = 0
@@ -31,7 +33,7 @@ def mksrc(google_project,
     assert flags <= ma | pop
     source = None if flags & pop else ':'.join([
         'tdr',
-        'bigquery',
+        source_type,
         'gcp',
         google_project,
         snapshot,
@@ -62,9 +64,9 @@ def mkdict(previous_catalog: dict[str, str],
 
 
 anvil_sources = mkdict({}, 3, mkdelta([
-    mksrc('datarepo-dev-e53e74aa', 'ANVIL_1000G_2019_Dev_20230609_ANV5_202306121732', 6804),
-    mksrc('datarepo-dev-42c70e6a', 'ANVIL_CCDG_Sample_1_20230228_ANV5_202302281520', 28),
-    mksrc('datarepo-dev-97ad270b', 'ANVIL_CMG_Sample_1_20230225_ANV5_202302281509', 25)
+    mksrc('bigquery', 'datarepo-dev-e53e74aa', 'ANVIL_1000G_2019_Dev_20230609_ANV5_202306121732', 6804),
+    mksrc('bigquery', 'datarepo-dev-42c70e6a', 'ANVIL_CCDG_Sample_1_20230228_ANV5_202302281520', 28),
+    mksrc('bigquery', 'datarepo-dev-97ad270b', 'ANVIL_CMG_Sample_1_20230225_ANV5_202302281509', 25)
 ]))
 
 

diff --git a/deployments/anvilprod/environment.py b/deployments/anvilprod/environment.py
diff --git a/deployments/dev/environment.py b/deployments/dev/environment.py
diff --git a/deployments/hammerbox/environment.py b/deployments/hammerbox/environment.py
diff --git a/deployments/prod/environment.py b/deployments/prod/environment.py
diff --git a/deployments/sandbox/environment.py b/deployments/sandbox/environment.py
diff --git a/deployments/tempdev/environment.py b/deployments/tempdev/environment.py
@@ -3,6 +3,7 @@
 )
 import json
 from typing import (
+    Literal,
     Optional,
 )
 
@@ -22,7 +23,8 @@ def partition_prefix_length(n: int) -> int:
 pop = 2  # remove snapshot
 
 
-def mksrc(google_project,
+def mksrc(source_type: Literal['bigquery', 'parquet'],
+          google_project,
           snapshot,
           subgraphs,
           flags: int = 0
@@ -31,7 +33,7 @@ def mksrc(google_project,
     assert flags <= ma | pop
     source = None if flags & pop else ':'.join([
         'tdr',
-        'bigquery',
+        source_type,
         'gcp',
         google_project,
         snapshot,
@@ -62,9 +64,9 @@ def mkdict(previous_catalog: dict[str, str],
 
 
 anvil_sources = mkdict({}, 3, mkdelta([
-    mksrc('datarepo-dev-e53e74aa', 'ANVIL_1000G_2019_Dev_20230609_ANV5_202306121732', 6804),
-    mksrc('datarepo-dev-42c70e6a', 'ANVIL_CCDG_Sample_1_20230228_ANV5_202302281520', 28),
-    mksrc('datarepo-dev-97ad270b', 'ANVIL_CMG_Sample_1_20230225_ANV5_202302281509', 25)
+    mksrc('bigquery', 'datarepo-dev-e53e74aa', 'ANVIL_1000G_2019_Dev_20230609_ANV5_202306121732', 6804),
+    mksrc('bigquery', 'datarepo-dev-42c70e6a', 'ANVIL_CCDG_Sample_1_20230228_ANV5_202302281520', 28),
+    mksrc('bigquery', 'datarepo-dev-97ad270b', 'ANVIL_CMG_Sample_1_20230225_ANV5_202302281509', 25)
 ]))