fixup! Parquet to BigQuery import for GCP-backed AnVIL snapshots (#6355)

DataBiosphere · Aug 1, 2024 · c4ae3c6 · c4ae3c6
1 parent dbd46f4
commit c4ae3c6
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/scripts/reindex.py b/scripts/reindex.py
@@ -32,6 +32,10 @@
 from azul.plugins.repository.tdr import (
     TDRPlugin,
 )
+from azul.terra import (
+    SourceType,
+    TDRSourceSpec,
+)
 
 log = logging.getLogger(__name__)
 
@@ -173,8 +177,10 @@ def main(argv: list[str]):
                 plugin = azul.repository_plugin(catalog)
                 assert isinstance(plugin, tdr_anvil.Plugin)
                 for source in sources:
-                    source = plugin.resolve_source(source)
-                    plugin.import_tables(source)
+                    spec = TDRSourceSpec.parse(source)
+                    if spec.type == SourceType.parquet:
+                        source = plugin.resolve_source(source)
+                        plugin.import_tables(source)
             else:
                 log.info('Skipping table import for catalog %r', catalog)
 

diff --git a/src/azul/plugins/repository/tdr_anvil/__init__.py b/src/azul/plugins/repository/tdr_anvil/__init__.py
@@ -751,7 +751,7 @@ def import_tables(self, source: TDRSourceRef):
         export API. Only tables defined in the AnVIL schema will be imported.
         Currently, only GS-backed snapshots are supported.
         """
-        require(source.spec.project == config.google_project(), source)
+        require(source.spec.subdomain == config.google_project(), source)
 
         dataset_name = source.spec.name
         self.tdr.create_dataset(dataset_name)