Add support for wildcard import files in manifest.json (datacommonsor…

…g#1198)

* Add support for wildcard import files in manifest.json

* copy source_files to GCS

* fix unit tests

* fix unit tests

import-automation/executor/app/executor/import_executor.py

@@ -34,6 +34,9 @@
             os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
 sys.path.append(os.path.join(REPO_DIR, 'tools', 'import_differ'))
 sys.path.append(os.path.join(REPO_DIR, 'tools', 'import_validation'))
+sys.path.append(os.path.join(REPO_DIR, 'util'))
+
+import file_util
 
 from import_differ import ImportDiffer
 from import_validation import ImportValidation
@@ -536,13 +539,32 @@ def _upload_import_inputs(
         for import_input in import_inputs:
             for input_type in self.config.import_input_types:
                 path = import_input.get(input_type)
-                if path:
-                    dest = f'{output_dir}/{version}/{os.path.basename(path)}'
-                    self._upload_file_helper(
-                        src=os.path.join(import_dir, path),
-                        dest=dest,
-                    )
-                    setattr(uploaded, input_type, dest)
+                if not path:
+                    continue
+                for file in file_util.file_get_matching(
+                        os.path.join(import_dir, path)):
+                    if file:
+                        dest = f'{output_dir}/{version}/{os.path.basename(file)}'
+                        self._upload_file_helper(
+                            src=file,
+                            dest=dest,
+                        )
+                uploaded_dest = f'{output_dir}/{version}/{os.path.basename(path)}'
+                setattr(uploaded, input_type, uploaded_dest)
+
+        # Upload any files downloaded from source
+        source_files = [
+            os.path.join(import_dir, file)
+            for file in import_spec.get('source_files', [])
+        ]
+        source_files = file_util.file_get_matching(source_files)
+        for file in source_files:
+            dest = f'{output_dir}/{version}/source_files/{os.path.basename(file)}'
+            self._upload_file_helper(
+                src=file,
+                dest=dest,
+            )
+
         self.uploader.upload_string(
             version,
             os.path.join(output_dir, self.config.storage_version_filename))
@@ -565,7 +587,7 @@ def _import_metadata_mcf_helper(self, import_spec: dict) -> str:
 
         Args:
             import_spec: Specification of the import as a dict.
-        
+
         Returns:
             import_metadata_mcf node.
         """

tools/statvar_importer/config_flags.py

@@ -156,8 +156,6 @@
 flags.DEFINE_bool('llm_generate_statvar_name', False,
                   'Generate names for Statvars.')
 
-_FLAGS(sys.argv)  # Allow invocation without app.run()
-
 
 def get_default_config() -> dict:
     """Returns the default config as dictionary of config parameters and values."""

util/dc_api_wrapper_test.py

@@ -82,7 +82,7 @@ def test_dc_api_is_defined_dcid(self):
 
     def test_dc_get_node_property_values(self):
         """Test API wrapper to get all property:values for a node."""
-        node_pvs = dc_api.dc_api_get_node_property_values(['dcs:Count_Person'])
+        node_pvs = dc_api.dc_api_get_node_property_values(['dcid:Count_Person'])
         self.assertTrue(node_pvs)
         # Verify the resposnse has dcid with the namespace prefix 'dcid:'
         self.assertTrue('dcid:Count_Person' in node_pvs)