diff --git a/import-automation/executor/app/executor/import_executor.py b/import-automation/executor/app/executor/import_executor.py index 85bb97afaa..d6369c83be 100644 --- a/import-automation/executor/app/executor/import_executor.py +++ b/import-automation/executor/app/executor/import_executor.py @@ -34,6 +34,9 @@ os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) sys.path.append(os.path.join(REPO_DIR, 'tools', 'import_differ')) sys.path.append(os.path.join(REPO_DIR, 'tools', 'import_validation')) +sys.path.append(os.path.join(REPO_DIR, 'util')) + +import file_util from import_differ import ImportDiffer from import_validation import ImportValidation @@ -536,13 +539,32 @@ def _upload_import_inputs( for import_input in import_inputs: for input_type in self.config.import_input_types: path = import_input.get(input_type) - if path: - dest = f'{output_dir}/{version}/{os.path.basename(path)}' - self._upload_file_helper( - src=os.path.join(import_dir, path), - dest=dest, - ) - setattr(uploaded, input_type, dest) + if not path: + continue + for file in file_util.file_get_matching( + os.path.join(import_dir, path)): + if file: + dest = f'{output_dir}/{version}/{os.path.basename(file)}' + self._upload_file_helper( + src=file, + dest=dest, + ) + uploaded_dest = f'{output_dir}/{version}/{os.path.basename(path)}' + setattr(uploaded, input_type, uploaded_dest) + + # Upload any files downloaded from source + source_files = [ + os.path.join(import_dir, file) + for file in import_spec.get('source_files', []) + ] + source_files = file_util.file_get_matching(source_files) + for file in source_files: + dest = f'{output_dir}/{version}/source_files/{os.path.basename(file)}' + self._upload_file_helper( + src=file, + dest=dest, + ) + self.uploader.upload_string( version, os.path.join(output_dir, self.config.storage_version_filename)) @@ -565,7 +587,7 @@ def _import_metadata_mcf_helper(self, import_spec: dict) -> str: Args: import_spec: Specification of the import as a dict. - + Returns: import_metadata_mcf node. """ diff --git a/tools/statvar_importer/config_flags.py b/tools/statvar_importer/config_flags.py index 2619f91d90..39c37ddca6 100644 --- a/tools/statvar_importer/config_flags.py +++ b/tools/statvar_importer/config_flags.py @@ -156,8 +156,6 @@ flags.DEFINE_bool('llm_generate_statvar_name', False, 'Generate names for Statvars.') -_FLAGS(sys.argv) # Allow invocation without app.run() - def get_default_config() -> dict: """Returns the default config as dictionary of config parameters and values.""" diff --git a/util/dc_api_wrapper_test.py b/util/dc_api_wrapper_test.py index 9d95c9151a..2a38d41e41 100644 --- a/util/dc_api_wrapper_test.py +++ b/util/dc_api_wrapper_test.py @@ -82,7 +82,7 @@ def test_dc_api_is_defined_dcid(self): def test_dc_get_node_property_values(self): """Test API wrapper to get all property:values for a node.""" - node_pvs = dc_api.dc_api_get_node_property_values(['dcs:Count_Person']) + node_pvs = dc_api.dc_api_get_node_property_values(['dcid:Count_Person']) self.assertTrue(node_pvs) # Verify the resposnse has dcid with the namespace prefix 'dcid:' self.assertTrue('dcid:Count_Person' in node_pvs)