From 9f786e9f0aafe6c0c34b4c4fd9de1a1f96fd6ab3 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Wed, 7 Nov 2018 11:11:05 -0500 Subject: [PATCH 1/3] Fix wording in error message --- classes/ETL/EtlOverseerOptions.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/classes/ETL/EtlOverseerOptions.php b/classes/ETL/EtlOverseerOptions.php index e433105c02..791bf96a0e 100644 --- a/classes/ETL/EtlOverseerOptions.php +++ b/classes/ETL/EtlOverseerOptions.php @@ -789,8 +789,7 @@ public function mapResourceCodesToIds(array $codes) { foreach ( $codes as $code ) { if ( false === ($resourceId = $this->getResourceIdFromCode($code)) ) { - $msg = "Unknown include resource code: '$code'"; - $this->logAndThrowException($msg); + $this->logAndThrowException(sprintf("Unknown resource code: '%s'", $code)); } else { $resourceIdList[] = $resourceId; } From 054829e9e50f5f0a22bee04ffb82f9ec96de36db Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Wed, 7 Nov 2018 11:11:38 -0500 Subject: [PATCH 2/3] Ensure resource codes are properly verified --- classes/ETL/EtlOverseer.php | 42 +++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/classes/ETL/EtlOverseer.php b/classes/ETL/EtlOverseer.php index f0905cd327..1c82cbd4e6 100644 --- a/classes/ETL/EtlOverseer.php +++ b/classes/ETL/EtlOverseer.php @@ -176,6 +176,14 @@ public function verifyActions( if ( isset($action->getOptions()->include_only_resource_codes) || isset($action->getOptions()->exclude_resource_codes) ) { $this->queryResourceCodeToIdMap($etlConfig); + + // Verify that specified resource codes are valid + + $codeList = array_merge( + $action->getOptions()->include_only_resource_codes, + $action->getOptions()->exclude_resource_codes + ); + $this->verifyResourceCodes($codeList); } $action->initialize($this->etlOverseerOptions); @@ -273,6 +281,32 @@ protected function queryResourceCodeToIdMap(EtlConfiguration $etlConfig) $this->etlOverseerOptions->setResourceCodeToIdMap($map); } + /** + * Verify that the specified resource codes are present in the resource code to resource id map. + * + * @param array $codeList A list of resource codes to verify. + * + * @return Nothing. + * @throws Exception if at least one resource code is not found in the map. + */ + + protected function verifyResourceCodes($codeList) + { + $missing = array(); + $map = $this->etlOverseerOptions->getResourceCodeToIdMap(); + foreach ( $codeList as $code ) { + if ( ! array_key_exists($code, $map) ) { + $missing[] = $code; + } + } + + if ( count($missing) > 0 ) { + $this->logAndThrowException( + sprintf("Unknown resource code(s) specified: '%s'", implode("','", $missing)) + ); + } + } + /* ------------------------------------------------------------------------------------------ * @see iEtlOverseer::execute() * ------------------------------------------------------------------------------------------ @@ -290,6 +324,14 @@ public function execute(EtlConfiguration $etlConfig) || count($this->etlOverseerOptions->getExcludeResourceCodes()) > 0 ) { $this->queryResourceCodeToIdMap($etlConfig); + + // Verify that specified resource codes are valid + + $codeList = array_merge( + $this->etlOverseerOptions->getIncludeOnlyResourceCodes(), + $this->etlOverseerOptions->getExcludeResourceCodes() + ); + $this->verifyResourceCodes($codeList); } // Pre-pend the default module name to any section names that are not already qualified. From 5ed516611e154f8a8bacdfa94b1dd81d179da185 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Wed, 7 Nov 2018 11:12:06 -0500 Subject: [PATCH 3/3] Add tests for valid/invalid resource codes --- classes/ETL/aAction.php | 4 +- .../lib/ETL/EtlOverseerTest.php | 180 ++++++++++++++++++ .../etl_action_defs_8.0.0.d/dummy_action.json | 7 + .../input/etl_tables_8.0.0.d/dummy_table.json | 19 ++ .../input/xdmod_etl_config_dummy_actions.json | 60 ++++++ 5 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 open_xdmod/modules/xdmod/component_tests/lib/ETL/EtlOverseerTest.php create mode 100644 tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_action_defs_8.0.0.d/dummy_action.json create mode 100644 tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/dummy_table.json create mode 100644 tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/xdmod_etl_config_dummy_actions.json diff --git a/classes/ETL/aAction.php b/classes/ETL/aAction.php index 302bac10c4..cc108c5ea7 100644 --- a/classes/ETL/aAction.php +++ b/classes/ETL/aAction.php @@ -399,7 +399,9 @@ private function initializeVariableStore() && 0 != count($value) ) { $resourceCode = current($value); - $resourceId = $this->etlOverseerOptions->getResourceIdFromCode($resourceCode); + if ( false === ($resourceId = $this->etlOverseerOptions->getResourceIdFromCode($resourceCode)) ) { + $this->logAndThrowException(sprintf("Unknown resource code: '%s'", $resourceCode)); + } if ( count($value) > 1 ) { $this->logger->info( sprintf( diff --git a/open_xdmod/modules/xdmod/component_tests/lib/ETL/EtlOverseerTest.php b/open_xdmod/modules/xdmod/component_tests/lib/ETL/EtlOverseerTest.php new file mode 100644 index 0000000000..1747b064b3 --- /dev/null +++ b/open_xdmod/modules/xdmod/component_tests/lib/ETL/EtlOverseerTest.php @@ -0,0 +1,180 @@ + 'xdmod') + ); + self::$etlConfig->initialize(); + + // Explicitly set the resource code map so we don't need to query the database + self::$overseerOptions = new EtlOverseerOptions( + array( + 'default-module-name' => 'xdmod', + 'process-sections' => array('dummy-actions'), + 'resource-code-map' => array( + 'resource1' => 1, + 'resource2' => 2 + ) + ) + ); + + } + + /** + * Reset values in shared classes. + */ + + public function setUp() + { + self::$overseerOptions->setIncludeOnlyResourceCodes(null); + self::$overseerOptions->setIncludeOnlyResourceCodes(null); + } + + + /** + * Test various cases of valid include and exclude resource codes + */ + + public function testValidResourceCodes() { + + // Single valid resource codes to include + + try { + self::$overseerOptions->setIncludeOnlyResourceCodes('resource1'); + $overseer = new EtlOverseer(self::$overseerOptions); + $overseer->execute(self::$etlConfig); + } catch ( Exception $e ) { + $this->assertTrue(false, $e->getMessage()); + } + + // Array of valid resource codes to include + + try { + self::$overseerOptions->setIncludeOnlyResourceCodes(array('resource1', 'resource2')); + $overseer = new EtlOverseer(self::$overseerOptions); + $overseer->execute(self::$etlConfig); + } catch ( Exception $e ) { + $this->assertTrue(false, $e->getMessage()); + } + + // Single valid resource code to exclude + + try { + self::$overseerOptions->setExcludeResourceCodes('resource1'); + $overseer = new EtlOverseer(self::$overseerOptions); + $overseer->execute(self::$etlConfig); + } catch ( Exception $e ) { + $this->assertTrue(false, $e->getMessage()); + } + + // Array of valid resource codes to exclude + + try { + self::$overseerOptions->setExcludeResourceCodes(array('resource1', 'resource2')); + $overseer = new EtlOverseer(self::$overseerOptions); + $overseer->execute(self::$etlConfig); + } catch ( Exception $e ) { + $this->assertTrue(false, $e->getMessage()); + } + } + + /** + * Test various cases of invali include and exclude resource codes + */ + + public function testInvalidResourceCodes() { + + // Single invalid resource code to include + + $unknownCode = 'unknown101'; + $exceptionThrown = true; + try { + self::$overseerOptions->setIncludeOnlyResourceCodes($unknownCode); + $overseer = new EtlOverseer(self::$overseerOptions); + $overseer->execute(self::$etlConfig); + $exceptionThrown = false; + } catch ( Exception $e ) { + $this->assertContains($unknownCode, $e->getMessage(), "Unknown resource code but did not find expected code '$unknownCode'"); + } + $this->assertTrue($exceptionThrown, "Expected exception to be thrown for unknown resource code '$unknownCode'"); + + + // Array with one invalid resource code to include + + $unknownCode = 'unknown102'; + $exceptionThrown = true; + try { + self::$overseerOptions->setIncludeOnlyResourceCodes(array('resource1',$unknownCode)); + $overseer = new EtlOverseer(self::$overseerOptions); + $overseer->execute(self::$etlConfig); + $exceptionThrown = false; + } catch ( Exception $e ) { + $this->assertContains($unknownCode, $e->getMessage(), "Unknown resource code but did not find expected code '$unknownCode'"); + } + $this->assertTrue($exceptionThrown, "Expected exception to be thrown for unknown resource code '$unknownCode'"); + + // Single invalid resource code to exclude + + $unknownCode = 'unknown101'; + $exceptionThrown = true; + try { + self::$overseerOptions->setExcludeResourceCodes($unknownCode); + $overseer = new EtlOverseer(self::$overseerOptions); + $overseer->execute(self::$etlConfig); + $exceptionThrown = false; + } catch ( Exception $e ) { + $this->assertContains($unknownCode, $e->getMessage(), "Unknown resource code but did not find expected code '$unknownCode'"); + } + $this->assertTrue($exceptionThrown, "Expected exception to be thrown for unknown resource code '$unknownCode'"); + + // Array with one invalid resource code to exclude + + $unknownCode = 'unknown102'; + $exceptionThrown = true; + try { + self::$overseerOptions->setExcludeResourceCodes(array('resource1',$unknownCode)); + $overseer = new EtlOverseer(self::$overseerOptions); + $overseer->execute(self::$etlConfig); + $exceptionThrown = false; + } catch ( Exception $e ) { + $this->assertContains($unknownCode, $e->getMessage(), "Unknown resource code but did not find expected code '$unknownCode'"); + } + $this->assertTrue($exceptionThrown, "Expected exception to be thrown for unknown resource code '$unknownCode'"); + } +} diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_action_defs_8.0.0.d/dummy_action.json b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_action_defs_8.0.0.d/dummy_action.json new file mode 100644 index 0000000000..e06bfb88b0 --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_action_defs_8.0.0.d/dummy_action.json @@ -0,0 +1,7 @@ +{ + "table_definition": [ + { + "$ref": "${table_definition_dir}/dummy_table.json#/table_definition" + } + ] +} diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/dummy_table.json b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/dummy_table.json new file mode 100644 index 0000000000..0068d20726 --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/dummy_table.json @@ -0,0 +1,19 @@ +{ + "table_definition": { + "name": "dummy_table", + "engine": "MyISAM", + "comment": "Use with dummy actions", + "columns": [ + { + "name": "my_int", + "type": "int", + "nullable": false + }, + { + "name": "my_varchar8", + "type": "varchar(8)", + "nullable": false + } + ] + } +} diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/xdmod_etl_config_dummy_actions.json b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/xdmod_etl_config_dummy_actions.json new file mode 100644 index 0000000000..33c18b8499 --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/xdmod_etl_config_dummy_actions.json @@ -0,0 +1,60 @@ +{ + "#": "Paths for the various configuraiton and data subdirectories. This is added to the global", + "#": "defaults and actions.", + + "paths": { + "table_definition_dir": "etl_tables_8.0.0.d", + "action_definition_dir": "etl_action_defs_8.0.0.d", + "specs_dir": "etl_specs.d", + "macro_dir": "etl_macros.d", + "sql_dir": "etl_sql.d", + "data_dir": "etl_data.d" + }, + + "defaults": { + + "#": "Global options are lowest priority and applied to all actions", + + "global" : { + "#": "The utility endpoint is used by the etl_overseer script to query resource codes.", + "endpoints": { + "utility": { + "type": "mysql", + "name": "Utility DB", + "config": "datawarehouse", + "schema": "modw" + }, + "source": { + "type": "mysql", + "name": "Utility DB", + "config": "datawarehouse", + "schema": "modw" + }, + "destination": { + "type": "mysql", + "name": "Utility DB", + "config": "datawarehouse", + "schema": "modw" + } + } + }, + + "#": "Options specific to the 'dummy-actions' pipeline", + + "dummy-actions": { + "namespace": "ETL\\Ingestor", + "options_class": "IngestorOptions" + } + + }, + + "dummy-actions": [ + { + "name": "dummy-ingestor", + "description": "Dummy ingestor for testing general ETL machinery", + "class": "DummyIngestor", + "definition_file": "dummy_action.json" + } + ] + +}