diff --git a/open_xdmod/modules/xdmod/tests/lib/BaseTest.php b/open_xdmod/modules/xdmod/tests/lib/BaseTest.php new file mode 100644 index 0000000000..0c988382e1 --- /dev/null +++ b/open_xdmod/modules/xdmod/tests/lib/BaseTest.php @@ -0,0 +1,40 @@ + &$value) + { + if ( in_array($key, $keyList) ) { + unset($input->$key); + } elseif ( is_object($value) ) { + $this->filterKeysRecursive($keyList, $value); + } elseif ( is_array($value) ) { + foreach ( $value as $element ) { + if ( is_object($element) ) { + $this->filterKeysRecursive($keyList, $element); + } + } + } + } + return $input; + } +} // BaseTest diff --git a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/ConfigurationTest.php b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/ConfigurationTest.php index 2ffcd69d0c..63e70cc16a 100644 --- a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/ConfigurationTest.php +++ b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/ConfigurationTest.php @@ -9,6 +9,7 @@ namespace UnitTesting\ETL\Configuration; +use CCR\Log; use Configuration\Configuration; class ConfigurationTest extends \PHPUnit_Framework_TestCase @@ -16,6 +17,20 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase const TEST_ARTIFACT_INPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input"; const TEST_ARTIFACT_OUTPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output"; + protected static $logger = null; + + public static function setupBeforeClass() + { + // Set up a logger so we can get warnings and error messages from the ETL infrastructure + $conf = array( + 'file' => false, + 'db' => false, + 'mail' => false, + 'consoleLogLevel' => Log::DEBUG + ); + self::$logger = Log::factory('PHPUnit', $conf); + } + /** * Test JSON parse errors * @@ -84,6 +99,18 @@ public function testRelativePathReference() $this->assertEquals($generated, $expected); } + /** + * Test inclusion of a reference with fully qualified path names. + * + * @expectedException Exception + */ + + public function testBadFragment() + { + $configObj = new Configuration(self::TEST_ARTIFACT_INPUT_PATH . '/rfc6901_bad_fragment.json'); + $configObj->initialize(); + } + /** * Test variables in the configuration file. */ @@ -101,4 +128,39 @@ public function testConfigurationVariables() $expected = json_decode(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/sample_config.expected')); $this->assertEquals($generated, $expected); } + + /** + * Test inclusion of a the following with: + * - A JSON reference with variables in the referenced JSON + * - A JSON-encoded include file with variables in the include path. Note that a comment is + * included in the reference object to ensure comments are removed before transformers are + * processed. + * - A nested JSON reference + */ + + public function testJsonReferenceAndIncludeWithVariables() + { + @copy(self::TEST_ARTIFACT_INPUT_PATH . '/sample_config_with_variables.json', '/tmp/sample_config_with_variables.json'); + @copy(self::TEST_ARTIFACT_INPUT_PATH . '/sample_config_with_reference.json', '/tmp/sample_config_with_reference.json'); + $configObj = new Configuration( + self::TEST_ARTIFACT_INPUT_PATH . '/sample_config_with_transformer_keys.json', + null, + self::$logger, + array( + 'config_variables' => array( + 'TABLE_NAME' => 'resource_allocations', + 'WIDTH' => 40, + 'TMPDIR' => '/tmp', + 'SQLDIR' => 'etl_sql.d', + 'SOURCE_SCHEMA' => 'modw' + ) + ) + ); + $configObj->initialize(); + $generated = json_decode($configObj->toJson()); + $expected = json_decode(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/sample_config_with_transformer_keys.expected')); + @unlink('/tmp/sample_config_with_variables.json'); + @unlink('/tmp/sample_config_with_reference.json'); + $this->assertEquals($generated, $expected, "Test multiple transformer directives"); + } } // class ConfigurationTest diff --git a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php index 8e74cd58c2..d0cae43bc2 100644 --- a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php +++ b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php @@ -16,7 +16,7 @@ use TestHarness\TestFiles; use Xdmod\Config; -class EtlConfigurationTest extends \PHPUnit_Framework_TestCase +class EtlConfigurationTest extends \UnitTesting\BaseTest { const TEST_ARTIFACT_INPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input"; const TEST_ARTIFACT_OUTPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output"; @@ -121,6 +121,7 @@ public function testConfigurationVariables() ); $configObj->initialize(); $generated = json_decode($configObj->toJson()); + $this->filterKeysRecursive(array('key'), $generated); $file = self::TEST_ARTIFACT_OUTPUT_PATH . '/xdmod_etl_config_with_variables.json'; $expected = json_decode(file_get_contents($file)); diff --git a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/Rfc6901Test.php b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/Rfc6901Test.php index bc27cf3954..16285fbcb7 100644 --- a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/Rfc6901Test.php +++ b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/Rfc6901Test.php @@ -107,8 +107,8 @@ public function testRfc6901LastArrayElement() } /** - * Include whole document - */ + * Include whole document + */ public function testRfc6901SpecialCharacter() { @@ -121,4 +121,18 @@ public function testRfc6901SpecialCharacter() $this->assertNull($key); $this->assertEquals($value, 'specialchar'); } + + /** + * Include whole document + * + * @expectedException Exception + */ + + public function testRfc6901BadFragment() + { + $key = '$ref'; + $value = 'rfc6901.json#/does-not-exist'; + $obj = (object) array($key => $value); + $this->transformer->transform($key, $value, $obj, $this->config); + } } // class Rfc6901Test diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_sql.d/query_with_variables.sql b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_sql.d/query_with_variables.sql new file mode 100644 index 0000000000..de7868d915 --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_sql.d/query_with_variables.sql @@ -0,0 +1,5 @@ +SELECT + name, + description +FROM ${SOURCE_SCHEMA}.jobs j +WHERE 1; diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/rfc6901_bad_fragment.json b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/rfc6901_bad_fragment.json new file mode 100644 index 0000000000..85f9b1ad47 --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/rfc6901_bad_fragment.json @@ -0,0 +1,5 @@ +{ + "reference": { + "$ref": "reference_target.json#/does-not-exist" + } +} diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_reference.json b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_reference.json new file mode 100644 index 0000000000..0baf7070ca --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_reference.json @@ -0,0 +1,5 @@ +{ + "subreference": { + "$ref": "file://${TMPDIR}/sample_config_with_variables.json#/key_two" + } +} diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_transformer_keys.json b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_transformer_keys.json new file mode 100644 index 0000000000..a2b6b9fc6f --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_transformer_keys.json @@ -0,0 +1,12 @@ +{ + "reference": { + "$ref": "file://${TMPDIR}/sample_config_with_variables.json#/key_one" + }, + "include": { + "#": "Implicit file reference", + "$include": "${SQLDIR}/query_with_variables.sql" + }, + "reference_with_subreference": { + "$ref": "file://${TMPDIR}/sample_config_with_reference.json" + } +} diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/sample_config_with_transformer_keys.expected b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/sample_config_with_transformer_keys.expected new file mode 100644 index 0000000000..2c9598aa00 --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/sample_config_with_transformer_keys.expected @@ -0,0 +1 @@ +{"reference":{"name":"resource_allocations","engine":"MyISAM","columns":[{"name":"resource","type":"varchar(40)","nullable":true}]},"include":"\"SELECT\\n name,\\n description\\nFROM modw.jobs j\\nWHERE 1;\\n\"","reference_with_subreference":{"subreference":[{"object1":"value"},{"object2":"value"}]}} diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/xdmod_etl_config_with_variables.json b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/xdmod_etl_config_with_variables.json index aa20eea6d1..092870252e 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/xdmod_etl_config_with_variables.json +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/xdmod_etl_config_with_variables.json @@ -1 +1 @@ -{"xdmod.jobs-cloud-common":[{"name":"xdmod.jobs-cloud-common.CloudTableManagement","class":"ManageTables","description":"Manage job tables not managed by other actions","namespace":"ETL\\Maintenance","options_class":"MaintenanceOptions","definition_file_list":["cloud_common\/cloud_resource_metadata.json"],"variables":{"ACTION_CONFIG":"\/tmp\/action_config","LOCAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","GLOBAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","LOCAL_PIPELINE_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","LOCAL_PIPELINE":"\/tmp\/local_pipeline","GLOBAL_GLOBAL":"\/tmp\/global_global","COMMAND_LINE_OVERRIDE_VALUE_2":"\/tmp\/global_global","GLOBAL_GLOBAL_OVERRIDE_LOCAL_GLOBAL":"\/tmp\/local_global","LOCAL_GLOBAL":"\/tmp\/local_global","CLI_OVERRIDE":"CommandLineOverride","CLI_SUBSTITUTE":"VariableInConfig","CLI_NEW":"NewCommandLineVariable","CLOUD_COMMON_DIR":"cloud_common","action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"truncate_destination":false,"enabled":true,"endpoints":{"utility":{"type":"mysql","name":"Utility DB","config":"datawarehouse","schema":"modw","create_schema_if_not_exists":true,"key":"2806a4e2a99daa367aab014ba690b957","paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"source":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"key":"04d8e2279be6616bc524830353c866e4"},"destination":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"key":"04d8e2279be6616bc524830353c866e4"}},"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},{"name":"xdmod.jobs-cloud-common.CloudAssetTypeInitializer","description":"Initialize cloud asset types including unknown","class":"StructuredFileIngestor","definition_file":"cloud_common\/asset_type.json","variables":{"COMMAND_LINE_OVERRIDE_VALUE_2":"CommandLineOverride","COMMAND_LINE_NEW_VALUE":"NewCommandLineVariable","LOCAL_PIPELINE":"\/tmp\/local_pipeline","LOCAL_PIPELINE_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/local_pipeline","GLOBAL_GLOBAL":"\/tmp\/global_global","GLOBAL_GLOBAL_OVERRIDE_LOCAL_GLOBAL":"\/tmp\/local_global","GLOBAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/global_global","LOCAL_GLOBAL":"\/tmp\/local_global","LOCAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/local_global","CLI_OVERRIDE":"CommandLineOverride","CLI_SUBSTITUTE":"VariableInConfig","CLI_NEW":"NewCommandLineVariable","CLOUD_COMMON_DIR":"cloud_common","action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"endpoints":{"source":{"type":"jsonfile","name":"Cloud asset types","path":"cloud_common\/asset_type.json","paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"key":"DataEndpoint11"},"utility":{"type":"mysql","name":"Utility DB","config":"datawarehouse","schema":"modw","create_schema_if_not_exists":true,"key":"2806a4e2a99daa367aab014ba690b957","paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"destination":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"key":"04d8e2279be6616bc524830353c866e4"}},"namespace":"ETL\\Ingestor","options_class":"IngestorOptions","truncate_destination":false,"enabled":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}}]} +{"xdmod.jobs-cloud-common":[{"name":"xdmod.jobs-cloud-common.CloudTableManagement","class":"ManageTables","description":"Manage job tables not managed by other actions","namespace":"ETL\\Maintenance","options_class":"MaintenanceOptions","definition_file_list":["cloud_common\/cloud_resource_metadata.json"],"variables":{"ACTION_CONFIG":"\/tmp\/action_config","LOCAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","GLOBAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","LOCAL_PIPELINE_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","LOCAL_PIPELINE":"\/tmp\/local_pipeline","GLOBAL_GLOBAL":"\/tmp\/global_global","COMMAND_LINE_OVERRIDE_VALUE_2":"\/tmp\/global_global","GLOBAL_GLOBAL_OVERRIDE_LOCAL_GLOBAL":"\/tmp\/local_global","LOCAL_GLOBAL":"\/tmp\/local_global","CLI_OVERRIDE":"CommandLineOverride","CLI_SUBSTITUTE":"VariableInConfig","CLI_NEW":"NewCommandLineVariable","CLOUD_COMMON_DIR":"cloud_common","action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"truncate_destination":false,"enabled":true,"endpoints":{"utility":{"type":"mysql","name":"Utility DB","config":"datawarehouse","schema":"modw","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"source":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"destination":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}}},"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},{"name":"xdmod.jobs-cloud-common.CloudAssetTypeInitializer","description":"Initialize cloud asset types including unknown","class":"StructuredFileIngestor","definition_file":"cloud_common\/asset_type.json","variables":{"COMMAND_LINE_OVERRIDE_VALUE_2":"CommandLineOverride","COMMAND_LINE_NEW_VALUE":"NewCommandLineVariable","LOCAL_PIPELINE":"\/tmp\/local_pipeline","LOCAL_PIPELINE_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/local_pipeline","GLOBAL_GLOBAL":"\/tmp\/global_global","GLOBAL_GLOBAL_OVERRIDE_LOCAL_GLOBAL":"\/tmp\/local_global","GLOBAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/global_global","LOCAL_GLOBAL":"\/tmp\/local_global","LOCAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/local_global","CLI_OVERRIDE":"CommandLineOverride","CLI_SUBSTITUTE":"VariableInConfig","CLI_NEW":"NewCommandLineVariable","CLOUD_COMMON_DIR":"cloud_common","action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"endpoints":{"source":{"type":"jsonfile","name":"Cloud asset types","path":"cloud_common\/asset_type.json","paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"utility":{"type":"mysql","name":"Utility DB","config":"datawarehouse","schema":"modw","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"destination":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}}},"namespace":"ETL\\Ingestor","options_class":"IngestorOptions","truncate_destination":false,"enabled":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}}]}