diff --git a/classes/Configuration/CommentTransformer.php b/classes/Configuration/CommentTransformer.php index 02c4c57824..0f1fad4278 100644 --- a/classes/Configuration/CommentTransformer.php +++ b/classes/Configuration/CommentTransformer.php @@ -16,7 +16,7 @@ class CommentTransformer extends Loggable implements iConfigFileKeyTransformer { - const COMMENT_CHAR = '#'; + const COMMENT_PREFIX = '#'; /* ------------------------------------------------------------------------------------------ * @see iConfigFileKeyTransformer::__construct() @@ -35,7 +35,7 @@ public function __construct(Log $logger = null) public function keyMatches($key) { - return ( 0 === strpos($key, self::COMMENT_CHAR) ); + return ( 0 === strpos($key, self::COMMENT_PREFIX) ); } // keyMatches() /* ------------------------------------------------------------------------------------------ @@ -48,7 +48,7 @@ public function keyMatches($key) public function transform(&$key, &$value, stdClass $obj, Configuration $config) { - $this->logger->trace("Remove comment '$key'"); + $this->logger->trace(sprintf("Remove comment '%s'", $key)); $key = null; $value = null; diff --git a/classes/Configuration/Configuration.php b/classes/Configuration/Configuration.php index a43ef34ade..199571bbd5 100644 --- a/classes/Configuration/Configuration.php +++ b/classes/Configuration/Configuration.php @@ -391,6 +391,7 @@ protected function preTransformTasks() $this->addKeyTransformer(new CommentTransformer($this->logger)); $this->addKeyTransformer(new JsonReferenceTransformer($this->logger)); $this->addKeyTransformer(new StripMergePrefixTransformer($this->logger)); + $this->addKeyTransformer(new IncludeTransformer($this->logger)); return $this; } //preTransformTasks() @@ -650,7 +651,11 @@ protected function processKeyTransformers(stdClass $obj) continue; } - $stop = ( ! $transformer->transform($transformKey, $value, $obj, $this) ); + try { + $stop = ( ! $transformer->transform($transformKey, $value, $obj, $this) ); + } catch ( Exception $e ) { + throw new Exception(sprintf("%s: %s", $this->filename, $e->getMessage())); + } if ( null === $transformKey && null === $value ) { @@ -707,6 +712,14 @@ protected function processKeyTransformers(stdClass $obj) } // foreach ( $value as $element ) } + // If we have replaced the object by something that is not Traversable (such as an + // included string) then do not continue the loop or the foreach will try to call + // valid() and next() on a non-Traversable. + + if ( ! ( is_array($obj) || is_object($obj) || ($obj instanceof \Traversable) ) ) { + break; + } + } // foreach ( $obj as $key => $value ) return $obj; diff --git a/classes/Configuration/IncludeTransformer.php b/classes/Configuration/IncludeTransformer.php new file mode 100644 index 0000000000..8f43c1e623 --- /dev/null +++ b/classes/Configuration/IncludeTransformer.php @@ -0,0 +1,49 @@ +logAndThrowException( + sprintf('References cannot be mixed with other keys in an object: "%s": "%s"', $key, $value) + ); + } + + $parsedUrl = null; + $contents = $this->getContentsFromUrl($value, $config); + $key = null; + $value = json_encode($contents); + + return false; + + } // transform() +} // class IncludeTransformer diff --git a/classes/Configuration/JsonReferenceTransformer.php b/classes/Configuration/JsonReferenceTransformer.php index 2808992fce..78a81fd024 100644 --- a/classes/Configuration/JsonReferenceTransformer.php +++ b/classes/Configuration/JsonReferenceTransformer.php @@ -1,5 +1,5 @@ qualifyPath($parsedUrl['path'], $config); - $this->logger->debug( - sprintf("(%s) Resolve JSON reference '%s' to file '%s'", get_class($this), $value, $path) - ); - - $fragment = ( array_key_exists('fragment', $parsedUrl) ? $parsedUrl['fragment'] : '' ); - - // If no scheme was provided, default to the file scheme. Also ensure that the - // file path is properly formatted. - - $scheme = ( array_key_exists('scheme', $parsedUrl) ? $parsedUrl['scheme'] : 'file' ); - if ( 'file' == $scheme ) { - $path = 'file://' . $path; - } - - // Open the file and return the contents. - - $contents = @file_get_contents($path); - if ( false === $contents ) { - $this->logAndThrowException('Failed to open file: ' . $path); - } - + $parsedUrl = null; + $contents = $this->getContentsFromUrl($value, $config); + $fragment = ( array_key_exists('fragment', $this->parsedUrl) ? $this->parsedUrl['fragment'] : '' ); $key = null; - JsonPointer::setLoggable($this); - $value = JsonPointer::extractFragment($contents, $fragment); - JsonPointer::setLoggable(null); + try { + $value = JsonPointer::extractFragment($contents, $fragment); + } catch ( \Exception $e ) { + // Re-throw the exception with additional file information + throw new \Exception(sprintf("%s in file %s", $e->getMessage(), $this->parsedUrl['path'])); + } if ( false === $value ) { $this->logAndThrowException( @@ -133,24 +105,4 @@ public function transform(&$key, &$value, stdClass $obj, Configuration $config) return true; } // transform() - - /* ------------------------------------------------------------------------------------------ - * Qualify the path using the base directory from the configuration object if it is - * not already fully qualified. - * - * @param string $path The path to qualify - * @param Configuration $config $The configuration object that called the transformer - * - * @returns A fully qualified path - * ------------------------------------------------------------------------------------------ - */ - - protected function qualifyPath($path, Configuration $config) - { - $path = $config->getVariableStore()->substitute( - $path, - "Undefined macros in JSON reference" - ); - return \xd_utilities\qualify_path($path, $config->getBaseDir()); - } } // class JsonReferenceTransformer diff --git a/classes/Configuration/aUrlTransformer.php b/classes/Configuration/aUrlTransformer.php new file mode 100644 index 0000000000..9c1df9785c --- /dev/null +++ b/classes/Configuration/aUrlTransformer.php @@ -0,0 +1,114 @@ +getVariableStore()->substitute( + $path, + "Undefined macros in file reference" + ); + return \xd_utilities\qualify_path($path, $config->getBaseDir()); + } + + /** ----------------------------------------------------------------------------------------- + * For transformers that support a value that is a URL perform the following steps: + * - Validate the URL + * - Qualify the path to resolve any configuration variables that are used + * - Extract the contents of the file + * + * Note: This method specifically supports file URLs. + * + * @param string $url The URL to process + * @param Configuration $config The Configuration object that called this method + * + * @return The contents of the file referenced by the URL + * @throws Exception if there was an error parsing the URL or accessing the file + * ------------------------------------------------------------------------------------------ + */ + + public function getContentsFromUrl($url, Configuration $config) + { + + $url = $config->getVariableStore()->substitute( + $url, + "Undefined macros in URL reference" + ); + + $this->parsedUrl = parse_url($url); + // We need to process variables on the url BEFORE parsing the URL... + + // Ensure the value contains a file path + + if ( empty($this->parsedUrl['path']) ) { + $this->logAndThrowException( + sprintf("(%s) Unable to extract path from URL: %s", get_class($this), $url) + ); + } + + $path = \xd_utilities\qualify_path($this->parsedUrl['path'], $config->getBaseDir()); + + // $path = $this->qualifyPath($this->parsedUrl['path'], $config); + $this->logger->debug( + sprintf("(%s) Resolved reference '%s' to '%s'", get_class($this), $this->parsedUrl['path'], $path) + ); + + // If no scheme was provided, default to the file scheme. + + $scheme = ( array_key_exists('scheme', $this->parsedUrl) ? $this->parsedUrl['scheme'] : 'file' ); + if ( 'file' == $scheme ) { + $path = 'file://' . $path; + } + + // Open the file and return the contents. + + $contents = @file_get_contents($path); + if ( false === $contents ) { + $error = error_get_last(); + $this->logAndThrowException( + sprintf("Failed to open file '%s'%s", $path, (null !== $error ? ": " . $error['message'] : "")) + ); + } + + return $contents; + } +} // class aUrlTransformer diff --git a/open_xdmod/modules/xdmod/tests/lib/BaseTest.php b/open_xdmod/modules/xdmod/tests/lib/BaseTest.php new file mode 100644 index 0000000000..0c988382e1 --- /dev/null +++ b/open_xdmod/modules/xdmod/tests/lib/BaseTest.php @@ -0,0 +1,40 @@ + &$value) + { + if ( in_array($key, $keyList) ) { + unset($input->$key); + } elseif ( is_object($value) ) { + $this->filterKeysRecursive($keyList, $value); + } elseif ( is_array($value) ) { + foreach ( $value as $element ) { + if ( is_object($element) ) { + $this->filterKeysRecursive($keyList, $element); + } + } + } + } + return $input; + } +} // BaseTest diff --git a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/ConfigurationTest.php b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/ConfigurationTest.php index 2ffcd69d0c..63e70cc16a 100644 --- a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/ConfigurationTest.php +++ b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/ConfigurationTest.php @@ -9,6 +9,7 @@ namespace UnitTesting\ETL\Configuration; +use CCR\Log; use Configuration\Configuration; class ConfigurationTest extends \PHPUnit_Framework_TestCase @@ -16,6 +17,20 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase const TEST_ARTIFACT_INPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input"; const TEST_ARTIFACT_OUTPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output"; + protected static $logger = null; + + public static function setupBeforeClass() + { + // Set up a logger so we can get warnings and error messages from the ETL infrastructure + $conf = array( + 'file' => false, + 'db' => false, + 'mail' => false, + 'consoleLogLevel' => Log::DEBUG + ); + self::$logger = Log::factory('PHPUnit', $conf); + } + /** * Test JSON parse errors * @@ -84,6 +99,18 @@ public function testRelativePathReference() $this->assertEquals($generated, $expected); } + /** + * Test inclusion of a reference with fully qualified path names. + * + * @expectedException Exception + */ + + public function testBadFragment() + { + $configObj = new Configuration(self::TEST_ARTIFACT_INPUT_PATH . '/rfc6901_bad_fragment.json'); + $configObj->initialize(); + } + /** * Test variables in the configuration file. */ @@ -101,4 +128,39 @@ public function testConfigurationVariables() $expected = json_decode(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/sample_config.expected')); $this->assertEquals($generated, $expected); } + + /** + * Test inclusion of a the following with: + * - A JSON reference with variables in the referenced JSON + * - A JSON-encoded include file with variables in the include path. Note that a comment is + * included in the reference object to ensure comments are removed before transformers are + * processed. + * - A nested JSON reference + */ + + public function testJsonReferenceAndIncludeWithVariables() + { + @copy(self::TEST_ARTIFACT_INPUT_PATH . '/sample_config_with_variables.json', '/tmp/sample_config_with_variables.json'); + @copy(self::TEST_ARTIFACT_INPUT_PATH . '/sample_config_with_reference.json', '/tmp/sample_config_with_reference.json'); + $configObj = new Configuration( + self::TEST_ARTIFACT_INPUT_PATH . '/sample_config_with_transformer_keys.json', + null, + self::$logger, + array( + 'config_variables' => array( + 'TABLE_NAME' => 'resource_allocations', + 'WIDTH' => 40, + 'TMPDIR' => '/tmp', + 'SQLDIR' => 'etl_sql.d', + 'SOURCE_SCHEMA' => 'modw' + ) + ) + ); + $configObj->initialize(); + $generated = json_decode($configObj->toJson()); + $expected = json_decode(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/sample_config_with_transformer_keys.expected')); + @unlink('/tmp/sample_config_with_variables.json'); + @unlink('/tmp/sample_config_with_reference.json'); + $this->assertEquals($generated, $expected, "Test multiple transformer directives"); + } } // class ConfigurationTest diff --git a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php index 8e74cd58c2..d0cae43bc2 100644 --- a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php +++ b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php @@ -16,7 +16,7 @@ use TestHarness\TestFiles; use Xdmod\Config; -class EtlConfigurationTest extends \PHPUnit_Framework_TestCase +class EtlConfigurationTest extends \UnitTesting\BaseTest { const TEST_ARTIFACT_INPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input"; const TEST_ARTIFACT_OUTPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output"; @@ -121,6 +121,7 @@ public function testConfigurationVariables() ); $configObj->initialize(); $generated = json_decode($configObj->toJson()); + $this->filterKeysRecursive(array('key'), $generated); $file = self::TEST_ARTIFACT_OUTPUT_PATH . '/xdmod_etl_config_with_variables.json'; $expected = json_decode(file_get_contents($file)); diff --git a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/IncludeTest.php b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/IncludeTest.php new file mode 100644 index 0000000000..0e4ff67890 --- /dev/null +++ b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/IncludeTest.php @@ -0,0 +1,103 @@ + + * @date 2017-04-21 + * ------------------------------------------------------------------------------------------ + */ + +namespace UnitTesting\ETL\Configuration; + +use CCR\Log; +use Configuration\Configuration; +use Configuration\IncludeTransformer; + +class IncludeTest extends \PHPUnit_Framework_TestCase +{ + const TEST_ARTIFACT_INPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input"; + const TEST_ARTIFACT_OUTPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output"; + + protected static $transformer = null; + protected static $config = null; + + public static function setupBeforeClass() + { + // Set up a logger so we can get warnings and error messages from the ETL infrastructure + $conf = array( + 'file' => false, + 'db' => false, + 'mail' => false, + 'consoleLogLevel' => Log::WARNING + ); + $logger = Log::factory('PHPUnit', $conf); + + // Configuration is used in the transformer to qualify relative paths + self::$config = new Configuration(self::TEST_ARTIFACT_INPUT_PATH . '/sample_config.json'); + self::$transformer = new IncludeTransformer($logger); + } + + /** + * Test invalid file + * + * @expectedException Exception + */ + + public function testIncludeInvalidFile() + { + $key = '$include'; + $value = 'file_does_not_exist.txt'; + $obj = (object) array($key => $value); + self::$transformer->transform($key, $value, $obj, self::$config); + } + + /** + * Badly formed URL + * + * @expectedException Exception + */ + + public function testBadUrl() + { + $key = '$include'; + $value = 'badscheme://string'; + $obj = (object) array($key => $value); + self::$transformer->transform($key, $value, $obj, self::$config); + } + + /** + * Include a document (e.g., SQL query) + */ + + public function testIncludeFile() + { + $key = '$include'; + $value = 'etl_sql.d/query.sql'; + $obj = (object) array($key => $value); + $expected = json_encode(file_get_contents(self::TEST_ARTIFACT_INPUT_PATH . '/' . $value)); + self::$transformer->transform($key, $value, $obj, self::$config); + + // A null key means replace the entire value object with the transformed value + $this->assertNull($key); + $this->assertEquals($expected, $value, "JSON-encoded value"); + } + + /** + * Test variables in the include URL. + */ + + public function testIncludeFileWithVariable() + { + self::$config->getVariableStore()->FILENAME = 'query'; + self::$config->getVariableStore()->SUBDIR = 'etl_sql.d'; + + $key = '$include'; + $value = '${SUBDIR}/${FILENAME}.sql'; + $obj = (object) array($key => $value); + self::$transformer->transform($key, $value, $obj, self::$config); + + $expected = json_encode(file_get_contents(self::TEST_ARTIFACT_INPUT_PATH . '/etl_sql.d/query.sql')); + $this->assertNull($key); + $this->assertEquals($expected, $value, "JSON-encoded value"); + } +} // class IncludeTest diff --git a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/Rfc6901Test.php b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/Rfc6901Test.php index bc27cf3954..16285fbcb7 100644 --- a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/Rfc6901Test.php +++ b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/Rfc6901Test.php @@ -107,8 +107,8 @@ public function testRfc6901LastArrayElement() } /** - * Include whole document - */ + * Include whole document + */ public function testRfc6901SpecialCharacter() { @@ -121,4 +121,18 @@ public function testRfc6901SpecialCharacter() $this->assertNull($key); $this->assertEquals($value, 'specialchar'); } + + /** + * Include whole document + * + * @expectedException Exception + */ + + public function testRfc6901BadFragment() + { + $key = '$ref'; + $value = 'rfc6901.json#/does-not-exist'; + $obj = (object) array($key => $value); + $this->transformer->transform($key, $value, $obj, $this->config); + } } // class Rfc6901Test diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_sql.d/query.sql b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_sql.d/query.sql new file mode 100644 index 0000000000..1b20811d1f --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_sql.d/query.sql @@ -0,0 +1,5 @@ +SELECT + name, + description +FROM jobs j +WHERE 1; diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_sql.d/query_with_variables.sql b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_sql.d/query_with_variables.sql new file mode 100644 index 0000000000..de7868d915 --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_sql.d/query_with_variables.sql @@ -0,0 +1,5 @@ +SELECT + name, + description +FROM ${SOURCE_SCHEMA}.jobs j +WHERE 1; diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/rfc6901_bad_fragment.json b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/rfc6901_bad_fragment.json new file mode 100644 index 0000000000..85f9b1ad47 --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/rfc6901_bad_fragment.json @@ -0,0 +1,5 @@ +{ + "reference": { + "$ref": "reference_target.json#/does-not-exist" + } +} diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_reference.json b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_reference.json new file mode 100644 index 0000000000..0baf7070ca --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_reference.json @@ -0,0 +1,5 @@ +{ + "subreference": { + "$ref": "file://${TMPDIR}/sample_config_with_variables.json#/key_two" + } +} diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_transformer_keys.json b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_transformer_keys.json new file mode 100644 index 0000000000..a2b6b9fc6f --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_transformer_keys.json @@ -0,0 +1,12 @@ +{ + "reference": { + "$ref": "file://${TMPDIR}/sample_config_with_variables.json#/key_one" + }, + "include": { + "#": "Implicit file reference", + "$include": "${SQLDIR}/query_with_variables.sql" + }, + "reference_with_subreference": { + "$ref": "file://${TMPDIR}/sample_config_with_reference.json" + } +} diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/sample_config_with_transformer_keys.expected b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/sample_config_with_transformer_keys.expected new file mode 100644 index 0000000000..2c9598aa00 --- /dev/null +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/sample_config_with_transformer_keys.expected @@ -0,0 +1 @@ +{"reference":{"name":"resource_allocations","engine":"MyISAM","columns":[{"name":"resource","type":"varchar(40)","nullable":true}]},"include":"\"SELECT\\n name,\\n description\\nFROM modw.jobs j\\nWHERE 1;\\n\"","reference_with_subreference":{"subreference":[{"object1":"value"},{"object2":"value"}]}} diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/xdmod_etl_config_with_variables.json b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/xdmod_etl_config_with_variables.json index aa20eea6d1..092870252e 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/xdmod_etl_config_with_variables.json +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/xdmod_etl_config_with_variables.json @@ -1 +1 @@ -{"xdmod.jobs-cloud-common":[{"name":"xdmod.jobs-cloud-common.CloudTableManagement","class":"ManageTables","description":"Manage job tables not managed by other actions","namespace":"ETL\\Maintenance","options_class":"MaintenanceOptions","definition_file_list":["cloud_common\/cloud_resource_metadata.json"],"variables":{"ACTION_CONFIG":"\/tmp\/action_config","LOCAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","GLOBAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","LOCAL_PIPELINE_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","LOCAL_PIPELINE":"\/tmp\/local_pipeline","GLOBAL_GLOBAL":"\/tmp\/global_global","COMMAND_LINE_OVERRIDE_VALUE_2":"\/tmp\/global_global","GLOBAL_GLOBAL_OVERRIDE_LOCAL_GLOBAL":"\/tmp\/local_global","LOCAL_GLOBAL":"\/tmp\/local_global","CLI_OVERRIDE":"CommandLineOverride","CLI_SUBSTITUTE":"VariableInConfig","CLI_NEW":"NewCommandLineVariable","CLOUD_COMMON_DIR":"cloud_common","action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"truncate_destination":false,"enabled":true,"endpoints":{"utility":{"type":"mysql","name":"Utility DB","config":"datawarehouse","schema":"modw","create_schema_if_not_exists":true,"key":"2806a4e2a99daa367aab014ba690b957","paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"source":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"key":"04d8e2279be6616bc524830353c866e4"},"destination":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"key":"04d8e2279be6616bc524830353c866e4"}},"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},{"name":"xdmod.jobs-cloud-common.CloudAssetTypeInitializer","description":"Initialize cloud asset types including unknown","class":"StructuredFileIngestor","definition_file":"cloud_common\/asset_type.json","variables":{"COMMAND_LINE_OVERRIDE_VALUE_2":"CommandLineOverride","COMMAND_LINE_NEW_VALUE":"NewCommandLineVariable","LOCAL_PIPELINE":"\/tmp\/local_pipeline","LOCAL_PIPELINE_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/local_pipeline","GLOBAL_GLOBAL":"\/tmp\/global_global","GLOBAL_GLOBAL_OVERRIDE_LOCAL_GLOBAL":"\/tmp\/local_global","GLOBAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/global_global","LOCAL_GLOBAL":"\/tmp\/local_global","LOCAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/local_global","CLI_OVERRIDE":"CommandLineOverride","CLI_SUBSTITUTE":"VariableInConfig","CLI_NEW":"NewCommandLineVariable","CLOUD_COMMON_DIR":"cloud_common","action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"endpoints":{"source":{"type":"jsonfile","name":"Cloud asset types","path":"cloud_common\/asset_type.json","paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"key":"DataEndpoint11"},"utility":{"type":"mysql","name":"Utility DB","config":"datawarehouse","schema":"modw","create_schema_if_not_exists":true,"key":"2806a4e2a99daa367aab014ba690b957","paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"destination":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"key":"04d8e2279be6616bc524830353c866e4"}},"namespace":"ETL\\Ingestor","options_class":"IngestorOptions","truncate_destination":false,"enabled":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}}]} +{"xdmod.jobs-cloud-common":[{"name":"xdmod.jobs-cloud-common.CloudTableManagement","class":"ManageTables","description":"Manage job tables not managed by other actions","namespace":"ETL\\Maintenance","options_class":"MaintenanceOptions","definition_file_list":["cloud_common\/cloud_resource_metadata.json"],"variables":{"ACTION_CONFIG":"\/tmp\/action_config","LOCAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","GLOBAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","LOCAL_PIPELINE_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/action_config","LOCAL_PIPELINE":"\/tmp\/local_pipeline","GLOBAL_GLOBAL":"\/tmp\/global_global","COMMAND_LINE_OVERRIDE_VALUE_2":"\/tmp\/global_global","GLOBAL_GLOBAL_OVERRIDE_LOCAL_GLOBAL":"\/tmp\/local_global","LOCAL_GLOBAL":"\/tmp\/local_global","CLI_OVERRIDE":"CommandLineOverride","CLI_SUBSTITUTE":"VariableInConfig","CLI_NEW":"NewCommandLineVariable","CLOUD_COMMON_DIR":"cloud_common","action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"truncate_destination":false,"enabled":true,"endpoints":{"utility":{"type":"mysql","name":"Utility DB","config":"datawarehouse","schema":"modw","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"source":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"destination":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}}},"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},{"name":"xdmod.jobs-cloud-common.CloudAssetTypeInitializer","description":"Initialize cloud asset types including unknown","class":"StructuredFileIngestor","definition_file":"cloud_common\/asset_type.json","variables":{"COMMAND_LINE_OVERRIDE_VALUE_2":"CommandLineOverride","COMMAND_LINE_NEW_VALUE":"NewCommandLineVariable","LOCAL_PIPELINE":"\/tmp\/local_pipeline","LOCAL_PIPELINE_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/local_pipeline","GLOBAL_GLOBAL":"\/tmp\/global_global","GLOBAL_GLOBAL_OVERRIDE_LOCAL_GLOBAL":"\/tmp\/local_global","GLOBAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/global_global","LOCAL_GLOBAL":"\/tmp\/local_global","LOCAL_GLOBAL_OVERRIDE_ACTION_CONFIG_1":"\/tmp\/local_global","CLI_OVERRIDE":"CommandLineOverride","CLI_SUBSTITUTE":"VariableInConfig","CLI_NEW":"NewCommandLineVariable","CLOUD_COMMON_DIR":"cloud_common","action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"},"endpoints":{"source":{"type":"jsonfile","name":"Cloud asset types","path":"cloud_common\/asset_type.json","paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"utility":{"type":"mysql","name":"Utility DB","config":"datawarehouse","schema":"modw","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}},"destination":{"type":"mysql","name":"Cloud DB","config":"datawarehouse","schema":"modw_cloud","create_schema_if_not_exists":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}}},"namespace":"ETL\\Ingestor","options_class":"IngestorOptions","truncate_destination":false,"enabled":true,"paths":{"action_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_action_defs_8.0.0.d","table_definition_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_tables_8.0.0.d","specs_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_specs.d","schema_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_schemas.d","macro_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_macros_override.d","sql_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_sql_override.d","data_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_data.d","local_config_dir":"\/tmp\/xdmod-etl-configuration-test\/etl_8.0.0.d","base_dir":"\/tmp\/xdmod-etl-configuration-test","new_path":"\/tmp\/xdmod-etl-configuration-test\/my_new_path"}}]}