Add and improve tests

open_xdmod/modules/xdmod/tests/lib/BaseTest.php

@@ -0,0 +1,40 @@
+<?php
+/**
+ * Abstract base class to encapsulate funcationality common to unit tests.
+ */
+
+namespace UnitTesting;
+
+abstract class BaseTest extends \PHPUnit_Framework_TestCase
+{
+    /**
+     * Recursively filter out any keys matching one in $keyList. This is a helper function to
+     * address the issue specified in Asana https://app.asana.com/0/807629084565719/1101232922862525
+     * where the key generated for JSON file DataEndpoints is unstable. Note that 2-dimensional
+     * arrays are not handled.
+     *
+     * @param  array $keyList The list of keys to remove
+     * @param  array $input The input object being filtered.
+     *
+     * @return array The filtered object with specified keys removed
+     */
+
+    protected function filterKeysRecursive(array $keyList, \stdClass $input)
+    {
+        foreach ($input as $key => &$value)
+        {
+            if ( in_array($key, $keyList) ) {
+                unset($input->$key);
+            } elseif ( is_object($value) ) {
+                $this->filterKeysRecursive($keyList, $value);
+            } elseif ( is_array($value) ) {
+                foreach ( $value as $element ) {
+                    if ( is_object($element) ) {
+                        $this->filterKeysRecursive($keyList, $element);
+                    }
+                }
+            }
+        }
+        return $input;
+    }
+}  // BaseTest

open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/ConfigurationTest.php

@@ -9,13 +9,28 @@
 
 namespace UnitTesting\ETL\Configuration;
 
+use CCR\Log;
 use Configuration\Configuration;
 
 class ConfigurationTest extends \PHPUnit_Framework_TestCase
 {
     const TEST_ARTIFACT_INPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input";
     const TEST_ARTIFACT_OUTPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output";
 
+    protected static $logger = null;
+
+    public static function setupBeforeClass()
+    {
+      // Set up a logger so we can get warnings and error messages from the ETL infrastructure
+        $conf = array(
+            'file' => false,
+            'db' => false,
+            'mail' => false,
+            'consoleLogLevel' => Log::DEBUG
+        );
+        self::$logger = Log::factory('PHPUnit', $conf);
+    }
+
     /**
      * Test JSON parse errors
      *
@@ -84,6 +99,18 @@ public function testRelativePathReference()
         $this->assertEquals($generated, $expected);
     }
 
+    /**
+     * Test inclusion of a reference with fully qualified path names.
+     *
+     * @expectedException Exception
+     */
+
+    public function testBadFragment()
+    {
+        $configObj = new Configuration(self::TEST_ARTIFACT_INPUT_PATH . '/rfc6901_bad_fragment.json');
+        $configObj->initialize();
+    }
+
     /**
      * Test variables in the configuration file.
      */
@@ -101,4 +128,39 @@ public function testConfigurationVariables()
         $expected = json_decode(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/sample_config.expected'));
         $this->assertEquals($generated, $expected);
     }
+
+    /**
+     * Test inclusion of a the following with:
+     * - A JSON reference with variables in the referenced JSON
+     * - A JSON-encoded include file with variables in the include path. Note that a comment is
+     *   included in the reference object to ensure comments are removed before transformers are
+     *   processed.
+     * - A nested JSON reference
+     */
+
+    public function testJsonReferenceAndIncludeWithVariables()
+    {
+        @copy(self::TEST_ARTIFACT_INPUT_PATH . '/sample_config_with_variables.json', '/tmp/sample_config_with_variables.json');
+        @copy(self::TEST_ARTIFACT_INPUT_PATH . '/sample_config_with_reference.json', '/tmp/sample_config_with_reference.json');
+        $configObj = new Configuration(
+            self::TEST_ARTIFACT_INPUT_PATH . '/sample_config_with_transformer_keys.json',
+            null,
+            self::$logger,
+            array(
+                'config_variables' => array(
+                    'TABLE_NAME' => 'resource_allocations',
+                    'WIDTH' => 40,
+                    'TMPDIR' => '/tmp',
+                    'SQLDIR'  => 'etl_sql.d',
+                    'SOURCE_SCHEMA' => 'modw'
+                )
+            )
+        );
+        $configObj->initialize();
+        $generated = json_decode($configObj->toJson());
+        $expected = json_decode(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/sample_config_with_transformer_keys.expected'));
+        @unlink('/tmp/sample_config_with_variables.json');
+        @unlink('/tmp/sample_config_with_reference.json');
+        $this->assertEquals($generated, $expected, "Test multiple transformer directives");
+    }
 } // class ConfigurationTest

open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php

@@ -16,7 +16,7 @@
 use TestHarness\TestFiles;
 use Xdmod\Config;
 
-class EtlConfigurationTest extends \PHPUnit_Framework_TestCase
+class EtlConfigurationTest extends \UnitTesting\BaseTest
 {
     const TEST_ARTIFACT_INPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input";
     const TEST_ARTIFACT_OUTPUT_PATH = "./artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output";
@@ -121,6 +121,7 @@ public function testConfigurationVariables()
         );
         $configObj->initialize();
         $generated = json_decode($configObj->toJson());
+        $this->filterKeysRecursive(array('key'), $generated);
         $file = self::TEST_ARTIFACT_OUTPUT_PATH . '/xdmod_etl_config_with_variables.json';
         $expected = json_decode(file_get_contents($file));
 

open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/Rfc6901Test.php

@@ -107,8 +107,8 @@ public function testRfc6901LastArrayElement()
     }
 
      /**
-     * Include whole document
-     */
+      * Include whole document
+      */
 
     public function testRfc6901SpecialCharacter()
     {
@@ -121,4 +121,18 @@ public function testRfc6901SpecialCharacter()
         $this->assertNull($key);
         $this->assertEquals($value, 'specialchar');
     }
+
+     /**
+      * Include whole document
+      *
+      * @expectedException Exception
+      */
+
+    public function testRfc6901BadFragment()
+    {
+        $key = '$ref';
+        $value = 'rfc6901.json#/does-not-exist';
+        $obj = (object) array($key => $value);
+        $this->transformer->transform($key, $value, $obj, $this->config);
+    }
 }  // class Rfc6901Test

tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/etl_sql.d/query_with_variables.sql

@@ -0,0 +1,5 @@
+SELECT
+  name,
+  description
+FROM ${SOURCE_SCHEMA}.jobs j
+WHERE 1;

tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/rfc6901_bad_fragment.json

@@ -0,0 +1,5 @@
+{
+    "reference": {
+        "$ref": "reference_target.json#/does-not-exist"
+    }
+}

tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_bad_reference_fragment.json

@@ -0,0 +1,5 @@
+{
+    "subreference": {
+        "$ref": "file://${TMPDIR}/sample_config_with_variables.json#/key_two"
+    }
+}

tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_reference.json

@@ -0,0 +1,5 @@
+{
+    "subreference": {
+        "$ref": "file://${TMPDIR}/sample_config_with_variables.json#/key_two"
+    }
+}

tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/input/sample_config_with_transformer_keys.json

@@ -0,0 +1,12 @@
+{
+    "reference": {
+        "$ref": "file://${TMPDIR}/sample_config_with_variables.json#/key_one"
+    },
+    "include": {
+        "#": "Implicit file reference",
+        "$include": "${SQLDIR}/query_with_variables.sql"
+    },
+    "reference_with_subreference": {
+        "$ref": "file://${TMPDIR}/sample_config_with_reference.json"
+    }
+}

tests/artifacts/xdmod-test-artifacts/xdmod/etlv2/configuration/output/sample_config_with_transformer_keys.expected

@@ -0,0 +1 @@
+{"reference":{"name":"resource_allocations","engine":"MyISAM","columns":[{"name":"resource","type":"varchar(40)","nullable":true}]},"include":"\"SELECT\\n  name,\\n  description\\nFROM modw.jobs j\\nWHERE 1;\\n\"","reference_with_subreference":{"subreference":[{"object1":"value"},{"object2":"value"}]}}