Support ETL '$include' directive

classes/Configuration/CommentTransformer.php

@@ -16,7 +16,7 @@
 
 class CommentTransformer extends Loggable implements iConfigFileKeyTransformer
 {
-    const COMMENT_CHAR = '#';
+    const COMMENT_PREFIX = '#';
 
     /* ------------------------------------------------------------------------------------------
      * @see iConfigFileKeyTransformer::__construct()
@@ -35,7 +35,7 @@ public function __construct(Log $logger = null)
 
     public function keyMatches($key)
     {
-        return ( 0 === strpos($key, self::COMMENT_CHAR) );
+        return ( 0 === strpos($key, self::COMMENT_PREFIX) );
     }  // keyMatches()
 
     /* ------------------------------------------------------------------------------------------
@@ -48,7 +48,7 @@ public function keyMatches($key)
 
     public function transform(&$key, &$value, stdClass $obj, Configuration $config)
     {
-        $this->logger->trace("Remove comment '$key'");
+        $this->logger->trace(sprintf("Remove comment '%s'", $key));
         $key = null;
         $value = null;
 

classes/Configuration/Configuration.php

@@ -391,6 +391,7 @@ protected function preTransformTasks()
         $this->addKeyTransformer(new CommentTransformer($this->logger));
         $this->addKeyTransformer(new JsonReferenceTransformer($this->logger));
         $this->addKeyTransformer(new StripMergePrefixTransformer($this->logger));
+        $this->addKeyTransformer(new IncludeTransformer($this->logger));
         return $this;
     }  //preTransformTasks()
 
@@ -650,7 +651,11 @@ protected function processKeyTransformers(stdClass $obj)
                     continue;
                 }
 
-                $stop = ( ! $transformer->transform($transformKey, $value, $obj, $this) );
+                try {
+                    $stop = ( ! $transformer->transform($transformKey, $value, $obj, $this) );
+                } catch ( Exception $e ) {
+                    throw new Exception(sprintf("%s: %s", $this->filename, $e->getMessage()));
+                }
 
                 if ( null === $transformKey && null === $value ) {
 
@@ -707,6 +712,14 @@ protected function processKeyTransformers(stdClass $obj)
                 }  // foreach ( $value as $element )
             }
 
+            // If we have replaced the object by something that is not Traversable (such as an
+            // included string) then do not continue the loop or the foreach will try to call
+            // valid() and next() on a non-Traversable.
+
+            if ( ! ( is_array($obj) || is_object($obj) || ($obj instanceof \Traversable) ) ) {
+                break;
+            }
+
         }  // foreach ( $obj as $key => $value )
 
         return $obj;

classes/Configuration/IncludeTransformer.php

@@ -0,0 +1,49 @@
+<?php
+/** =========================================================================================
+ * Process the "$include" directive to include the contents of a file as a JSON encoded string.
+ * ==========================================================================================
+ */
+
+namespace Configuration;
+
+use stdClass;
+
+class IncludeTransformer extends aUrlTransformer implements iConfigFileKeyTransformer
+{
+    const REFERENCE_KEY = '$include';
+
+    /** -----------------------------------------------------------------------------------------
+     * @see iConfigFileKeyTransformer::keyMatches()
+     * ------------------------------------------------------------------------------------------
+     */
+
+    public function keyMatches($key)
+    {
+        return (self::REFERENCE_KEY == $key);
+    }  // keyMatches()
+
+    /** -----------------------------------------------------------------------------------------
+     * Include the JSON-endoced contents of the file as the value of the specified key.
+     *
+     * @see iConfigFileKeyTransformer::transform()
+     * ------------------------------------------------------------------------------------------
+     */
+
+    public function transform(&$key, &$value, stdClass $obj, Configuration $config)
+    {
+
+        if( count(get_object_vars($obj)) != 1 ) {
+            $this->logAndThrowException(
+                sprintf('References cannot be mixed with other keys in an object: "%s": "%s"', $key, $value)
+            );
+        }
+
+        $parsedUrl = null;
+        $contents = $this->getContentsFromUrl($value, $config);
+        $key = null;
+        $value = json_encode($contents);
+
+        return false;
+
+    }  // transform()
+}  // class IncludeTransformer

classes/Configuration/JsonReferenceTransformer.php

@@ -1,5 +1,5 @@
 <?php
-/* ==========================================================================================
+/** =========================================================================================
  * Evaluate JSON references (https://tools.ietf.org/html/draft-pbryan-zyp-json-ref-03) where
  * the `$ref` gets logically replaced with the thing that it points to. For example,
  * { "$ref": "http://example.com/example.json#/foo/bar" }
@@ -17,21 +17,11 @@
 use CCR\Loggable;
 use ETL\JsonPointer;
 
-class JsonReferenceTransformer extends Loggable implements iConfigFileKeyTransformer
+class JsonReferenceTransformer extends aUrlTransformer implements iConfigFileKeyTransformer
 {
     const REFERENCE_KEY = '$ref';
 
-    /* ------------------------------------------------------------------------------------------
-     * @see iConfigFileKeyTransformer::__construct()
-     * ------------------------------------------------------------------------------------------
-     */
-
-    public function __construct(Log $logger = null)
-    {
-        parent::__construct($logger);
-    }  // construct()
-
-    /* ------------------------------------------------------------------------------------------
+    /** -----------------------------------------------------------------------------------------
      * @see iConfigFileKeyTransformer::keyMatches()
      * ------------------------------------------------------------------------------------------
      */
@@ -41,9 +31,8 @@ public function keyMatches($key)
         return (self::REFERENCE_KEY == $key);
     }  // keyMatches()
 
-    /* ------------------------------------------------------------------------------------------
-     * Comments remove both the key and the value from the configuration and stop processing of the
-     * key.
+    /** -----------------------------------------------------------------------------------------
+     * Transform the JSON pointer into the actual JSON that it references.
      *
      * @see iConfigFileKeyTransformer::transform()
      * ------------------------------------------------------------------------------------------
@@ -94,34 +83,17 @@ public function transform(&$key, &$value, stdClass $obj, Configuration $config)
             );
         }
 
-        $parsedUrl = parse_url($value);
-        $path = $this->qualifyPath($parsedUrl['path'], $config);
-        $this->logger->debug(
-            sprintf("(%s) Resolve JSON reference '%s' to file '%s'", get_class($this), $value, $path)
-        );
-
-        $fragment = ( array_key_exists('fragment', $parsedUrl) ? $parsedUrl['fragment'] : '' );
-
-        // If no scheme was provided, default to the file scheme. Also ensure that the
-        // file path is properly formatted.
-
-        $scheme = ( array_key_exists('scheme', $parsedUrl) ? $parsedUrl['scheme'] : 'file' );
-        if ( 'file' == $scheme ) {
-            $path = 'file://' . $path;
-        }
-
-        // Open the file and return the contents.
-
-        $contents = @file_get_contents($path);
-        if ( false === $contents ) {
-            $this->logAndThrowException('Failed to open file: ' . $path);
-        }
-
+        $parsedUrl = null;
+        $contents = $this->getContentsFromUrl($value, $config);
+        $fragment = ( array_key_exists('fragment', $this->parsedUrl) ? $this->parsedUrl['fragment'] : '' );
         $key = null;
 
-        JsonPointer::setLoggable($this);
-        $value = JsonPointer::extractFragment($contents, $fragment);
-        JsonPointer::setLoggable(null);
+        try {
+            $value = JsonPointer::extractFragment($contents, $fragment);
+        } catch ( \Exception $e ) {
+            // Re-throw the exception with additional file information
+            throw new \Exception(sprintf("%s in file %s", $e->getMessage(), $this->parsedUrl['path']));
+        }
 
         if ( false === $value ) {
             $this->logAndThrowException(
@@ -133,24 +105,4 @@ public function transform(&$key, &$value, stdClass $obj, Configuration $config)
         return true;
 
     }  // transform()
-
-    /* ------------------------------------------------------------------------------------------
-     * Qualify the path using the base directory from the configuration object if it is
-     * not already fully qualified.
-     *
-     * @param string $path The path to qualify
-     * @param Configuration $config $The configuration object that called the transformer
-     *
-     * @returns A fully qualified path
-     * ------------------------------------------------------------------------------------------
-     */
-
-    protected function qualifyPath($path, Configuration $config)
-    {
-        $path = $config->getVariableStore()->substitute(
-            $path,
-            "Undefined macros in JSON reference"
-        );
-        return \xd_utilities\qualify_path($path, $config->getBaseDir());
-    }
 }  // class JsonReferenceTransformer

classes/Configuration/aUrlTransformer.php

@@ -0,0 +1,114 @@
+<?php
+/**
+ * Abstract base class to encapsulate functionality common to transformers that operate on URLs,
+ * such as a JSON pointer or file include.
+ *
+ */
+
+namespace Configuration;
+
+use Log;
+use CCR\Loggable;
+
+abstract class aUrlTransformer extends Loggable
+{
+    /**
+     * @var array $parsedUrl
+     *
+     * The results of the url parsed by parse_url()
+     */
+
+    protected $parsedUrl = null;
+
+    /** -----------------------------------------------------------------------------------------
+     * @see iConfigFileKeyTransformer::__construct()
+     * ------------------------------------------------------------------------------------------
+     */
+
+    public function __construct(Log $logger = null)
+    {
+        parent::__construct($logger);
+    }
+
+    /** -----------------------------------------------------------------------------------------
+     * Qualify the path using the base directory from the configuration object if it is
+     * not already fully qualified.
+     *
+     * @param string $path The path to qualify
+     * @param Configuration $config $The configuration object that called the transformer
+     *
+     * @return A fully qualified path
+     * ------------------------------------------------------------------------------------------
+     */
+
+    protected function qualifyPath($path, Configuration $config)
+    {
+        $path = $config->getVariableStore()->substitute(
+            $path,
+            "Undefined macros in file reference"
+        );
+        return \xd_utilities\qualify_path($path, $config->getBaseDir());
+    }
+
+    /** -----------------------------------------------------------------------------------------
+     * For transformers that support a value that is a URL perform the following steps:
+     * - Validate the URL
+     * - Qualify the path to resolve any configuration variables that are used
+     * - Extract the contents of the file
+     *
+     * Note: This method specifically supports file URLs.
+     *
+     * @param string $url The URL to process
+     * @param Configuration $config The Configuration object that called this method
+     *
+     * @return The contents of the file referenced by the URL
+     * @throws Exception if there was an error parsing the URL or accessing the file
+     * ------------------------------------------------------------------------------------------
+     */
+
+    public function getContentsFromUrl($url, Configuration $config)
+    {
+
+        $url = $config->getVariableStore()->substitute(
+            $url,
+            "Undefined macros in URL reference"
+        );
+
+        $this->parsedUrl = parse_url($url);
+        // We need to process variables on the url BEFORE parsing the URL...
+
+        // Ensure the value contains a file path
+
+        if ( empty($this->parsedUrl['path']) ) {
+            $this->logAndThrowException(
+                sprintf("(%s) Unable to extract path from URL: %s", get_class($this), $url)
+            );
+        }
+
+        $path = \xd_utilities\qualify_path($this->parsedUrl['path'], $config->getBaseDir());
+
+        // $path = $this->qualifyPath($this->parsedUrl['path'], $config);
+        $this->logger->debug(
+            sprintf("(%s) Resolved reference '%s' to '%s'", get_class($this), $this->parsedUrl['path'], $path)
+        );
+
+        // If no scheme was provided, default to the file scheme.
+
+        $scheme = ( array_key_exists('scheme', $this->parsedUrl) ? $this->parsedUrl['scheme'] : 'file' );
+        if ( 'file' == $scheme ) {
+            $path = 'file://' . $path;
+        }
+
+        // Open the file and return the contents.
+
+        $contents = @file_get_contents($path);
+        if ( false === $contents ) {
+            $error = error_get_last();
+            $this->logAndThrowException(
+                sprintf("Failed to open file '%s'%s", $path, (null !== $error ? ": " . $error['message'] : ""))
+            );
+        }
+
+        return $contents;
+    }
+} // class aUrlTransformer

open_xdmod/modules/xdmod/tests/lib/BaseTest.php

@@ -0,0 +1,40 @@
+<?php
+/**
+ * Abstract base class to encapsulate funcationality common to unit tests.
+ */
+
+namespace UnitTesting;
+
+abstract class BaseTest extends \PHPUnit_Framework_TestCase
+{
+    /**
+     * Recursively filter out any keys matching one in $keyList. This is a helper function to
+     * address the issue specified in Asana https://app.asana.com/0/807629084565719/1101232922862525
+     * where the key generated for JSON file DataEndpoints is unstable. Note that 2-dimensional
+     * arrays are not handled.
+     *
+     * @param  array $keyList The list of keys to remove
+     * @param  array $input The input object being filtered.
+     *
+     * @return array The filtered object with specified keys removed
+     */
+
+    protected function filterKeysRecursive(array $keyList, \stdClass $input)
+    {
+        foreach ($input as $key => &$value)
+        {
+            if ( in_array($key, $keyList) ) {
+                unset($input->$key);
+            } elseif ( is_object($value) ) {
+                $this->filterKeysRecursive($keyList, $value);
+            } elseif ( is_array($value) ) {
+                foreach ( $value as $element ) {
+                    if ( is_object($element) ) {
+                        $this->filterKeysRecursive($keyList, $element);
+                    }
+                }
+            }
+        }
+        return $input;
+    }
+}  // BaseTest