Merge branch 'xdmod8.1' into fix_jobviewer_deeplink

ubccr · Feb 14, 2019 · e1b22b4 · e1b22b4
2 parents 8a74ab4 + f25c870
commit e1b22b4
Show file tree

Hide file tree

Showing 260 changed files with 2,231 additions and 1,359 deletions.
diff --git a/bin/xdmod-ingestor b/bin/xdmod-ingestor
@@ -269,6 +269,10 @@ function main()
                 if($datatypeValue == 'genericcloud'){
                     $dwi->ingestCloudDataGeneric();
                 }
+
+                if ($datatypeValue == 'storage') {
+                    $dwi->ingestStorageData();
+                }
             }
         } catch (Exception $e) {
             $logger->crit(array(
@@ -291,6 +295,10 @@ function main()
             if($realmToAggregate == 'cloud' || $realmToAggregate === false){
                 $dwi->aggregateCloudData();
             }
+
+            if ($realmToAggregate == 'storage' || $realmToAggregate === false) {
+                $dwi->aggregateStorageData();
+            }
         } catch (Exception $e) {
             $logger->crit(array(
                 'message'    => 'Aggregation failed: ' . $e->getMessage(),

diff --git a/bin/xdmod-shredder b/bin/xdmod-shredder
@@ -234,15 +234,15 @@ function main()
     }
 
     if (!$dryRun) {
-        $logger->notice('Normalizing data!');
-
         try {
             $ingestor = $shredder->getJobIngestor();
-            // The cloud shredders do not have jobs to ingest and return false when
-            // getJobInestor is called for them so we don't have to hard code skippping 
-            // those formats here.
-            if($ingestor !== false){
+            // The cloud and storage shredders do not have jobs to ingest and
+            // return false when getJobInestor is called for them so we don't
+            // have to hard code skipping those formats here.
+            if ($ingestor !== false) {
+                $logger->notice('Normalizing data');
                 $ingestor->ingest();
+                $logger->notice('Done normalizing data');
             }
         } catch (Exception $e) {
             $logger->crit(array(
@@ -251,8 +251,6 @@ function main()
             ));
             exit(1);
         }
-
-        $logger->notice('Done normalizing data');
     }
 
     // NOTE: "process_end_time" is needed for the log summary.
@@ -291,10 +289,11 @@ Usage: xdmod-shredder [-v] -r resource -f format [-i file|-d dir]
     -f, --format *format*
         Specify the log file format ("pbs" for PBS/TORQUE, "sge" for Sun
         Grid Engine, "uge" for Univa Grid Engine 8.2+, "slurm" for
-        Slurm or "lsf" for LSF).
+        Slurm or "lsf" for LSF, "storage" for storage data).
 
     -i, --input *file*
-        Specify a single log file to shred.
+        Specify a single log file to shred.  Not applicable to cloud and
+        storage data.
 
     -d, --dir *directory*
         Specify a directory containing log files to shred.  Log files in

diff --git a/classes/Authentication/SAML/XDSamlAuthentication.php b/classes/Authentication/SAML/XDSamlAuthentication.php
@@ -18,18 +18,18 @@ class XDSamlAuthentication
     protected $_as = null;
 
     /**
-     * Enumerated potential auth sources
+     * The selected auth source name (used for logout)
      *
-     * @var array
+     * @var string
      */
-    protected $_sources = null;
 
+    protected $authSourceName = null;
     /**
-     * Whether or not SAML is configured. Defaults to false.
+     * Enumerated potential auth sources
      *
-     * @var boolean
+     * @var array
      */
-    protected $_isConfigured = false;
+    protected $_sources = null;
 
     const BASE_ADMIN_EMAIL = <<<EML
 
@@ -70,8 +70,10 @@ public function __construct()
                 $authSource = null;
             }
             if (!is_null($authSource) && array_search($authSource, $this->_sources) !== false) {
+                $this->authSourceName = $authSource;
                 $this->_as = new \SimpleSAML\Auth\Simple($authSource);
             } else {
+                $this->authSourceName = $this->_sources[0];
                 $this->_as = new \SimpleSAML\Auth\Simple($this->_sources[0]);
             }
         }
@@ -84,10 +86,17 @@ public function __construct()
      */
     public function isSamlConfigured()
     {
-        $this->_isConfigured = count($this->_sources) > 0 ? true : false;
-        return $this->_isConfigured;
+        return !empty($this->_sources);
     }
 
+    /**
+     * Logs out of the saml session
+     */
+    public function logout(){
+        if ($this->isSamlConfigured()) {
+            \SimpleSAML_Session::getSessionFromRequest()->doLogout($this->authSourceName);
+        }
+    }
     /**
      * Attempts to find a valid XDMoD user associated with the attributes we receive from SAML
      *
@@ -97,7 +106,10 @@ public function isSamlConfigured()
     public function getXdmodAccount()
     {
         $samlAttrs = $this->_as->getAttributes();
-
+        /*
+         * SimpleSAMLphp uses its own session, this sets it back.
+         */
+        \SimpleSAML_Session::getSessionFromRequest()->cleanup();
         if ($this->_as->isAuthenticated()) {
             $userName = $samlAttrs['username'][0];
 

diff --git a/classes/Configuration/CommentTransformer.php b/classes/Configuration/CommentTransformer.php
@@ -16,7 +16,7 @@
 
 class CommentTransformer extends Loggable implements iConfigFileKeyTransformer
 {
-    const COMMENT_CHAR = '#';
+    const COMMENT_PREFIX = '#';
 
     /* ------------------------------------------------------------------------------------------
      * @see iConfigFileKeyTransformer::__construct()
@@ -35,7 +35,7 @@ public function __construct(Log $logger = null)
 
     public function keyMatches($key)
     {
-        return ( 0 === strpos($key, self::COMMENT_CHAR) );
+        return ( 0 === strpos($key, self::COMMENT_PREFIX) );
     }  // keyMatches()
 
     /* ------------------------------------------------------------------------------------------
@@ -48,7 +48,7 @@ public function keyMatches($key)
 
     public function transform(&$key, &$value, stdClass $obj, Configuration $config)
     {
-        $this->logger->trace("Remove comment '$key'");
+        $this->logger->trace(sprintf("Remove comment '%s'", $key));
         $key = null;
         $value = null;
 

diff --git a/classes/Configuration/Configuration.php b/classes/Configuration/Configuration.php
@@ -270,7 +270,7 @@ public function initialize($force = false)
             return;
         }
 
-        $this->logger->info("Loading" . ( $this->isLocalConfig ? " local" : "" ) . " configuration file " . $this->filename);
+        $this->logger->debug("Loading" . ( $this->isLocalConfig ? " local" : "" ) . " configuration file " . $this->filename);
 
         // Parse the configuration file
 
@@ -390,6 +390,8 @@ protected function preTransformTasks()
     {
         $this->addKeyTransformer(new CommentTransformer($this->logger));
         $this->addKeyTransformer(new JsonReferenceTransformer($this->logger));
+        $this->addKeyTransformer(new StripMergePrefixTransformer($this->logger));
+        $this->addKeyTransformer(new IncludeTransformer($this->logger));
         return $this;
     }  //preTransformTasks()
 
@@ -485,19 +487,12 @@ protected function processLocalConfig($localConfigFile)
     protected function merge(Configuration $localConfigObj, $overwrite = false)
     {
 
-        // If overwriting or the key doesn't exist, set it. Otherwise if the value is an
-        // array append it. If not overwriting and the value is not an array silently skip
-        // it.
-
-        foreach ( $localConfigObj->getTransformedConfig() as $k => $v ) {
-            if ( $overwrite || ! isset($this->transformedConfig->$k) ) {
-                $this->transformedConfig->$k = $v;
-            } elseif ( is_array($this->transformedConfig->$k) ) {
-                array_push($this->transformedConfig->$k, $v);
-            } else {
-                $this->logger->debug("Skip duplicate key in local config (overwrite == false)");
-            }
-        }
+        $this->transformedConfig = $this->mergeLocal(
+            $this->transformedConfig,
+            $localConfigObj->getTransformedConfig(),
+            $localConfigObj->getFilename(),
+            $overwrite
+        );
 
         foreach ( $localConfigObj->getSectionNames() as $sectionName ) {
             $localConfigData = $localConfigObj->getSectionData($sectionName);
@@ -515,6 +510,47 @@ protected function merge(Configuration $localConfigObj, $overwrite = false)
 
     }  // merge()
 
+    /**
+     * Merge $incoming object into the $existing object recursively.
+     *
+     * @param \stdClass $existing the object to be merged into.
+     * @param \stdClass $incoming the object to be merged from.
+     * @param string    $incomingFileName the file that $incoming originates from.
+     * @param boolean   $overwrite whether or not to force overwriting of $existing w/ $incoming.
+     * @return \stdClass the updated $existing object.
+     */
+    protected function mergeLocal(\stdClass &$existing, \stdClass $incoming, $incomingFileName, $overwrite)
+    {
+        foreach($incoming as $property => $incomingValue) {
+
+            if ( $overwrite || ! isset($existing->$property) ) {
+                $existing->$property = $incoming->$property;
+            } else {
+                $existingValue = $existing->$property;
+
+                if (is_object($existingValue) && is_object($incomingValue)) {
+                    $existing->$property = $this->mergeLocal($existingValue, $incomingValue, $incomingFileName, $overwrite);
+                } elseif (is_array($existingValue) && is_array($incomingValue)) {
+                    $existing->$property = array_merge($existingValue, $incomingValue);
+                } elseif (is_scalar($existingValue) && is_scalar($incomingValue)) {
+                    $existing->$property = $incomingValue;
+                } else {
+                    $this->logger->warning(
+                        sprintf(
+                            "Type mismatch. Unable to merge local value for key '%s' (type: %s) with global value (type: %s) from local file %s",
+                            $property,
+                            gettype($incomingValue),
+                            gettype($existingValue),
+                            $incomingFileName
+                        )
+                    );
+                }
+            }
+        }
+
+        return $existing;
+    }
+
     /**
      * Perform any tasks that need to occur after merging the local configuration objects into the
      * global configuration object. By default no actions are performed, allowing child classes to
@@ -615,7 +651,11 @@ protected function processKeyTransformers(stdClass $obj)
                     continue;
                 }
 
-                $stop = ( ! $transformer->transform($transformKey, $value, $obj, $this) );
+                try {
+                    $stop = ( ! $transformer->transform($transformKey, $value, $obj, $this) );
+                } catch ( Exception $e ) {
+                    throw new Exception(sprintf("%s: %s", $this->filename, $e->getMessage()));
+                }
 
                 if ( null === $transformKey && null === $value ) {
 
@@ -672,6 +712,14 @@ protected function processKeyTransformers(stdClass $obj)
                 }  // foreach ( $value as $element )
             }
 
+            // If we have replaced the object by something that is not Traversable (such as an
+            // included string) then do not continue the loop or the foreach will try to call
+            // valid() and next() on a non-Traversable.
+
+            if ( ! ( is_array($obj) || is_object($obj) || ($obj instanceof \Traversable) ) ) {
+                break;
+            }
+
         }  // foreach ( $obj as $key => $value )
 
         return $obj;
@@ -983,6 +1031,16 @@ public function __isset($property)
         return ( array_key_exists($property, $this->sectionData) && null !== $this->sectionData[$property] );
     }  // __isset()
 
+    /**
+     * Return this Configuration's $filename property.
+     *
+     * @return string
+     */
+    public function getFilename()
+    {
+        return $this->filename;
+    }
+
     /** -----------------------------------------------------------------------------------------
      * Return the JSON representation of the parsed and translated Configuration.
      *

diff --git a/classes/Configuration/IncludeTransformer.php b/classes/Configuration/IncludeTransformer.php
@@ -0,0 +1,49 @@
+<?php
+/** =========================================================================================
+ * Process the "$include" directive to include the contents of a file as a JSON encoded string.
+ * ==========================================================================================
+ */
+
+namespace Configuration;
+
+use stdClass;
+
+class IncludeTransformer extends aUrlTransformer implements iConfigFileKeyTransformer
+{
+    const REFERENCE_KEY = '$include';
+
+    /** -----------------------------------------------------------------------------------------
+     * @see iConfigFileKeyTransformer::keyMatches()
+     * ------------------------------------------------------------------------------------------
+     */
+
+    public function keyMatches($key)
+    {
+        return (self::REFERENCE_KEY == $key);
+    }  // keyMatches()
+
+    /** -----------------------------------------------------------------------------------------
+     * Include the JSON-endoced contents of the file as the value of the specified key.
+     *
+     * @see iConfigFileKeyTransformer::transform()
+     * ------------------------------------------------------------------------------------------
+     */
+
+    public function transform(&$key, &$value, stdClass $obj, Configuration $config)
+    {
+
+        if( count(get_object_vars($obj)) != 1 ) {
+            $this->logAndThrowException(
+                sprintf('References cannot be mixed with other keys in an object: "%s": "%s"', $key, $value)
+            );
+        }
+
+        $parsedUrl = null;
+        $contents = $this->getContentsFromUrl($value, $config);
+        $key = null;
+        $value = json_encode($contents);
+
+        return false;
+
+    }  // transform()
+}  // class IncludeTransformer