Tests

classes/OpenXdmod/Migration/DatabasesMigration.php

@@ -150,4 +150,39 @@ protected function requestMysqlAdminCredentials()
 
         return self::$mysqlAdminCredentials;
     }
+
+    /**
+     * Run an etlv2 action.
+     *
+     * This helper function provides the necessary boilerplate to execute
+     * an etlv2 pipeline.
+     *
+     * @param array $scriptOptions the options to pass to the etlv2 class
+     */
+    protected function runEtlv2(array $scriptOptions) {
+        if (empty($scriptOptions['chunk-size-days'])) {
+            $scriptOptions['chunk-size-days'] = 365;
+        }
+        if (empty($scriptOptions['default-module-name'])) {
+            $scriptOptions['default-module-name'] = 'xdmod';
+        }
+        if(empty($scriptOptions['start-date'])){
+            $scriptOptions['start-date'] = date('Y-m-d', strtotime('2000-01-01'));
+        }
+        if(empty($scriptOptions['end-date'])){
+            $scriptOptions['end-date'] = date('Y-m-d', strtotime('2038-01-18'));
+        }
+        if(empty($scriptOptions['last-modified-start-date'])){
+            $scriptOptions['last-modified-start-date'] = date('Y-m-d');
+        }
+
+        $etlConfig = new \ETL\Configuration\EtlConfiguration(CONFIG_DIR . '/etl/etl.json', null, $this->logger, array('default_module_name' => $scriptOptions['default-module-name']));
+        $etlConfig->initialize();
+        \ETL\Utilities::setEtlConfig($etlConfig);
+
+        $overseerOptions = new \ETL\EtlOverseerOptions($scriptOptions, $this->logger);
+
+        $overseer = new \ETL\EtlOverseer($overseerOptions, $this->logger);
+        $overseer->execute($etlConfig);
+    }
 }

classes/OpenXdmod/Migration/Version800To810/DatabasesMigration.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace OpenXdmod\Migration\Version800To810;
+
+use CCR\DB;
+use OpenXdmod\DataWarehouseInitializer;
+use FilterListBuilder;
+use TimePeriodGenerator;
+use OpenXdmod\Setup\Console;
+use CCR\DB\MySQLHelper;
+
+/**
+ * Migrate databases from version 8.0.0 to 8.1.0
+ */
+class DatabasesMigration extends \OpenXdmod\Migration\DatabasesMigration
+{
+    /**
+     * @see \OpenXdmod\Migration\Migration::execute
+     **/
+    public function execute()
+    {
+        parent::execute();
+
+        $console = Console::factory();
+        $console->displayMessage(<<<"EOT"
+This release includes an update that enables the Job Viewer tab and the 'show raw
+data' drilldown in the Metric Explorer for Jobs realm data. It is recommended
+to reaggregate all jobs. Depending on the amount of data this could take multiple
+hours. If the job data is not reaggregated then existing jobs will not be viewable
+in the Job Viewer.
+EOT
+        );
+        $runaggregation = $console->promptBool(
+            'Do you want to run aggregation now?',
+            false
+        );
+        if (true === $runaggregation) {
+            $this->runEtlv2(
+                array(
+                    'process-sections' => array('jobs-xdw-aggregate'),
+                    'last-modified-start-date' => date('Y-m-d', strtotime('2000-01-01')),
+                )
+            );
+            $this->logger->notice('Rebuilding filter lists');
+            try {
+                $builder = new FilterListBuilder();
+                $builder->setLogger($this->logger);
+                $builder->buildRealmLists('Jobs');
+            } catch (Exception $e) {
+                $this->logger->notice('Failed BuildAllLists: '  . $e->getMessage());
+                $this->logger->crit(array(
+                    'message'    => 'Filter list building failed: ' . $e->getMessage(),
+                    'stacktrace' => $e->getTraceAsString(),
+                ));
+                throw new \Exception('Filter list building failed: ' . $e->getMessage());
+            }
+            $this->logger->notice('Done building filter lists');
+        }
+        else {
+            $console->displayMessage(<<<"EOT"
+Aggregation not run.  Aggregation may be run manually with the following command:
+xdmod-ingestor --aggregate --last-modified-start-date '2000-01-01'
+EOT
+            );
+        }
+    }
+}

configuration/etl/etl.d/xdmod-migration-8_0_0-8_1_0.json

@@ -0,0 +1,24 @@
+{
+    "module": "xdmod",
+    "migration-8_0_0-8_1_0": [
+        {
+            "name": "update-modw_aggregates-tables",
+            "description": "Update modw_aggregates tables",
+            "namespace": "ETL\\Maintenance",
+            "class": "ManageTables",
+            "options_class": "MaintenanceOptions",
+            "definition_file_list": [
+                "jobs/xdw/jobfact_by_day.json",
+                "jobs/xdw/jobfact_by_day_joblist.json"
+            ],
+            "endpoints": {
+                "destination": {
+                    "type": "mysql",
+                    "name": "Aggregate tables",
+                    "config": "database",
+                    "schema": "modw_aggregates"
+                }
+            }
+        }
+    ]
+}

open_xdmod/modules/xdmod/integration_tests/scripts/xdmod-upgrade.tcl

@@ -19,6 +19,14 @@ proc confirmUpgrade { } {
 set timeout 180
 spawn "xdmod-upgrade"
 confirmUpgrade
+expect {
+    -re "\nDo you want to run aggregation now.*\\\]" {
+        send yes\n
+    }
+    timeout {
+        send_user "\nFailed to get prompt\n"; exit 1
+    }
+}
 expect {
     timeout {
         send_user "\nFailed to get prompt\n"; exit 1

tests/artifacts/xdmod-test-artifacts/xdmod/acls/output/roles.json

@@ -38,6 +38,15 @@
                     "userManualSectionName": "Report Generator",
                     "tooltip": ""
                 },
+                {
+                    "name": "job_viewer",
+                    "title": "Job Viewer",
+                    "position": 5000,
+                    "javascriptClass": "XDMoD.Module.JobViewer",
+                    "javascriptReference": "CCR.xdmod.ui.jobViewer",
+                    "userManualSectionName": "Job Viewer",
+                    "tooltip": "View detailed job-level metrics"
+                },
                 {
                     "name": "about_xdmod",
                     "title": "About",