Merge 8.5 changes into GroupBy feature branch (#1055)

* Add file locking to Data Warehouse Export script (#1029) * Remove data files for deleted export requests (#1030) * update roadmap url (#1031) * Bug fixes: Add mask and make chart name unique * Refresh clears the search field * Add help window for Dashboard Components * Update profile editor for renamed rest interface. * Update dashboard chart display algorithm * Add help images for core dashboard components (#1044) * Add help images for core dashboard components * Minor text changes (#1045) * Fix metric explorer metric and dimension display bug. * Add ability to hide realm from the metric catalog * Fix Usage UI Regression (#1053) * Fix Usage UI Regression Co-Authored-By:Benjamin D. Plessinger <ben@plessinger.us>
ubccr · Sep 19, 2019 · fb7269e · fb7269e
1 parent da6e8b5
commit fb7269e
Show file tree

Hide file tree

Showing 38 changed files with 312 additions and 1,425 deletions.
diff --git a/background_scripts/batch_export_manager.php b/background_scripts/batch_export_manager.php
@@ -13,6 +13,19 @@
 ini_set('memory_limit', -1);
 
 try {
+    $lockFile = '/var/tmp/xdmod-batch-export.lock';
+    $lockFileHandle = @fopen($lockFile, 'w');
+
+    if ($lockFileHandle === false) {
+        fwrite(STDERR, sprintf("Failed to open lock file \"%s\": %s\n", $lockFile, error_get_last()));
+        exit(1);
+    }
+
+    if (!@flock($lockFileHandle, LOCK_EX | LOCK_NB)) {
+        fwrite(STDERR, "XDMoD Data Warehouse Batch Export not running due to another process holding the lock.\n");
+        exit(1);
+    }
+
     $help = false;
     $dryRun = false;
     $logLevel = -1;
@@ -76,6 +89,9 @@
     $batchProcessor->processRequests();
     // NOTE: "process_end_time" is needed for the log summary.
     $logger->notice(['message' => 'batch_export_manager end', 'process_end_time' => date('Y-m-d H:i:s')]);
+    @flock($lockFileHandle, LOCK_UN);
+    @fclose($lockFileHandle);
+    @unlink($lockFile);
     exit;
 } catch (Exception $e) {
     // Write any unexpected exceptions directly to STDERR since they may not

diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php
@@ -93,8 +93,10 @@ public function setDryRun($dryRun)
      */
     public function processRequests()
     {
-        $this->processSubmittedRequests();
+        // Delete removed and expiring files before creating new files.
+        $this->processDeletedRequests();
         $this->processExpiringRequests();
+        $this->processSubmittedRequests();
     }
 
     /**
@@ -213,6 +215,25 @@ private function processExpiringRequest(array $request)
         }
     }
 
+    /**
+     * Process requests that have been deleted.
+     *
+     * If a request has been deleted then the associated data file needs to be
+     * removed from the file system.
+     */
+    private function processDeletedRequests()
+    {
+        $this->logger->info('Processing deleted requests');
+        $this->fileManager->removeDeletedRequests(
+            array_map(
+                function ($request) {
+                    return $request['id'];
+                },
+                $this->queryHandler->listAvailableRecords()
+            )
+        );
+    }
+
     /**
      * Get the data set for the given request.
      *

diff --git a/classes/DataWarehouse/Export/FileManager.php b/classes/DataWarehouse/Export/FileManager.php
@@ -271,4 +271,33 @@ public function removeExportFile($id)
             throw new Exception(sprintf('Failed to delete "%s"', $zipFile));
         }
     }
+
+    /**
+     * Remove all data files corresponding to deleted requests.
+     *
+     * @param array $availableRequestIds Request IDs for "Available" export
+     *   files.  These correspond to data files that should not be deleted.
+     */
+    public function removeDeletedRequests(array $availableRequestIds)
+    {
+        $availableFiles = array_map(
+            [$this, 'getExportDataFilePath'],
+            $availableRequestIds
+        );
+
+        foreach (glob($this->exportDir . '/*.zip') as $exportFile) {
+            if (!in_array($exportFile, $availableFiles)) {
+                $this->logger->info([
+                    'message' => 'Removing export file',
+                    'zip_file' => $exportFile
+                ]);
+                if (!unlink($exportFile)) {
+                    throw new Exception(sprintf(
+                        'Failed to delete "%s"',
+                        $exportFile
+                    ));
+                }
+            }
+        }
+    }
 }
diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php
@@ -204,6 +204,18 @@ public function listSubmittedRecords()
         return $this->dbh->query($sql);
     }
 
+    /**
+     * Return details of all export requests presently in Available state.
+     *
+     * @return array
+     */
+    public function listAvailableRecords()
+    {
+        $sql = 'SELECT id, user_id, realm, start_date, end_date, export_file_format, requested_datetime
+            FROM batch_export_requests ' . $this->whereAvailable . ' ORDER BY requested_datetime, id';
+        return $this->dbh->query($sql);
+    }
+
     /**
      * Return export requests in Available state that should expire.
      *

diff --git a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php
@@ -103,9 +103,14 @@ public function execute()
             'data_warehouse_export_export_directory' => '/var/spool/xdmod/export',
             'data_warehouse_export_retention_duration_days' => 30
         ]);
-
+        $data = parse_ini_file($this->portalSettingsPath, true);
+        $roadmapUrl = 'https://trello.com/embed/board?id=mdFESh6j';
+        if('https://trello.com/b/mdFESh6j.html' !== $data['roadmap']['url']){
+            $roadmapUrl = $data['roadmap']['url'];
+        }
         $this->writePortalSettingsFile(array_merge(
             ['features_user_dashboard' => $dashboard],
+            ['roadmap_url' => $roadmapUrl],
             $exportSettings
         ));
     }

diff --git a/classes/Rest/Controllers/DashboardControllerProvider.php b/classes/Rest/Controllers/DashboardControllerProvider.php
@@ -71,7 +71,26 @@ private function getConfigVariables($user)
     }
 
     /**
+     * The individual dashboard components have a namespace prefix to simplify
+     * the implementation of the algorithm that determines which
+     * components to display. There are two sources of configuration data for
+     * the components. The roles configuration file and the user configuration
+     * (in the database). The user configuration only contains chart components.
+     * The user configuration is handled via the "Show in Summary tab" checkbox
+     * in the metric explorer.
+     *
+     * Non-chart components and the full-width components are defined in the roles
+     * configuration file and are not overrideable.
+     *
+     * Chart components are handled as follows:
+     * - All user charts with "show in summary tab" checked will be displayed
+     * - If a user chart has the same name as a chart in the role configuration
+     *   then its settings will be used in place of the role chart.
      */
+    const TOP_COMPONENT = 't.';
+    const CHART_COMPONENT = 'c.';
+    const NON_CHART_COMPONENT = 'p.';
+
     public function getComponents(Request $request, Application $app)
     {
         $user = $this->getUserFromRequest($request);
@@ -94,47 +113,37 @@ public function getComponents(Request $request, Application $app)
         if (isset($presets['dashboard_components'])) {
 
             foreach($presets['dashboard_components'] as $component) {
+
+                $componentType = self::NON_CHART_COMPONENT;
+
                 if (isset($component['region']) && $component['region'] === 'top') {
-                    $chartLocation = 'FW' . $component['name'];
+                    $componentType = self::TOP_COMPONENT;
+                    $chartLocation = $componentType . $component['name'];
                     $column = -1;
                 } else {
+                    if ($component['type'] === 'xdmod-dash-chart-cmp') {
+                        $componentType = self::CHART_COMPONENT;
+                        $component['config']['name'] = $component['name'];
+                        $component['config']['chart']['featured'] = true;
+                    }
+
                     $defaultLayout = null;
                     if (isset($component['location']) && isset($component['location']['row']) && isset($component['location']['column'])) {
                         $defaultLayout = array($component['location']['row'], $component['location']['column']);
                     }
 
-                    list($chartLocation, $column) = $layout->getLocation('PP' . $component['name'], $defaultLayout);
+                    list($chartLocation, $column) = $layout->getLocation($componentType . $component['name'], $defaultLayout);
                 }
 
                 $dashboardComponents[$chartLocation] = array(
-                        'name' => 'PP' . $component['name'],
+                        'name' => $componentType . $component['name'],
                         'type' => $component['type'],
                         'config' => isset($component['config']) ? $component['config'] : array(),
                         'column' => $column
                 );
             }
         }
 
-        $presetCharts = isset($presets['summary_charts']) ? $presets['summary_charts'] : $roleConfig['roles']['default']['summary_charts'];
-
-        foreach ($presetCharts as $index => $presetChart)
-        {
-            $presetChart['featured'] = true;
-            $presetChart['aggregation_unit'] = 'Auto';
-            $presetChart['timeframe_label'] = 'Previous month';
-
-            list($chartLocation, $column) = $layout->getLocation('PC' . $index);
-            $dashboardComponents[$chartLocation] = array(
-                'name' => 'PC' . $index,
-                'type' => 'xdmod-dash-chart-cmp',
-                'config' => array(
-                    'name' => 'summary_' . $index,
-                    'chart' => $presetChart
-                ),
-                'column' => $column
-            );
-        }
-
         if ($user->isPublicUser() === false)
         {
             $queryStore = new \UserStorage($user, 'queries_store');
@@ -152,13 +161,7 @@ public function getComponents(Request $request, Application $app)
                         continue;
                     }
 
-                    $name = 'UC' . $query['name'];
-
-                    if (preg_match('/summary_(?P<index>\S+)/', $query['name'], $matches) > 0) {
-                        if ($layout->hasLayout('PC' . $matches['index'])) {
-                            $name = 'PC' . $matches['index'];
-                        }
-                    }
+                    $name = self::CHART_COMPONENT . $query['name'];
 
                     list($chartLocation, $column) = $layout->getLocation($name);
 

diff --git a/configuration/assets.json b/configuration/assets.json
@@ -9,6 +9,7 @@
             ],
             "css": [
                 "gui/css/JobComponent.css",
+                "gui/css/ChartComponent.css",
                 "gui/css/ReportThumbnailsComponent.css"
             ]
         }

diff --git a/configuration/etl/etl_data.d/xdb/report-templates.json b/configuration/etl/etl_data.d/xdb/report-templates.json
@@ -1,6 +1,6 @@
 [
     ["id", "name", "description", "template", "title", "header", "footer", "format", "font", "schedule", "delivery", "charts_per_page", "use_submenu"],
     [1, "Quarterly Report - Center Director", "Quarterly Report - Center Director", "GenericReportTemplate", "Quarterly Report - Center Director", "", "", "Pdf", "Arial", "Quarterly", "E-mail", 1, 0],
-    [2, "Summary Tab Report", "Summary Tab Report", "GenericReportTemplate", "Summary Tab Report", "", "", "Pdf", "Arial", "Once", "E-mail", 1, 0],
-    [3, "Summary Tab Report", "Summary Tab Report", "GenericReportTemplate", "Summary Tab Report", "", "", "Pdf", "Arial", "Once", "E-mail", 1, 0]
+    [2, "Dashboard Tab Report", "Dashboard Tab Report", "GenericReportTemplate", "Dashboard Tab Report", "", "", "Pdf", "Arial", "Once", "E-mail", 1, 0],
+    [3, "Dashboard Tab Report", "Dashboard Tab Report", "GenericReportTemplate", "Dashboard Tab Report", "", "", "Pdf", "Arial", "Once", "E-mail", 1, 0]
 ]
diff --git a/configuration/portal_settings.ini b/configuration/portal_settings.ini
@@ -61,7 +61,7 @@ dw_desc_cache = "off"
 
 ; Settings that control the "Roadmap" action on the "About" tab.
 [roadmap]
-url = "https://trello.com/b/mdFESh6j.html"
+url = "https://trello.com/embed/board?id=mdFESh6j"
 header = "Located below is the XDMoD Development roadmap, organized by XDMoD release and powered by Trello.com. To view the full roadmap as well as vote and comment on features click any one of the elements on the roadmap.  This will take you to the full roadmap on the Trello.com site in a new browser window (or tab).  All users will be able to view the roadmap, however if you wish to vote or comment on a feature you will need to create a (free) Trello account if you do not already have one."
 
 [rest]

diff --git a/configuration/roles.d/dashboard.json b/configuration/roles.d/dashboard.json
@@ -67,7 +67,7 @@
                     }
                 },
                 {
-                    "name": "summary_0",
+                    "name": "DashBoardChart_PiWallHoursByPerson",
                     "type": "xdmod-dash-chart-cmp",
                     "config": {
                         "chart": {
@@ -171,7 +171,7 @@
                     }
                 },
                 {
-                    "name": "summary_1",
+                    "name": "DashBoardChart_PiWaitHoursByJobSize",
                     "type": "xdmod-dash-chart-cmp",
                     "config": {
                         "chart": {
@@ -223,7 +223,7 @@
                                         "category": "Jobs",
                                         "color": "8BBC21",
                                         "combine_type": "side",
-                                        "display_type": "bar",
+                                        "display_type": "column",
                                         "enabled": true,
                                         "filters": {
                                             "data": [],
@@ -254,17 +254,6 @@
                             },
                             "font_size": 2,
                             "hide_tooltip": false,
-                            "legend": {
-                                "Std Err: Wait Hours: Per Job": {
-                                    "title": "Std Err"
-                                },
-                                "Std Err: Wait Hours: Per Job {User =  ${PERSON_NAME}}": {
-                                    "title": "Std Err"
-                                },
-                                "Wait Hours: Per Job {User =  ${PERSON_NAME}}": {
-                                    "title": "My Wait Hours: Per Job"
-                                }
-                            },
                             "legend_type": "top_center",
                             "limit": 20,
                             "share_y_axis": false,
@@ -361,7 +350,7 @@
                                         "category": "Jobs",
                                         "color": "8BBC21",
                                         "combine_type": "side",
-                                        "display_type": "bar",
+                                        "display_type": "column",
                                         "enabled": true,
                                         "filters": {
                                             "data": [],
@@ -392,17 +381,6 @@
                             },
                             "font_size": 2,
                             "hide_tooltip": false,
-                            "legend": {
-                                "Std Err: Wait Hours: Per Job": {
-                                    "title": "Std Err"
-                                },
-                                "Std Err: Wait Hours: Per Job {User =  ${PERSON_NAME}}": {
-                                    "title": "Std Err"
-                                },
-                                "Wait Hours: Per Job {User =  ${PERSON_NAME}}": {
-                                    "title": "My Wait Hours: Per Job"
-                                }
-                            },
                             "legend_type": "top_center",
                             "limit": 20,
                             "share_y_axis": false,
@@ -420,7 +398,7 @@
                         "column": 1,
                         "row": 0
                     },
-                    "name": "summary_0",
+                    "name": "DashBoardChart_WaitHoursByJobSize",
                     "type": "xdmod-dash-chart-cmp"
                 },
                 {
@@ -461,22 +439,6 @@
                                 "total": 1
                             },
                             "font_size": 3,
-                            "global_filters": {
-                                "data": [
-                                    {
-                                        "categories": "",
-                                        "checked": true,
-                                        "dimension_id": "resource",
-                                        "id": "resource=13",
-                                        "realms": [
-                                            "Jobs"
-                                        ],
-                                        "value_id": "13",
-                                        "value_name": "ub hpc"
-                                    }
-                                ],
-                                "total": 1
-                            },
                             "hide_tooltip": false,
                             "legend_type": "off",
                             "limit": 10,
@@ -501,7 +463,7 @@
                         "column": 0,
                         "row": 1
                     },
-                    "name": "summary_1",
+                    "name": "DashBoardChart_WaitHoursByQueue",
                     "type": "xdmod-dash-chart-cmp"
                 }
             ]