Merge current 8.5 changes into feature branch (#1081)

* Bug fixes: Add mask and make chart name unique * Refresh clears the search field * Check for alternate timezone field (#1061) * Data warehouse export tests and bug fix (#1063) * Fix dry run behavior * Add data warehouse export tests * Add Usage Explorer Tests (#1062) * Add Usage Tests for disabled Display Menu Items * Document required file name extension (#1066) * Fixing an infinite loop when discarding user changes (#1068) * Replace previous year with year to date * Ensure Jobs realm data access class provides required fields * Force refresh dashboard tab
ubccr · Oct 1, 2019 · 1ee0708 · 1ee0708
1 parent aeed36d
commit 1ee0708
Show file tree

Hide file tree

Showing 31 changed files with 1,084 additions and 38 deletions.
diff --git a/classes/DataWarehouse/Data/RawDataset.php b/classes/DataWarehouse/Data/RawDataset.php
@@ -94,6 +94,9 @@ public function export()
                 if (isset($this->query_results[0]['timezone'])) {
                     $value = $this->formattime($value, $this->query_results[0]['timezone']);
                     $units = '';
+                } elseif (isset($this->query_results[0]['Timezone'])) {
+                    $value = $this->formattime($value, $this->query_results[0]['Timezone']);
+                    $units = '';
                 }
             }
 

diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php
@@ -143,10 +143,12 @@ private function processSubmittedRequest(array $request)
             $dataSet = $this->getDataSet($request, $user);
             $format = $this->dryRun ? 'null' : $request['export_file_format'];
             $dataFile = $this->fileManager->writeDataSetToFile($dataSet, $format);
-            $this->fileManager->createZipFile($dataFile, $request);
+            if (!$this->dryRun) {
+                $this->fileManager->createZipFile($dataFile, $request);
+            }
 
             // Delete file that was added to zip archive.
-            if (!unlink($dataFile)) {
+            if (!$this->dryRun && !unlink($dataFile)) {
                 $this->logger->err(sprintf(
                     'Failed to delete temporary data file "%s"',
                     $dataFile

diff --git a/classes/DataWarehouse/Export/FileManager.php b/classes/DataWarehouse/Export/FileManager.php
@@ -197,6 +197,7 @@ public function writeDataSetToFile(BatchDataset $dataSet, $format)
      *
      * @param string $dataFile Path to file that will be put in the zip file.
      * @param array $request Batch export request data.
+     * @return string Zip file path.
      * @throws \Exception If creating the zip file fails.
      */
     public function createZipFile($dataFile, array $request)

diff --git a/classes/DataWarehouse/Query/Jobs/RawData.php b/classes/DataWarehouse/Query/Jobs/RawData.php
@@ -3,6 +3,7 @@
 
 use \DataWarehouse\Query\Model\Table;
 use \DataWarehouse\Query\Model\TableField;
+use \DataWarehouse\Query\Model\FormulaField;
 use \DataWarehouse\Query\Model\WhereCondition;
 use \DataWarehouse\Query\Model\Schema;
 
@@ -73,6 +74,9 @@ public function __construct(
 
         $this->addField(new TableField($factTable, "job_id", "jobid"));
         $this->addField(new TableField($factTable, "local_jobid", "local_job_id"));
+        $this->addField(new TableField($factTable, 'start_time_ts'));
+        $this->addField(new TableField($factTable, 'end_time_ts'));
+        $this->addField(new FormulaField('-1', 'cpu_user'));
 
         $this->addTable($joblistTable);
         $this->addTable($factTable);
@@ -99,6 +103,14 @@ public function __construct(
                 // All other metrics show running job count
                 break;
         }
+
+        $this->prependOrder(
+            new \DataWarehouse\Query\Model\OrderBy(
+                new TableField($factTable, 'end_time_ts'),
+                'DESC',
+                'end_time_ts'
+            )
+        );
     }
 
     /**

diff --git a/configuration/etl/etl_action_defs.d/xdb/report-template-charts.json b/configuration/etl/etl_action_defs.d/xdb/report-template-charts.json
@@ -37,11 +37,11 @@
                 "schema": "${SOURCE_SCHEMA}",
                 "alias": "rtc",
                 "type": "left outer",
-                "on": "rtcs.template_id=rtc.template_id",
-                "where": [
-                    "rtc.template_id IS NULL"
-                ]
+                "on": "rtcs.template_id=rtc.template_id"
             }
+        ],
+        "where": [
+            "rtc.template_id IS NULL"
         ]
     }
 }
diff --git a/configuration/etl/etl_action_defs.d/xdb/report-templates.json b/configuration/etl/etl_action_defs.d/xdb/report-templates.json
@@ -49,11 +49,11 @@
                 "schema": "${SOURCE_SCHEMA}",
                 "alias": "rt",
                 "type": "left outer",
-                "on": "rts.id=rt.id",
-                "where": [
-                    "rt.id IS NULL"
-                ]
+                "on": "rts.id=rt.id"
             }
+        ],
+        "where": [
+            "rt.id IS NULL"
         ]
     }
 }
diff --git a/configuration/roles.d/dashboard.json b/configuration/roles.d/dashboard.json
@@ -4,7 +4,7 @@
             "dashboard_components": [
                 {
                     "config": {
-                        "timeframe": "Previous year"
+                        "timeframe": "1 year"
                     },
                     "name": "Chart Thumbnails",
                     "region": "top",

diff --git a/docs/storage.md b/docs/storage.md
@@ -25,7 +25,8 @@ installed from source they will need to be adjusted accordingly.
 
 ## Input Format
 
-Storage data must be formatted in JSON files.  These files will be validated
+Storage data must be formatted in JSON files and these files must use the
+`.json` file extension (e.g. `2019-01-01.json`).  These files will be validated
 against the JSON Schema
 `/etc/xdmod/etl/etl_schemas.d/storage/usage.schema.json`.
 

diff --git a/html/gui/js/modules/dashboard/ReportThumbnailsComponent.js b/html/gui/js/modules/dashboard/ReportThumbnailsComponent.js
@@ -286,18 +286,17 @@ XDMoD.Module.Dashboard.ReportThumbnailsComponent = Ext.extend(Ext.Panel, {
                         }
                     },
                     {
-                        text: 'Previous year',
-                        checked: timeframe_label === 'Previous year',
+                        text: '1 year',
+                        checked: timeframe_label === '1 year',
                         group: 'timeframe',
                         listeners: {
                             click: function () {
                                 var today = new Date();
-                                var oneYearAgoStart = new Date(today.getFullYear() - 1, 0, 1);
-                                var oneYearAgoEnd = new Date(today.getFullYear() - 1, 11, 31);
-                                var start = oneYearAgoStart;
-                                var end = oneYearAgoEnd;
+                                var lastYear = today.add(Date.YEAR, -1);
+                                var start = lastYear;
+                                var end = today;
                                 this.ownerCt.ownerCt.ownerCt.ownerCt.fireEvent('timeframe_change', start, end);
-                                this.ownerCt.ownerCt.ownerCt.items.items[1].setText('Previous year');
+                                this.ownerCt.ownerCt.ownerCt.items.items[1].setText('1 year');
                                 this.ownerCt.ownerCt.ownerCt.items.items[2].setText('<b>' + self.timeframe.start_date + ' to ' + self.timeframe.end_date + '</b>');
                             }
                         }

diff --git a/html/gui/js/report_builder/ReportsOverview.js b/html/gui/js/report_builder/ReportsOverview.js
@@ -377,6 +377,7 @@ XDMoD.ReportsOverview = Ext.extend(Ext.Panel,  {
                      }
 
                   });//conn.request
+                  CCR.xdmod.ui.tgSummaryViewer.fireEvent('request_refresh');
 
                }//if (resp == 'yes')
                else {

diff --git a/html/internal_dashboard/js/admin_panel/SectionExistingUsers.js b/html/internal_dashboard/js/admin_panel/SectionExistingUsers.js
@@ -784,7 +784,28 @@ XDMoD.ExistingUsers = Ext.extend(Ext.Panel, {
             return false;
         };
 
+        /**
+         * Reverts the User Settings ( E-Mail Address, User Type, Map To, Institution ) to their original values.
+         */
+        var revertUserSettings = function () {
+            // eslint-disable-next-line no-use-before-define
+            var fieldValues = userSettings.getForm().getFieldValues();
+            for (var id in fieldValues) {
+                if (fieldValues.hasOwnProperty(id)) {
+                    var field = Ext.getCmp(id);
+
+                    // Only check a field if it's been rendered.
+                    if (field.rendered) {
+                        if (field.originalValue !== field.getValue()) {
+                            field.setValue(field.originalValue);
+                        }
+                    }
+                }
+            }
+        };
+
         self.resetDirtyState = function () {
+            revertUserSettings();
             roleGrid.setDirtyState(false);
         };
 

diff --git a/tests/artifacts/xdmod/component/export/file_manager/input/export-request-ids.json b/tests/artifacts/xdmod/component/export/file_manager/input/export-request-ids.json
@@ -0,0 +1,23 @@
+[
+    [
+        1
+    ],
+    [
+        2
+    ],
+    [
+        3
+    ],
+    [
+        10
+    ],
+    [
+        100
+    ],
+    [
+        12345
+    ],
+    [
+        4294967296
+    ]
+]
diff --git a/tests/artifacts/xdmod/component/export/file_manager/input/export-requests.json b/tests/artifacts/xdmod/component/export/file_manager/input/export-requests.json
@@ -0,0 +1,36 @@
+[
+    [
+        {
+            "id": 1,
+            "user_id": 1,
+            "realm": "jobs",
+            "start_date": "2018-01-01",
+            "end_date": "2018-02-28",
+            "export_file_format": "CSV",
+            "requested_datetime": "2019-01-01 00:00:00",
+            "export_succeeded": null,
+            "export_created_datetime": null,
+            "downloaded_datetime": null,
+            "export_expires_datetime": null,
+            "export_expired": 0,
+            "last_modified": "2019-01-01 00:00:00"
+        }
+    ],
+    [
+        {
+            "id": 2,
+            "user_id": 1,
+            "realm": "jobs",
+            "start_date": "2001-01-01",
+            "end_date": "2001-01-02",
+            "export_file_format": "JSON",
+            "requested_datetime": "2019-01-01 00:00:01",
+            "export_succeeded": null,
+            "export_created_datetime": null,
+            "downloaded_datetime": null,
+            "export_expires_datetime": null,
+            "export_expired": 0,
+            "last_modified": "2019-01-01 00:00:01"
+        }
+    ]
+]
diff --git a/tests/artifacts/xdmod/component/export/file_manager/output/1.csv b/tests/artifacts/xdmod/component/export/file_manager/output/1.csv
@@ -0,0 +1,3 @@
+heading1,heading2,heading3
+0,1,2
+a,b,c
diff --git a/tests/artifacts/xdmod/component/export/file_manager/output/2.json b/tests/artifacts/xdmod/component/export/file_manager/output/2.json
@@ -0,0 +1,17 @@
+[
+    [
+        "heading1",
+        "heading2",
+        "heading3"
+    ],
+    [
+        0,
+        1,
+        2
+    ],
+    [
+        "a",
+        "b",
+        "c"
+    ]
+]
diff --git a/tests/artifacts/xdmod/unit/data_warehouse/export/file_writer/csv/write_records.json b/tests/artifacts/xdmod/unit/data_warehouse/export/file_writer/csv/write_records.json
@@ -0,0 +1,32 @@
+{
+    "Single record with one element": [
+        [
+            [
+                "a"
+            ]
+        ],
+        "a\n"
+    ],
+    "Single record with two elements": [
+        [
+            [
+                "a",
+                "b"
+            ]
+        ],
+        "a,b\n"
+    ],
+    "Two records with two elements each": [
+        [
+            [
+                "a",
+                "b"
+            ],
+            [
+                "c",
+                "d"
+            ]
+        ],
+        "a,b\nc,d\n"
+    ]
+}
diff --git a/tests/artifacts/xdmod/unit/data_warehouse/export/file_writer/factory/create_file_writer.json b/tests/artifacts/xdmod/unit/data_warehouse/export/file_writer/factory/create_file_writer.json
@@ -0,0 +1,14 @@
+{
+    "CSV File Writer": [
+        "csv",
+        "\\DataWarehouse\\Export\\FileWriter\\CsvFileWriter"
+    ],
+    "JSON File Writer": [
+        "json",
+        "\\DataWarehouse\\Export\\FileWriter\\JsonFileWriter"
+    ],
+    "NULL File Writer": [
+        "null",
+        "\\DataWarehouse\\Export\\FileWriter\\NullFileWriter"
+    ]
+}
diff --git a/tests/artifacts/xdmod/unit/data_warehouse/export/file_writer/json/write_records.json b/tests/artifacts/xdmod/unit/data_warehouse/export/file_writer/json/write_records.json
@@ -0,0 +1,32 @@
+{
+    "Single record with one element": [
+        [
+            [
+                "a"
+            ]
+        ],
+        "[[\"a\"]]"
+    ],
+    "Single record with two elements": [
+        [
+            [
+                "a",
+                "b"
+            ]
+        ],
+        "[[\"a\",\"b\"]]"
+    ],
+    "Two records with two elements each": [
+        [
+            [
+                "a",
+                "b"
+            ],
+            [
+                "c",
+                "d"
+            ]
+        ],
+        "[[\"a\",\"b\"],[\"c\",\"d\"]]"
+    ]
+}
diff --git a/tests/artifacts/xdmod/unit/data_warehouse/export/file_writer/null/write_records.json b/tests/artifacts/xdmod/unit/data_warehouse/export/file_writer/null/write_records.json
@@ -0,0 +1,29 @@
+{
+    "Single record with one element": [
+        [
+            [
+                "a"
+            ]
+        ]
+    ],
+    "Single record with two elements": [
+        [
+            [
+                "a",
+                "b"
+            ]
+        ]
+    ],
+    "Two records with two elements each": [
+        [
+            [
+                "a",
+                "b"
+            ],
+            [
+                "c",
+                "d"
+            ]
+        ]
+    ]
+}