diff --git a/.eslintrc.json b/.eslintrc.json index 0ee407d3..60eb9e7e 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -10,6 +10,14 @@ "error", "all" ], + "no-underscore-dangle": [ + "error", + { + "allow": [ + "_extend" + ] + } + ], "brace-style": [ "error", "1tbs", diff --git a/classes/DataWarehouse/Query/SUPREMM/SupremmDbInterface.php b/classes/DataWarehouse/Query/SUPREMM/SupremmDbInterface.php index b136f41e..ef64c629 100644 --- a/classes/DataWarehouse/Query/SUPREMM/SupremmDbInterface.php +++ b/classes/DataWarehouse/Query/SUPREMM/SupremmDbInterface.php @@ -22,6 +22,10 @@ public function __construct() { } $sresource['database'] = \xd_utilities\getConfigurationSection($dbsection); + if (!isset($sresource['collection']) ) { + $sresource['collection'] = 'resource_' . $sresource['resource_id']; + } + $this->resource_rmap[$sresource['resource_id']] = $sresource; } diff --git a/classes/OpenXdmod/Setup/SupremmEditResourceSetup.php b/classes/OpenXdmod/Setup/SupremmEditResourceSetup.php index 0d85a866..ff39b914 100644 --- a/classes/OpenXdmod/Setup/SupremmEditResourceSetup.php +++ b/classes/OpenXdmod/Setup/SupremmEditResourceSetup.php @@ -44,7 +44,6 @@ public function handle() if($this->resourceConf['enabled']) { $this->resourceConf['datasetmap'] = $this->console->prompt('Dataset mapping', $this->resourceConf['datasetmap']); - $this->resourceConf['collection'] = $this->console->prompt('MongoDB collection name', $this->resourceConf['collection']); $this->resourceConf['hardware']['gpfs'] = $this->console->prompt('GPFS mount point (leave empty if no GPFS)', $this->resourceConf['hardware']['gpfs']); } diff --git a/classes/OpenXdmod/Setup/SupremmResourcesSetup.php b/classes/OpenXdmod/Setup/SupremmResourcesSetup.php index 4938812b..0e7dfc5c 100644 --- a/classes/OpenXdmod/Setup/SupremmResourcesSetup.php +++ b/classes/OpenXdmod/Setup/SupremmResourcesSetup.php @@ -110,7 +110,6 @@ private function defaultSettings($id) 'resource' => '', 'enabled' => false, 'datasetmap' => 'pcp', - 'collection' => "resource_$id", 'hardware' => array( 'gpfs' => '' ) diff --git a/configuration/supremm_resources.json b/configuration/supremm_resources.json index d566ca8f..67c3c655 100644 --- a/configuration/supremm_resources.json +++ b/configuration/supremm_resources.json @@ -3,7 +3,6 @@ { "resource": "pcp", "resource_id": 1, - "collection": "resource_1", "datasetmap": "pcp", "hardware": { "gpfs": "gpfs0" diff --git a/docs/supremm-configuration.md b/docs/supremm-configuration.md index a3ed545f..8f5909d2 100644 --- a/docs/supremm-configuration.md +++ b/docs/supremm-configuration.md @@ -51,8 +51,7 @@ By default all the resources are disabled. You must select the "Edit resource" option for each resource that you wish to configure to appear in the SUPReMM realm and follow the prompt to enable the resource and set the correct options. The "Dataset mapping" should be set to pcp if processing job summaries -generated from PCP data. The MongoDB collection name should be set to the name -of the mongoDB collection that contains the job summaries for the resource. +generated from PCP data. SUPReMM configuration files --------------------------- @@ -75,7 +74,6 @@ the `resources.json` and `resource_specs.json` main configuration files "resource_id": 1, "enabled": true, "datasetmap": "pcp", - "collection": "resource_1", "hardware": { "gpfs": "" } @@ -99,9 +97,6 @@ where `%resource%` should be replaced with the `resource` parameter from the The `datasetmap` option allows the ingestion of SUPReMM data from different data sources. Currently PCP is the only supported data source. -The collection name should be set to the name of the mongoDB collection that -contains the job summaries for the resource. - The `hardware` property is used by the dataset mapping code to process PCP metrics that have device-specific names. The only configurable mapping in this release is the name of the GPFS mount point. If the resource has @@ -124,6 +119,22 @@ the database name in the connection URI.** If the database is not specifed then the mongo driver defaults to the 'test' database, which will not contain the job summary data. +Advanced Configuration Options +--------------------------- + +The resource configuration file `supremm_resources.json` has optional advanced +configuration settings for each resource. + +The `$.resources[*].collection` option overrides the collection name in the +MongoDB. This option can be used to set a non default collection name. + +The `$.resources[*].db` option specifies the name of the section in the +portal_settings file that contains the database configuration settings. This +setting can be used to support an XDMoD instance ingesting data from multiple +MongoDB databases. + + + [mongo documentation]: https://docs.mongodb.org/manual/reference/connection-string/ [shredder]: shredder.md [ingestor]: ingestor.md diff --git a/etl/js/config/supremm/etl.profile.js b/etl/js/config/supremm/etl.profile.js index 262ba364..6836069a 100644 --- a/etl/js/config/supremm/etl.profile.js +++ b/etl/js/config/supremm/etl.profile.js @@ -43,26 +43,31 @@ var markAsProcessedMongoUpdate = function(collection, _id, config, endFn, dbkey) }); }; -var getMongoSettings = function(config, datasetConfig) { - - var section = "jobsummarydb"; - if(datasetConfig.db) { +var getMongoSettings = function (config, datasetConfig) { + var section = 'jobsummarydb'; + if (datasetConfig.db) { section = datasetConfig.db; } - var dbsettings = config.xdmodConfig[section]; - - if(!dbsettings) { - throw new Error("MongoDB configuration section \"" + section + "\" missing from XDMoD portal_settings."); + if (!config.xdmodConfig[section]) { + throw new Error('MongoDB configuration section "' + section + '" missing from XDMoD portal_settings.'); } + var dbsettings = util._extend({}, config.xdmodConfig[section]); + // Backwards compatibility with host/port specification - if(!dbsettings.uri && dbsettings.host && dbsettings.port && dbsettings.db) { - dbsettings.uri = "mongodb://" + dbsettings.host + ":" + dbsettings.port + "/" + dbsettings.db; + if (!dbsettings.uri && dbsettings.host && dbsettings.port && dbsettings.db) { + dbsettings.uri = 'mongodb://' + dbsettings.host + ':' + dbsettings.port + '/' + dbsettings.db; + } + + if (!(dbsettings.uri && dbsettings.db_engine)) { + throw new Error('Missing MongoDB configuration settings in section "' + section + '" of the XDMoD portal_settings.'); } - if(!(dbsettings.uri && dbsettings.db_engine)) { - throw new Error("Missing MongoDB configuration settings in section \"" + section + "\" of the XDMoD portal_settings."); + if (datasetConfig.collection) { + dbsettings.collection = datasetConfig.collection; + } else { + dbsettings.collection = 'resource_' + datasetConfig.resource_id; } return dbsettings; @@ -114,9 +119,7 @@ module.exports = { mapping: this.getDatasetMap(datasetConfig[i].datasetmap, datasetConfig[i]), input: { dbEngine: db.db_engine.toLowerCase(), - config: util._extend({ - collection: datasetConfig[i].collection - }, db), + config: db, sortQuery: {}, getQuery: function() { return getJobsToProcessMongoQuery(this.databasekey); diff --git a/tests/integration_tests/scripts/xdmod-setup.tcl b/tests/integration_tests/scripts/xdmod-setup.tcl index a484eed4..f15d8dce 100644 --- a/tests/integration_tests/scripts/xdmod-setup.tcl +++ b/tests/integration_tests/scripts/xdmod-setup.tcl @@ -75,7 +75,6 @@ for {set i 1} {$i < 6} {incr i} { selectMenuOption $i answerQuestion {Enabled \(yes, no\)} {yes} answerQuestion {Dataset mapping} {pcp} - answerQuestion {MongoDB collection name} "resource_$i" provideInput {GPFS mount point (leave empty if no GPFS)} {gpfs0} } selectMenuOption s