Modernize CondTools/HLT and add unit tests

CondTools/HLT/test/README.md → CondTools/HLT/README.md

@@ -2,7 +2,7 @@ Commands to run the workflow multi-IOV:
 
 ## TO UPDATE
 ```
-python run_wf.py -f frontier://PromptProd/CMS_CONDITIONS -i AlCaRecoHLTpaths8e29_1e31_v24_offline -d AlCaRecoHLTpaths_TEST
+run_AlCaRecoTriggerBitsUpdateWorkflow.py -f frontier://PromptProd/CMS_CONDITIONS -i AlCaRecoHLTpaths8e29_1e31_v24_offline -d AlCaRecoHLTpaths_TEST
 ```
 
 will create an update sqlite file called `AlCaRecoHLTpaths_TEST.db` with an updated tag ` AlCaRecoHLTpaths_TEST` (the same IOV structure will be preserved)
@@ -16,5 +16,5 @@ Options available:
 
 ## TO READ BACK
 ```
-cmsRun AlCaRecoTriggerBitsRcdRead_TEMPL_cfg.py inputDB=sqlite_file:AlCaRecoHLTpaths_TEST.db inputTag=AlCaRecoHLTpaths_TEST
+cmsRun $CMSSW_BASE/src/CondTools/HLT/test/AlCaRecoTriggerBitsRcdRead_TEMPL_cfg.py inputDB=sqlite_file:AlCaRecoHLTpaths_TEST.db inputTag=AlCaRecoHLTpaths_TEST
 ```

CondTools/HLT/src/AlCaRecoTriggerBitsRcdRead.cc → CondTools/HLT/plugins/AlCaRecoTriggerBitsRcdRead.cc

@@ -26,7 +26,6 @@
 // Framework
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
 #include "FWCore/Framework/interface/EventSetup.h"
-#include "FWCore/Framework/interface/ESHandle.h"
 #include "FWCore/Framework/interface/ESWatcher.h"
 #include "FWCore/Framework/interface/one/EDAnalyzer.h"
 #include "FWCore/Framework/interface/MakerMacros.h"
@@ -41,12 +40,14 @@
 class AlCaRecoTriggerBitsRcdRead : public edm::one::EDAnalyzer<edm::one::WatchRuns> {
 public:
   explicit AlCaRecoTriggerBitsRcdRead(const edm::ParameterSet &cfg);
-  ~AlCaRecoTriggerBitsRcdRead() override {}
+  ~AlCaRecoTriggerBitsRcdRead() override = default;
 
   void analyze(const edm::Event &evt, const edm::EventSetup &evtSetup) override {}
   void beginRun(const edm::Run &run, const edm::EventSetup &evtSetup) override;
   void endRun(edm::Run const &, edm::EventSetup const &) override;
 
+  static void fillDescriptions(edm::ConfigurationDescriptions &descriptions);
+
 private:
   // types
   enum OutputType { kText, kTwiki, kPython };  //kHtml};
@@ -56,7 +57,7 @@ class AlCaRecoTriggerBitsRcdRead : public edm::one::EDAnalyzer<edm::one::WatchRu
   void printMap(edm::RunNumber_t firstRun, edm::RunNumber_t lastRun, const AlCaRecoTriggerBits &triggerMap) const;
 
   // members
-  edm::ESGetToken<AlCaRecoTriggerBits, AlCaRecoTriggerBitsRcd> triggerBitsToken_;
+  const edm::ESGetToken<AlCaRecoTriggerBits, AlCaRecoTriggerBitsRcd> triggerBitsToken_;
   const OutputType outputType_;
   edm::ESWatcher<AlCaRecoTriggerBitsRcd> watcher_;
   edm::RunNumber_t firstRun_;
@@ -66,9 +67,15 @@ class AlCaRecoTriggerBitsRcdRead : public edm::one::EDAnalyzer<edm::one::WatchRu
 };
 
 ///////////////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////
+void AlCaRecoTriggerBitsRcdRead::fillDescriptions(edm::ConfigurationDescriptions &descriptions) {
+  edm::ParameterSetDescription desc;
+  desc.setComment("Plugin to read payloads of type AlCaRecoTriggerBits");
+  desc.addUntracked<std::string>("rawFileName", "");
+  desc.addUntracked<std::string>("outputType", "twiki");
+  descriptions.addWithDefaultLabel(desc);
+}
 
+///////////////////////////////////////////////////////////////////////
 AlCaRecoTriggerBitsRcdRead::AlCaRecoTriggerBitsRcdRead(const edm::ParameterSet &cfg)
     : triggerBitsToken_(esConsumes<edm::Transition::BeginRun>()),
       outputType_(this->stringToEnum(cfg.getUntrackedParameter<std::string>("outputType"))),

CondTools/HLT/src/AlCaRecoTriggerBitsRcdUpdate.cc → CondTools/HLT/plugins/AlCaRecoTriggerBitsRcdUpdate.cc

@@ -15,7 +15,6 @@
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
 #include "FWCore/ServiceRegistry/interface/Service.h"
 #include "FWCore/Framework/interface/EventSetup.h"
-#include "FWCore/Framework/interface/ESHandle.h"
 #include "FWCore/Framework/interface/one/EDAnalyzer.h"
 #include "FWCore/Framework/interface/MakerMacros.h"
 #include "FWCore/MessageLogger/interface/MessageLogger.h"
@@ -28,14 +27,13 @@
 // Rcd for reading old one:
 #include "CondFormats/DataRecord/interface/AlCaRecoTriggerBitsRcd.h"
 
-class AlCaRecoTriggerBitsRcdUpdate : public edm::one::EDAnalyzer<edm::one::WatchRuns> {
+class AlCaRecoTriggerBitsRcdUpdate : public edm::one::EDAnalyzer<> {
 public:
   explicit AlCaRecoTriggerBitsRcdUpdate(const edm::ParameterSet &cfg);
-  ~AlCaRecoTriggerBitsRcdUpdate() override {}
+  ~AlCaRecoTriggerBitsRcdUpdate() override = default;
 
   void analyze(const edm::Event &evt, const edm::EventSetup &evtSetup) override;
-  void beginRun(const edm::Run &run, const edm::EventSetup &evtSetup) override {}
-  void endRun(edm::Run const &, edm::EventSetup const &) override {}
+  static void fillDescriptions(edm::ConfigurationDescriptions &descriptions);
 
 private:
   typedef std::map<std::string, std::string> TriggerMap;
@@ -44,7 +42,7 @@ class AlCaRecoTriggerBitsRcdUpdate : public edm::one::EDAnalyzer<edm::one::Watch
   bool addTriggerLists(const std::vector<edm::ParameterSet> &triggerListsAdd, AlCaRecoTriggerBits &bits) const;
   void writeBitsToDB(const AlCaRecoTriggerBits &bitsToWrite) const;
 
-  edm::ESGetToken<AlCaRecoTriggerBits, AlCaRecoTriggerBitsRcd> triggerBitsToken_;
+  const edm::ESGetToken<AlCaRecoTriggerBits, AlCaRecoTriggerBitsRcd> triggerBitsToken_;
   unsigned int nEventCalls_;
   const unsigned int firstRunIOV_;
   const int lastRunIOV_;
@@ -55,18 +53,39 @@ class AlCaRecoTriggerBitsRcdUpdate : public edm::one::EDAnalyzer<edm::one::Watch
 };
 
 ///////////////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////
+void AlCaRecoTriggerBitsRcdUpdate::fillDescriptions(edm::ConfigurationDescriptions &descriptions) {
+  edm::ParameterSetDescription desc;
+  desc.setComment("Plugin to write payloads of type AlCaRecoTriggerBits");
+  desc.add<unsigned int>("firstRunIOV", 1);
+  desc.add<int>("lastRunIOV", -1);
+  desc.add<bool>("startEmpty", true);
+  desc.add<std::vector<std::string>>("listNamesRemove", {});
+
+  edm::ParameterSetDescription desc_triggeListsToAdd;
+  desc_triggeListsToAdd.add<std::string>("listName");
+  desc_triggeListsToAdd.add<std::vector<std::string>>("hltPaths");
+  std::vector<edm::ParameterSet> default_triggerListsToAdd;
+  desc.addVPSet("triggerListsAdd", desc_triggeListsToAdd, default_triggerListsToAdd);
+
+  edm::ParameterSetDescription desc_alcarecoToReplace;
+  desc_alcarecoToReplace.add<std::string>("oldKey");
+  desc_alcarecoToReplace.add<std::string>("newKey");
+  std::vector<edm::ParameterSet> default_alcarecoToReplace;
+  desc.addVPSet("alcarecoToReplace", desc_alcarecoToReplace, default_alcarecoToReplace);
+
+  descriptions.addWithDefaultLabel(desc);
+}
 
+///////////////////////////////////////////////////////////////////////
 AlCaRecoTriggerBitsRcdUpdate::AlCaRecoTriggerBitsRcdUpdate(const edm::ParameterSet &cfg)
     : triggerBitsToken_(esConsumes()),
       nEventCalls_(0),
       firstRunIOV_(cfg.getParameter<unsigned int>("firstRunIOV")),
       lastRunIOV_(cfg.getParameter<int>("lastRunIOV")),
       startEmpty_(cfg.getParameter<bool>("startEmpty")),
-      listNamesRemove_(cfg.getParameter<std::vector<std::string> >("listNamesRemove")),
-      triggerListsAdd_(cfg.getParameter<std::vector<edm::ParameterSet> >("triggerListsAdd")),
-      alcarecoReplace_(cfg.getParameter<std::vector<edm::ParameterSet> >("alcarecoToReplace")) {}
+      listNamesRemove_(cfg.getParameter<std::vector<std::string>>("listNamesRemove")),
+      triggerListsAdd_(cfg.getParameter<std::vector<edm::ParameterSet>>("triggerListsAdd")),
+      alcarecoReplace_(cfg.getParameter<std::vector<edm::ParameterSet>>("alcarecoToReplace")) {}
 
 ///////////////////////////////////////////////////////////////////////
 void AlCaRecoTriggerBitsRcdUpdate::analyze(const edm::Event &evt, const edm::EventSetup &iSetup) {
@@ -122,7 +141,7 @@ bool AlCaRecoTriggerBitsRcdUpdate::removeKeysFromMap(const std::vector<std::stri
 ///////////////////////////////////////////////////////////////////////
 bool AlCaRecoTriggerBitsRcdUpdate::replaceKeysFromMap(const std::vector<edm::ParameterSet> &alcarecoReplace,
                                                       TriggerMap &triggerMap) const {
-  std::vector<std::pair<std::string, std::string> > keyPairs;
+  std::vector<std::pair<std::string, std::string>> keyPairs;
   keyPairs.reserve(alcarecoReplace.size());
 
   for (auto &iSet : alcarecoReplace) {
@@ -155,7 +174,7 @@ bool AlCaRecoTriggerBitsRcdUpdate::addTriggerLists(const std::vector<edm::Parame
   // loop on PSets, each containing the key (filter name) and a vstring with triggers
   for (std::vector<edm::ParameterSet>::const_iterator iSet = triggerListsAdd.begin(); iSet != triggerListsAdd.end();
        ++iSet) {
-    const std::vector<std::string> paths(iSet->getParameter<std::vector<std::string> >("hltPaths"));
+    const std::vector<std::string> paths(iSet->getParameter<std::vector<std::string>>("hltPaths"));
     // We must avoid a map<string,vector<string> > in DB for performance reason,
     // so we have to merge the paths into one string that will be decoded when needed:
     const std::string mergedPaths = bits.compose(paths);

CondTools/HLT/BuildFile.xml → CondTools/HLT/plugins/BuildFile.xml

@@ -1,8 +1,10 @@
-<use name="FWCore/ParameterSet"/>
-<use name="FWCore/ServiceRegistry"/>
-<use name="FWCore/Framework"/>
-<use name="FWCore/MessageLogger"/>
 <use name="CondCore/DBOutputService"/>
-<use name="CondFormats/HLTObjects"/>
 <use name="CondFormats/DataRecord"/>
-<flags EDM_PLUGIN="1"/>
+<use name="CondFormats/HLTObjects"/>
+<use name="FWCore/Framework"/>
+<use name="FWCore/MessageLogger"/>
+<use name="FWCore/ServiceRegistry"/>
+<use name="FWCore/ParameterSet"/>
+<library name="CondToolsHLTPlugins" file="*.cc">
+  <flags EDM_PLUGIN="1"/>
+</library>

CondTools/HLT/scripts/run_AlCaRecoTriggerBitsUpdateWorkflow.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+
+"""
+Example script to test reading from local sqlite db.
+"""
+from __future__ import print_function
+import os
+import sys
+import ast
+import optparse
+import multiprocessing
+import CondCore.Utilities.conddblib as conddb
+
+officialdbs = {
+    # frontier connections
+    'frontier://PromptProd/CMS_CONDITIONS'   :'pro',
+    'frontier://FrontierProd/CMS_CONDITIONS' :'pro',
+    'frontier://FrontierArc/CMS_CONDITIONS'  :'arc',
+    'frontier://FrontierInt/CMS_CONDITIONS'  :'int',
+    'frontier://FrontierPrep/CMS_CONDITIONS' :'dev'
+}
+
+##############################################
+def getCommandOutput(command):
+##############################################
+    """This function executes `command` and returns it output.
+    Arguments:
+    - `command`: Shell command to be invoked by this function.
+    """
+    print(command)
+    child = os.popen(command)
+    data = child.read()
+    err = child.close()
+    if err:
+        raise Exception('%s failed w/ exit code %d' % (command, err))
+    return data
+
+##############################################
+def get_iovs(db, tag):
+##############################################
+    """Retrieve the list of IOVs from `db` for `tag`.
+
+    Arguments:
+    - `db`: database connection string
+    - `tag`: tag of database record
+    """
+
+    ### unfortunately some gymnastics is needed here
+    ### to make use of the conddb library
+    if db in officialdbs.keys():
+        db = officialdbs[db]
+
+    ## allow to use input sqlite files as well
+    db = db.replace("sqlite_file:", "").replace("sqlite:", "")
+
+    con = conddb.connect(url = conddb.make_url(db))
+    session = con.session()
+    IOV = session.get_dbtype(conddb.IOV)
+
+    iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
+    if len(iovs) == 0:
+        print("No IOVs found for tag '"+tag+"' in database '"+db+"'.")
+        sys.exit(1)
+
+    session.close()
+
+    return sorted([int(item[0]) for item in iovs])
+
+##############################################
+def updateBits(blob):
+##############################################
+    if(blob[0]<blob[1]):
+        command = 'conddb --yes --db %s copy %s --destdb %s --from %s --to %s' % (blob[4],
+                                                                                  blob[5],
+                                                                                  blob[5]+"_IOV_"+str(blob[2])+".db",
+                                                                                  str(blob[2]) ,
+                                                                                  str(blob[3]))
+        getCommandOutput(command)
+    else:
+        # last IOV needs special command
+        command = 'conddb --yes --db %s copy %s --destdb %s --from %s' % (blob[4],
+                                                                          blob[5],
+                                                                          blob[5]+"_IOV_"+str(blob[2])+".db",
+                                                                          str(blob[2]))
+        getCommandOutput(command)
+
+    # update the trigger bits
+    cmsRunCommand='cmsRun $CMSSW_BASE/src/CondTools/HLT/test/AlCaRecoTriggerBitsRcdUpdate_TEMPL_cfg.py \
+    inputDB=%s inputTag=%s outputDB=%s outputTag=%s firstRun=%s' % ("sqlite_file:"+blob[5]+"_IOV_"+str(blob[2])+".db",
+                                                                    blob[5],
+                                                                    "sqlite_file:"+blob[5]+"_IOV_"+str(blob[2])+"_updated.db",
+                                                                    blob[6],
+                                                                    str(blob[2]))
+    getCommandOutput(cmsRunCommand)
+
+##############################################
+def main():
+##############################################
+
+    defaultDB     = 'sqlite_file:mySqlite.db'
+    defaultInTag  = 'myInTag'
+    defaultOutTag = 'myOutTag'
+    defaultProc   = 20
+
+    parser = optparse.OptionParser(usage = 'Usage: %prog [options] <file> [<file> ...]\n')
+    parser.add_option('-f', '--inputDB',
+                      dest = 'inputDB',
+                      default = defaultDB,
+                      help = 'file to inspect')
+    parser.add_option('-i', '--inputTag',
+                      dest = 'InputTag',
+                      default = defaultInTag,
+                      help = 'tag to be inspected')
+    parser.add_option('-d', '--destTag',
+                      dest = 'destTag',
+                      default = defaultOutTag,
+                      help = 'tag to be written')
+    parser.add_option('-p', '--processes',
+                      dest = 'nproc',
+                      default = defaultProc,
+                      help = 'multiprocesses to run')
+    parser.add_option("-C", '--clean',
+                      dest="doTheCleaning",
+                      action="store_true",
+                      default = True,
+                      help = 'if true remove the transient files')
+
+    (options, arguments) = parser.parse_args()
+
+    db_url = options.inputDB
+    if db_url in officialdbs.keys():
+        db_url = officialdbs[db_url]
+
+    ## allow to use input sqlite files as well
+    db_url = db_url.replace("sqlite_file:", "").replace("sqlite:", "")
+
+    sinces = get_iovs(options.inputDB,options.InputTag)
+
+    print("List of sinces: %s" % sinces)
+
+    myInputTuple=[]
+
+    for i,since in enumerate(sinces):
+        if(i<len(sinces)-1):
+            #                   0             1        2               3      4                5               6
+            myInputTuple.append((i,len(sinces)-1,sinces[i],sinces[i+1]-1,db_url,options.InputTag,options.destTag))
+        else:
+            myInputTuple.append((i,len(sinces)-1,sinces[i],-1,db_url,options.InputTag,options.destTag))
+
+    pool = multiprocessing.Pool(processes=options.nproc)  # start nproc worker processes
+    count = pool.map(updateBits,myInputTuple)
+
+    # merge the output
+    for i,since in enumerate(sinces):
+        mergeCommand='conddb --yes --db %s copy %s --destdb %s --from %s' % (options.InputTag+"_IOV_"+str(sinces[i])+"_updated.db",
+                                                                             options.destTag,
+                                                                             options.destTag+".db",
+                                                                             str(sinces[i]))
+        getCommandOutput(mergeCommand)
+
+    # clean the house (after all is done)
+    if(options.doTheCleaning):
+        cleanCommand = 'rm -fr *updated*.db *IOV_*.db'
+        getCommandOutput(cleanCommand)
+    else:
+        print("======> keeping the transient files")
+
+if __name__ == "__main__":        
+     main()