APE tool: Trend plots, some minor fixes

Alignment/APEEstimation/test/autoSubmitter/autoSubmitterTemplates.py

@@ -6,7 +6,7 @@
 
 eval `scramv1 runtime -csh`
 
-source /afs/cern.ch/cms/caf/setup.csh
+#~ source /afs/cern.ch/cms/caf/setup.csh
 cd $curDir
 
 xrdcp {inputFile} reco.root
@@ -25,11 +25,27 @@
 request_memory = 2000M
 request_disk = 400M
 batch_name = {jobName}
-+JobFlavour = "longlunch"
++JobFlavour = "workday"
 Queue Arguments from (
 {arguments})
 """
 
+# Use this one only if connected to lxplus-t0.cern.ch
+#~ condorSubTemplate="""
+#~ Executable = {jobFile}
+#~ Universe = vanilla
+#~ Output = {outputFile}
+#~ Error  = {errorFile}
+#~ Log  = {logFile}
+#~ request_memory = 2000M
+#~ request_disk = 400M
+#~ batch_name = {jobName}
+#~ +JobFlavour = "workday"
+#~ +AccountingGroup = "group_u_CMS.CAF.ALCA" 
+#~ Queue Arguments from (
+#~ {arguments})
+#~ """
+
 condorArgumentTemplate="""{fileNumber} {inputFile}
 """
 

Alignment/APEEstimation/test/autoSubmitter/config.ini

@@ -15,7 +15,7 @@ isMC=False
 # in apeEstimation_cfg.py. If it is not defined or None, it will not overwrite
 # anything defined in apeEstimation_cfg.py
 globalTag=None
-# name as defined in apeEstimation_cfg.py
+# name as defined in apeEstimation_cfg.py, optional if alignment is loaded by condition
 alignmentName=alignmentObjectName
 # name of baseline folder. Only used if isDesign is False, elsewise the 
 # name of the APE measurement is used, Design by default
@@ -26,6 +26,10 @@ isDesign=False
 # it will override the one defined by alignmentName
 condition TrackerAlignmentRcd=sqlite_file:/afs/asdf.db Alignments
 condition TrackerSurfaceDeformationRcd=sqlite_file:/afs/asdf.db Deformations
+# another way to load these is:
+condition Alignments+Deformations:sqlite_file:/afs/asdf.db
+# in this case, the Rcd name has to be guessed by the autoSubmitter, 
+# so functionality is limited
 
 # define measurements like this
 [measurements]
@@ -50,6 +54,8 @@ exampleName: exampleDataset alignmentObject
 wait label:exampleName
 # load APE from these files (in labels ~ is replaced with whitespaces)
 load label~with~empty~spaces:/path/to/allData_iterationApe.root
+# optional attributes
+load label~with~empty~spaces2:/path/to/allData_iterationApe.root color=ROOT.kRed+2 marker=0
 # output path. If not defined, store in hists/Name
 outPath=/asdf/
 # title to the plot, optional

Alignment/APEEstimation/test/autoSubmitter/helpers.py

@@ -0,0 +1,135 @@
+from __future__ import print_function
+import re
+import os
+import errno
+shortcuts = {}
+
+# regex matching on key, replacement of groups on value
+# implement any other shortcuts that you want to use
+#sources
+shortcuts["mp([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm/alignments_MP.db"
+shortcuts["mp([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm{1}/alignments_MP.db"
+shortcuts["sm([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/sm{0}/alignments_iter{1}.db"
+shortcuts["um([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm/um{0}.db"
+shortcuts["um([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm{1}/um{0}.db"
+shortcuts["hp([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/hp{0}/alignments_iter{1}.db"
+shortcuts["prod"] = "frontier://FrontierProd/CMS_CONDITIONS"
+
+# Exact numbers don't really matter, but it is important that each one has a unique
+# number, so that states are distinguishable
+STATE_NONE = -1
+STATE_ITERATION_START=0
+STATE_BJOBS_WAITING=1
+STATE_BJOBS_DONE=2
+STATE_BJOBS_FAILED=12
+STATE_MERGE_WAITING=3
+STATE_MERGE_DONE=4
+STATE_MERGE_FAILED=14
+STATE_SUMMARY_WAITING=5
+STATE_SUMMARY_DONE=6
+STATE_SUMMARY_FAILED=16
+STATE_LOCAL_WAITING=7
+STATE_LOCAL_DONE=8
+STATE_LOCAL_FAILED=18
+STATE_FINISHED=9
+
+status_map = {}
+status_map[STATE_NONE] = "none"
+status_map[STATE_ITERATION_START] = "starting iteration"
+status_map[STATE_BJOBS_WAITING] = "waiting for jobs"
+status_map[STATE_BJOBS_DONE] = "jobs finished"
+status_map[STATE_BJOBS_FAILED] = "jobs failed"
+status_map[STATE_MERGE_WAITING] = "waiting for merging"
+status_map[STATE_MERGE_DONE] = "merging done"
+status_map[STATE_MERGE_FAILED] = "merging failed"
+status_map[STATE_SUMMARY_WAITING] = "waiting for APE determination"
+status_map[STATE_SUMMARY_DONE] = "APE determination done"
+status_map[STATE_SUMMARY_FAILED] = "APE determination failed"
+status_map[STATE_LOCAL_WAITING] = "waiting for APE saving"
+status_map[STATE_LOCAL_DONE] = "APE saving done"
+status_map[STATE_LOCAL_FAILED] = "APE saving failed"
+status_map[STATE_FINISHED] = "finished"
+
+records = {}
+records["Alignments"] = "TrackerAlignmentRcd"
+records["TrackerAlignment"] = "TrackerAlignmentRcd"
+records["Deformations"] = "TrackerSurfaceDeformationRcd"
+records["TrackerSurfaceDeformations"] = "TrackerSurfaceDeformationRcd"
+records["SiPixelTemplateDBObject"] = "SiPixelTemplateDBObjectRcd"
+records["BeamSpotObjects"] = "BeamSpotObjectsRcd"
+
+
+def ensurePathExists(path):
+    try:
+        os.makedirs(path)
+    except OSError as exception:
+        if exception.errno != errno.EEXIST:
+            raise
+
+
+def replaceAllRanges(string):
+    if "[" in string and "]" in string:
+        strings = []
+        posS = string.find("[")
+        posE = string.find("]")
+        nums = string[posS+1:posE].split(",")
+        expression = string[posS:posE+1]
+
+        nums = string[string.find("[")+1:string.find("]")]
+        for interval in nums.split(","):
+            interval = interval.strip()
+            if "-" in interval:
+                lowNum = int(interval.split("-")[0])
+                upNum = int(interval.split("-")[1])
+                for i in range(lowNum, upNum+1):
+                    newstring = string[0:posS]+str(i)+string[posE+1:]
+                    newstring = replaceAllRanges(newstring)
+                    strings += newstring
+            else:
+                newstring = string[0:posS]+interval+string[posE+1:]
+                newstring = replaceAllRanges(newstring)
+                strings += newstring
+        return strings
+    else:
+        return [string,]
+
+
+def replaceShortcuts(toScan):
+    global shortcuts
+    for key, value in shortcuts.items():
+        match = re.search(key, toScan)
+        if match and match.group(0) == toScan:
+            return value.format(*match.groups())
+    # no match
+    return toScan
+
+def loadConditions(dictionary):
+    hasAlignmentCondition = False
+    conditions = []
+    for key, value in dictionary.items():
+        if key.startswith("condition"):
+            if len(value.split(" ")) > 1: # structure is "condition rcd:source tag"
+                record = key.split(" ")[1]
+                connect, tag = value.split(" ")
+                if record == "TrackerAlignmentRcd":
+                    hasAlignmentCondition = True
+                conditions.append({"record":record, "connect":replaceShortcuts(connect), "tag":tag})
+            else:
+                # structure is "condition tag:source", so we have to guess rcd from the tag. might also be "condition tag1+tag2+...+tagN:source"
+                global records
+                connect = value.strip()
+                tags = key.split(" ")[1]
+                for tag in tags.split("+"):
+                    foundTag = False
+                    for possibleTag, possibleRcd in records.items():
+                        if tag.startswith(possibleTag):
+                            conditions.append({"record":possibleRcd, "connect":replaceShortcuts(connect), "tag":tag})
+                            if possibleRcd == "TrackerAlignmentRcd":
+                                hasAlignmentCondition = True
+                            foundTag = True
+                            break
+                    if not foundTag:
+                        print("Did not find a record corresponding to {} tag".format(tag))
+                        exit()
+
+    return conditions, hasAlignmentCondition 

Alignment/APEEstimation/test/batch/startSkim.py

@@ -121,7 +121,7 @@ def main(argv):
                           help="Name of sample as defined in skimProducer_cfg.py. Multiple inputs possible")
     parser.add_argument("-c", "--consecutive", action="store_true", dest="consecutive", default=False,
                           help="Do consecutive instead of parallel skims")
-    parser.add_argument("-n", "--ncores", action="store", dest="ncores", default=-1, type="int",
+    parser.add_argument("-n", "--ncores", action="store", dest="ncores", default=-1, type=int,
                           help="Set maximum number of parallel skims to run")
 
     args = parser.parse_args()
@@ -145,6 +145,8 @@ def main(argv):
             doSkim(sample) 
     else:
         try:
+            # In a later PR, this should be migrated to condor to avoid 
+            # overloading lxplus and transfer load to lxbatch
             pool = mp.Pool(args.ncores)
             pool.map_async(doSkim, args.samples)
             pool.close()

Alignment/APEEstimation/test/cfgTemplate/apeEstimator_cfg.py

@@ -169,93 +169,16 @@
 if options.alignRcd=='fromConditions':
     pass # Alignment is read from the conditions file in this case
 elif options.alignRcd=='design':
-    CondDBAlignment = CondDB.clone(connect = cms.string('frontier://FrontierProd/CMS_CONDITIONS'))
-    process.myTrackerAlignment = cms.ESSource("PoolDBESSource",
-        CondDBAlignment,
-        timetype = cms.string("runnumber"),
-        toGet = cms.VPSet(
-            cms.PSet(
-                record = cms.string('TrackerAlignmentRcd'),
-                tag = cms.string('TrackerAlignment_Upgrade2017_design_v3')
-                )
-            )
-        )
-    process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment")
-
-
-elif options.alignRcd == 'misalTest':
-    CondDBAlignment = CondDB.clone(connect = cms.string('frontier://FrontierProd/CMS_CONDITIONS'))
-    process.myTrackerAlignment = cms.ESSource("PoolDBESSource",
-        CondDBAlignment,
-        timetype = cms.string("runnumber"),
-        toGet = cms.VPSet(
-            cms.PSet(
-                record = cms.string('TrackerAlignmentRcd'),
-                tag = cms.string('TrackerAlignment_Phase1Realignment_CRUZET_2M'),
-            )
-        )
-    )
-    process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment")
-
-elif options.alignRcd == 'mp2705':
-    CondDBAlignment = CondDB.clone(connect = cms.string('sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp2705/jobData/jobm/alignments_MP.db'))
-    process.myTrackerAlignment = cms.ESSource("PoolDBESSource",
-        CondDBAlignment,
-        timetype = cms.string("runnumber"),
-        toGet = cms.VPSet(
-            cms.PSet(
-                record = cms.string('TrackerAlignmentRcd'),
-                tag = cms.string('Alignments'),
-            )
-        )
-    )
-    process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment")
-
-elif options.alignRcd == 'mp2853':
-    CondDBAlignment = CondDB.clone()
-    process.myTrackerAlignment = cms.ESSource("PoolDBESSource",
-        CondDBAlignment,
-        timetype = cms.string("runnumber"),
-        toGet = cms.VPSet(
-            cms.PSet(
-                connect = cms.string('sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp2853/jobData/jobm3/alignments_MP.db'),
-                record = cms.string('TrackerAlignmentRcd'),
-                tag = cms.string('Alignments'),
-            ),
-            #~ cms.PSet(
-                #~ connect=cms.string('frontier://FrontierProd/CMS_CONDITIONS'),
-                #~ record=cms.string('SiPixelTemplateDBObjectRcd'),
-                #~ tag=cms.string('SiPixelTemplateDBObject_38T_TempForAlignmentReReco2018_v3'),
-            #~ )
-        )
-    )
-    process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment")
-
-elif options.alignRcd == 'hp1370':
-    CondDBAlignment = CondDB.clone(connect = cms.string('sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HIP/xiaomeng/CMSSW_7_4_6_patch5/src/Alignment/HIPAlignmentAlgorithm/hp1370/alignments.db'))
-    process.myTrackerAlignment = cms.ESSource("PoolDBESSource",
-        CondDBAlignment,
-        timetype = cms.string("runnumber"),
-        toGet = cms.VPSet(
-            cms.PSet(
-                record = cms.string('TrackerAlignmentRcd'),
-                tag = cms.string('Alignments'),
-            )
-        )
-    )
-    process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment")
-
-
-
+    pass
 elif options.alignRcd == 'globalTag':
-  pass
+    pass
 elif options.alignRcd == 'useStartGlobalTagForAllConditions':
-  pass
+    pass
 elif options.alignRcd == '':
-  pass
+    pass
 else:
-  print('ERROR --- incorrect alignment: ', options.alignRcd)
-  exit(8888)
+    print('ERROR --- incorrect alignment: ', options.alignRcd)
+    exit(8888)
 
 ## APE
 if options.iterNumber!=0:

Alignment/APEEstimation/test/plottingTools/drawIterations.py

@@ -1,17 +1,17 @@
 # Implementation to draw all iterations of an APE measurement
 # to check convergence
-
 from iterationsPlotter import *
 from granularity import *
+import os
 
 try:
     base = os.environ['CMSSW_BASE']+"/src/Alignment/APEEstimation"
 except KeyError:
     base = ""
 
 plot = IterationsPlotter()
-plot.setOutputPath(base+"/workingArea/")
-plot.setInputFile(base+"/workingArea/iter14/allData_iterationApe.root")
-plot.setTitle("")
+plot.setOutputPath(base+"/hists/iterations/")
+plot.setInputFile(base+"/src/Alignment/APEEstimation/hists/workingArea/iter14/allData_iterationApe.root")
+plot.setTitle("Title")
 plot.setGranularity(standardGranularity)
 plot.draw()

Alignment/APEEstimation/test/plottingTools/drawResults.py

@@ -1,21 +1,18 @@
 # Implementation to draw all iterations of an APE measurement
 # to check convergence
-
-import ROOT
 from resultPlotter import *
-from systematics import *
+from systematicErrors import *
 from granularity import *
-
+import os
 try:
     base = os.environ['CMSSW_BASE']+"/src/Alignment/APEEstimation"
 except KeyError:
     base = ""
 
 plot = ResultPlotter()
-plot.setOutputPath(base+"/workingArea/") 
-# internal name (used for example when adding systematic errors), path to file, label, color (optional)
-plot.addInputFile("placeholder1", base+"/workingArea/iter14/allData_iterationApe.root", "measurement A")
-plot.addInputFile("placeholder2", base+"/workingArea2/iter14/allData_iterationApe.root", "measurement B", ROOT.kRed)
-plot.setTitle("")
+plot.setOutputPath(base+"/hists/workingArea/") 
+# label(also used as name when adding systematic errors), inputFile, color (optional, automatic by default), 
+# marker (optional, 20 by default, 0 is line), hitNumbers (optional, file number of hits in each sector, allData.root)
+plot.addInputFile("label", "base/hists/workingArea/iter14/allData_iterationApe.root",  color = ROOT.kGray+2)
 plot.setGranularity(standardGranularity)
 plot.draw()