Merge pull request #32096 from cvuosalo/addwfs

Add DD4hep Run 3 workflows 11624.911 and 11642.91 to PR/IB tests

Configuration/PyReleaseValidation/python/relval_2017.py

@@ -51,6 +51,7 @@
            10824.501,10824.502, # 10824.503,10824.504,
            10824.511,10824.512, # 10824.513,10824.514,
            10824.521,10824.522, # 10824.523,10824.524,
+           11624.911, 11642.911,
            11650.0,11634.0,11646.0,11640.0,11834.0,11834.99,11846.0,12024.0,
            11634.1,11634.5,11634.7,
            11650.501,11650.502, # 11650.503,11650.504,

Configuration/PyReleaseValidation/python/relval_steps.py

@@ -3471,7 +3471,10 @@ def gen2021HiMix(fragment,howMuch):
                         #however, there can be a conflict of beam spots but this is lost in the dataset name
                         #so please be careful
                         s=frag[:-4]+'_'+key
-                        if 'FastSim' not in k and s+'INPUT' not in steps and s in baseDataSetReleaseBetter and defaultDataSets[key] != '': # exclude upgradeKeys without input dataset
+                        # exclude upgradeKeys without input dataset, and special WFs that disable reuse
+                        istep = step+preventReuseKeyword
+                        if 'FastSim' not in k and s+'INPUT' not in steps and s in baseDataSetReleaseBetter and defaultDataSets[key] != '' and \
+                           (istep not in upgradeStepDict or key not in upgradeStepDict[istep] or upgradeStepDict[istep][key] is not None):
                             steps[k+'INPUT']={'INPUT':InputInfo(dataSet='/RelVal'+info.dataset+'/%s/GEN-SIM'%(baseDataSetReleaseBetter[s],),location='STD')}
    else:
         for key in [key for year in upgradeKeys for key in upgradeKeys[year]]:

Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py

@@ -83,10 +83,12 @@
 # setupPU() and setupPU_() operate similarly -> called in relval_steps.py *after* merging PUDataSets w/ regular steps
 # workflow() adds a concrete workflow to the list based on condition() -> called in relval_upgrade.py
 # every special workflow gets its own derived class, which must then be added to the global dict upgradeWFs
+preventReuseKeyword = 'NOREUSE'
 class UpgradeWorkflow(object):
     def __init__(self,steps,PU,suffix,offset):
         self.steps = steps
         self.PU = PU
+        self.allowReuse = True
 
         # ensure all PU steps are in normal step list
         for step in self.PU:
@@ -98,23 +100,27 @@ def __init__(self,steps,PU,suffix,offset):
         self.offset = offset
         if self.offset < 0.0 or self.offset > 1.0:
             raise ValueError("Special workflow offset must be between 0.0 and 1.0")
-    def getStepName(self, step):
-        stepName = step + self.suffix
+    def getStepName(self, step, extra=""):
+        stepName = step + self.suffix + extra
         return stepName
-    def getStepNamePU(self, step):
-        stepNamePU = step + 'PU' + self.suffix
+    def getStepNamePU(self, step, extra=""):
+        stepNamePU = step + 'PU' + self.suffix + extra
         return stepNamePU
     def init(self, stepDict):
         for step in self.steps:
             stepDict[self.getStepName(step)] = {}
+            if not self.allowReuse: stepDict[self.getStepName(step,preventReuseKeyword)] = {}
         for step in self.PU:
             stepDict[self.getStepNamePU(step)] = {}
+            if not self.allowReuse: stepDict[self.getStepNamePU(step,preventReuseKeyword)] = {}
     def setup(self, stepDict, k, properties):
         for step in self.steps:
             self.setup_(step, self.getStepName(step), stepDict, k, properties)
+            if not self.allowReuse: self.preventReuse(self.getStepName(step,preventReuseKeyword), stepDict, k)
     def setupPU(self, stepDict, k, properties):
         for step in self.PU:
             self.setupPU_(step, self.getStepNamePU(step), stepDict, k, properties)
+            if not self.allowReuse: self.preventReuse(self.getStepNamePU(step,preventReuseKeyword), stepDict, k)
     def setup_(self, step, stepName, stepDict, k, properties):
         pass
     def setupPU_(self, step, stepName, stepDict, k, properties):
@@ -128,6 +134,9 @@ def workflow_(self, workflows, num, fragment, stepList, key):
         workflows[num+self.offset] = [ fragmentTmp, stepList ]
     def condition(self, fragment, stepList, key, hasHarvest):
         return False
+    def preventReuse(self, stepName, stepDict, k):
+        if "Sim" in stepName:
+            stepDict[stepName][k] = None
 upgradeWFs = OrderedDict()
 
 class UpgradeWorkflow_baseline(UpgradeWorkflow):
@@ -872,6 +881,7 @@ def condition(self, fragment, stepList, key, hasHarvest):
     suffix = '_DD4hep',
     offset = 0.911,
 )
+upgradeWFs['DD4hep'].allowReuse = False
 
 # check for duplicate offsets
 offsets = [specialWF.offset for specialType,specialWF in six.iteritems(upgradeWFs)]

Configuration/PyReleaseValidation/scripts/runTheMatrix.py

@@ -83,7 +83,9 @@ def runSelected(opt):
                      10024.0, #2017 ttbar
                      10224.0, #2017 ttbar PU
                      10824.0, #2018 ttbar
+                     11624.911, #2021 DD4hep ttbar
                      11634.0, #2021 ttbar
+                     11642.911, #2021 DD4hep Zmm
                      12434.0, #2023 ttbar
                      23234.0, #2026D49 ttbar (HLT TDR baseline w/ HGCal v11)
                      23434.999, #2026D49 ttbar premixing stage1+stage2, PU50