From 08856dd364102dd6e3f6a660816199c06defa3cc Mon Sep 17 00:00:00 2001 From: Carl Vuosalo Date: Wed, 11 Nov 2020 01:22:52 +0100 Subject: [PATCH 1/3] Add DD4hep Run 3 workflows 11624.911 and 11642.91 --- Configuration/PyReleaseValidation/python/relval_2017.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Configuration/PyReleaseValidation/python/relval_2017.py b/Configuration/PyReleaseValidation/python/relval_2017.py index a1586077d3e71..d7218ad988efd 100644 --- a/Configuration/PyReleaseValidation/python/relval_2017.py +++ b/Configuration/PyReleaseValidation/python/relval_2017.py @@ -51,6 +51,7 @@ 10824.501,10824.502, # 10824.503,10824.504, 10824.511,10824.512, # 10824.513,10824.514, 10824.521,10824.522, # 10824.523,10824.524, + 11624.911, 11642.911, 11650.0,11634.0,11646.0,11640.0,11834.0,11834.99,11846.0,12024.0, 11634.1,11634.5,11634.7, 11650.501,11650.502, # 11650.503,11650.504, From 5986c00325289980f63b8419a6d7e6b064de8e53 Mon Sep 17 00:00:00 2001 From: Carl Vuosalo Date: Thu, 12 Nov 2020 18:50:48 +0100 Subject: [PATCH 2/3] Added 11624.911 and 11642.911 --- Configuration/PyReleaseValidation/scripts/runTheMatrix.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Configuration/PyReleaseValidation/scripts/runTheMatrix.py b/Configuration/PyReleaseValidation/scripts/runTheMatrix.py index a9dd344863d14..6b559f4a1b955 100755 --- a/Configuration/PyReleaseValidation/scripts/runTheMatrix.py +++ b/Configuration/PyReleaseValidation/scripts/runTheMatrix.py @@ -83,7 +83,9 @@ def runSelected(opt): 10024.0, #2017 ttbar 10224.0, #2017 ttbar PU 10824.0, #2018 ttbar + 11624.911, #2021 DD4hep ttbar 11634.0, #2021 ttbar + 11642.911, #2021 DD4hep Zmm 12434.0, #2023 ttbar 23234.0, #2026D49 ttbar (HLT TDR baseline w/ HGCal v11) 23434.999, #2026D49 ttbar premixing stage1+stage2, PU50 From e6aa3652db773c5f8748a88a1e08cc1e9cd59692 Mon Sep 17 00:00:00 2001 From: Kevin Pedro Date: Fri, 13 Nov 2020 15:50:58 -0600 Subject: [PATCH 3/3] add function to prevent GEN-SIM reuse for special workflows --- .../PyReleaseValidation/python/relval_steps.py | 5 ++++- .../python/upgradeWorkflowComponents.py | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index ddf6606d939fd..b62a16dc92cb8 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -3471,7 +3471,10 @@ def gen2021HiMix(fragment,howMuch): #however, there can be a conflict of beam spots but this is lost in the dataset name #so please be careful s=frag[:-4]+'_'+key - if 'FastSim' not in k and s+'INPUT' not in steps and s in baseDataSetReleaseBetter and defaultDataSets[key] != '': # exclude upgradeKeys without input dataset + # exclude upgradeKeys without input dataset, and special WFs that disable reuse + istep = step+preventReuseKeyword + if 'FastSim' not in k and s+'INPUT' not in steps and s in baseDataSetReleaseBetter and defaultDataSets[key] != '' and \ + (istep not in upgradeStepDict or key not in upgradeStepDict[istep] or upgradeStepDict[istep][key] is not None): steps[k+'INPUT']={'INPUT':InputInfo(dataSet='/RelVal'+info.dataset+'/%s/GEN-SIM'%(baseDataSetReleaseBetter[s],),location='STD')} else: for key in [key for year in upgradeKeys for key in upgradeKeys[year]]: diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index 09c7f84313ddd..109bddaa621c8 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -83,10 +83,12 @@ # setupPU() and setupPU_() operate similarly -> called in relval_steps.py *after* merging PUDataSets w/ regular steps # workflow() adds a concrete workflow to the list based on condition() -> called in relval_upgrade.py # every special workflow gets its own derived class, which must then be added to the global dict upgradeWFs +preventReuseKeyword = 'NOREUSE' class UpgradeWorkflow(object): def __init__(self,steps,PU,suffix,offset): self.steps = steps self.PU = PU + self.allowReuse = True # ensure all PU steps are in normal step list for step in self.PU: @@ -98,23 +100,27 @@ def __init__(self,steps,PU,suffix,offset): self.offset = offset if self.offset < 0.0 or self.offset > 1.0: raise ValueError("Special workflow offset must be between 0.0 and 1.0") - def getStepName(self, step): - stepName = step + self.suffix + def getStepName(self, step, extra=""): + stepName = step + self.suffix + extra return stepName - def getStepNamePU(self, step): - stepNamePU = step + 'PU' + self.suffix + def getStepNamePU(self, step, extra=""): + stepNamePU = step + 'PU' + self.suffix + extra return stepNamePU def init(self, stepDict): for step in self.steps: stepDict[self.getStepName(step)] = {} + if not self.allowReuse: stepDict[self.getStepName(step,preventReuseKeyword)] = {} for step in self.PU: stepDict[self.getStepNamePU(step)] = {} + if not self.allowReuse: stepDict[self.getStepNamePU(step,preventReuseKeyword)] = {} def setup(self, stepDict, k, properties): for step in self.steps: self.setup_(step, self.getStepName(step), stepDict, k, properties) + if not self.allowReuse: self.preventReuse(self.getStepName(step,preventReuseKeyword), stepDict, k) def setupPU(self, stepDict, k, properties): for step in self.PU: self.setupPU_(step, self.getStepNamePU(step), stepDict, k, properties) + if not self.allowReuse: self.preventReuse(self.getStepNamePU(step,preventReuseKeyword), stepDict, k) def setup_(self, step, stepName, stepDict, k, properties): pass def setupPU_(self, step, stepName, stepDict, k, properties): @@ -128,6 +134,9 @@ def workflow_(self, workflows, num, fragment, stepList, key): workflows[num+self.offset] = [ fragmentTmp, stepList ] def condition(self, fragment, stepList, key, hasHarvest): return False + def preventReuse(self, stepName, stepDict, k): + if "Sim" in stepName: + stepDict[stepName][k] = None upgradeWFs = OrderedDict() class UpgradeWorkflow_baseline(UpgradeWorkflow): @@ -872,6 +881,7 @@ def condition(self, fragment, stepList, key, hasHarvest): suffix = '_DD4hep', offset = 0.911, ) +upgradeWFs['DD4hep'].allowReuse = False # check for duplicate offsets offsets = [specialWF.offset for specialType,specialWF in six.iteritems(upgradeWFs)]