Merge pull request #38660 from srimanob/125_ReOrganizeFastSimWFs

Reorganize FastSim Run3 workflow

Configuration/PyReleaseValidation/README.md

@@ -77,7 +77,6 @@ The offsets currently in use are:
 * 0.103: Phase-2 aging, 3000fb-1
 * 0.201: HGCAL special TICL Pattern recognition Workflows: clue3D
 * 0.202: HGCAL special TICL Pattern recognition Workflows: FastJet
-* 0.301: FastSim Run-3
 * 0.302: FastSim Run-3 trackingOnly validation
 * 0.303: FastSim Run-3 MB for mixing
 * 0.9001: Sonic Triton

Configuration/PyReleaseValidation/python/relval_2017.py

@@ -38,7 +38,7 @@
 #        (Patatrack ECAL-only: TTbar - on CPU)
 #        (Patatrack HCAL-only: TTbar - on CPU)
 #        (TTbar 0T, TTbar PU 0T)
-#        (TTbar FastSim)
+#        (TTbar FastSim, TTbar FastSim PU, MinBiasFS for mixing)
 #        (TTbar PU MLPF ecal_deepsc)
 #        (ZEE ecal_deesc)
 #        (TTbar PU prod-like)
@@ -71,7 +71,7 @@
            11634.511,
            11634.521,
            11634.24,11834.24,
-           11634.301,
+           13234.0,13434.0,13240.303,
            11834.13, 11834.19,
            11846.19,
            11834.21,

Configuration/PyReleaseValidation/python/relval_steps.py

@@ -3587,6 +3587,7 @@ def gen2021HiMix(fragment,howMuch):
 defaultDataSets['2018Design']='CMSSW_12_0_0_pre4-113X_upgrade2018_design_v5-v'
 defaultDataSets['2021']='CMSSW_12_0_0_pre4-120X_mcRun3_2021_realistic_v2-v'
 defaultDataSets['2021Design']='CMSSW_12_0_0_pre4-120X_mcRun3_2021_design_v2-v'
+defaultDataSets['2021FS']='CMSSW_12_4_0_pre4-124X_mcRun3_2021_realistic_v1_Run3FastSim_FastSim-v'
 defaultDataSets['2023']='CMSSW_12_0_0_pre4-120X_mcRun3_2023_realistic_v2-v'
 defaultDataSets['2024']='CMSSW_12_0_0_pre4-120X_mcRun3_2024_realistic_v2-v'
 defaultDataSets['2026D49']='CMSSW_12_0_0_pre4-113X_mcRun4_realistic_v7_2026D49noPU-v'
@@ -3619,7 +3620,10 @@ def gen2021HiMix(fragment,howMuch):
     elif '2018' in ds:
         PUDataSets[ds]={'-n':10,'--pileup':'AVE_50_BX_25ns','--pileup_input':'das:/RelValMinBias_13/%s/GEN-SIM'%(name,)}
     elif '2021' in ds:
-      PUDataSets[ds]={'-n':10,'--pileup':'Run3_Flat55To75_PoissonOOTPU','--pileup_input':'das:/RelValMinBias_14TeV/%s/GEN-SIM'%(name,)}
+        if 'FS' not in ds:
+            PUDataSets[ds]={'-n':10,'--pileup':'Run3_Flat55To75_PoissonOOTPU','--pileup_input':'das:/RelValMinBias_14TeV/%s/GEN-SIM'%(name,)}
+        else:
+            PUDataSets[ds]={'-n':10,'--pileup':'Run3_Flat55To75_PoissonOOTPU','--pileup_input':'das:/RelValMinBias_14TeV/%s/GEN-SIM-RECO'%(name,)}
     elif 'postLS2' in ds:
         PUDataSets[ds]={'-n':10,'--pileup':'AVE_50_BX_25ns','--pileup_input':'das:/RelValMinBias_13/%s/GEN-SIM'%(name,)}
     elif '2026' in ds:
@@ -3801,14 +3805,33 @@ def gen2021HiMix(fragment,howMuch):
                                    '--fast':'',
                                    '--geometry' : geom,
                                    '--relval':'27000,3000'}
-
+    
     upgradeStepDict['HARVESTFast'][k]={'-s':'HARVESTING:validationHarvesting',
                                     '--conditions':gt,
                                     '--mc':'',
                                     '--geometry' : geom,
                                     '--scenario' : 'pp'
                                     }
 
+    upgradeStepDict['FastSimRun3'][k]={'-s':'GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,PAT,VALIDATION:@standardValidation,DQM:@standardDQMFS',
+                                       '--fast':'',
+                                       '--era':'Run3_FastSim',
+                                       '--beamspot':beamspot,
+                                       '--conditions':gt,
+                                       '--geometry':geom,
+                                       '--eventcontent':'FEVTDEBUGHLT,MINIAODSIM,DQM',
+                                       '--datatier':'GEN-SIM-DIGI-RECO,MINIAODSIM,DQMIO',
+                                       '--relval':'27000,3000'}
+
+    upgradeStepDict['HARVESTFastRun3'][k]={'-s':'HARVESTING:validationHarvesting',
+                                           '--conditions':gt,
+                                           '--mc':'',
+                                           '--fast':'',
+                                           '--geometry':geom,
+                                           '--scenario':'pp',
+                                           '--filetype':'DQM',
+                                           '--filein':'file:step1_inDQM.root'}
+
     upgradeStepDict['Nano'][k] = {'-s':'NANO,DQM:@nanoAODDQM',
                                       '--conditions':gt,
                                       '--datatier':'NANOAODSIM,DQMIO',
@@ -3869,7 +3892,7 @@ def gen2021HiMix(fragment,howMuch):
                         s=frag[:-4]+'_'+key
                         # exclude upgradeKeys without input dataset, and special WFs that disable reuse
                         istep = step+preventReuseKeyword
-                        if 'FastSim' not in k and 'Run3FS' not in k and s+'INPUT' not in steps and s in baseDataSetReleaseBetter and defaultDataSets[key] != '' and \
+                        if 'FastSim' not in k and s+'INPUT' not in steps and s in baseDataSetReleaseBetter and defaultDataSets[key] != '' and \
                            (istep not in upgradeStepDict or key not in upgradeStepDict[istep] or upgradeStepDict[istep][key] is not None):
                             steps[k+'INPUT']={'INPUT':InputInfo(dataSet='/RelVal'+info.dataset+'/%s/GEN-SIM'%(baseDataSetReleaseBetter[s],),location='STD')}
    else:

Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py

@@ -24,6 +24,8 @@
     '2023PU',
     '2024',
     '2024PU',
+    '2021FS',
+    '2021FSPU',
 ]
 
 upgradeKeys[2026] = [
@@ -183,6 +185,8 @@ def condition(self, fragment, stepList, key, hasHarvest):
         'Nano',
         'MiniAOD',
         'HLT75e33',
+        'FastSimRun3',
+        'HARVESTFastRun3',
     ],
     PU =  [
         'DigiTrigger',
@@ -199,6 +203,8 @@ def condition(self, fragment, stepList, key, hasHarvest):
         'MiniAOD',
         'Nano',
         'HLT75e33',
+        'FastSimRun3',
+        'HARVESTFastRun3',
     ],
     suffix = '',
     offset = 0.0,
@@ -345,7 +351,7 @@ def setup__(self, step, stepName, stepDict, k, properties):
         if 'Reco' in step: stepDict[stepName][k] = merge([self.step3, stepDict[step][k]])
         elif 'HARVEST' in step: stepDict[stepName][k] = merge([{'-s': 'HARVESTING:@trackingOnlyValidation+@pixelTrackingOnlyDQM'}, stepDict[step][k]])
     def condition_(self, fragment, stepList, key, hasHarvest):
-        return '2017' in key or '2018' in key or '2021' in key or '2026' in key
+        return ('2017' in key or '2018' in key or '2021' in key or '2026' in key) and ('FS' not in key)
 upgradeWFs['pixelTrackingOnly'] = UpgradeWorkflow_pixelTrackingOnly(
     steps = [
         'Reco',
@@ -396,7 +402,7 @@ def setup__(self, step, stepName, stepDict, k, properties):
         if 'Reco' in step and stepDict[step][k]['--era']=='Run3':
             stepDict[stepName][k] = merge([{'--era': 'Run3_ckfPixelLessStep'}, stepDict[step][k]])
     def condition_(self, fragment, stepList, key, hasHarvest):
-        return '2021' in key
+        return '2021' in key and 'FS' not in key
 upgradeWFs['trackingRun3CkfPixelLessStep'] = UpgradeWorkflow_trackingRun3CkfPixelLessStep(
     steps = [
         'Reco',
@@ -414,7 +420,7 @@ def setup__(self, step, stepName, stepDict, k, properties):
             stepDict[stepName][k] = merge([{'--era': 'Run3_ckfPixelLessStep'}, self.step3, stepDict[step][k]])
         elif 'HARVEST' in step: stepDict[stepName][k] = merge([{'-s': 'HARVESTING:@trackingOnlyValidation+@trackingOnlyDQM'}, stepDict[step][k]])
     def condition_(self, fragment, stepList, key, hasHarvest):
-        return '2021' in key
+        return '2021' in key and 'FS' not in key
 upgradeWFs['trackingOnlyRun3CkfPixelLessStep'] = UpgradeWorkflow_trackingOnlyRun3CkfPixelLessStep(
     steps = [
         'Reco',
@@ -667,9 +673,9 @@ def condition(self, fragment, stepList, key, hasHarvest):
         # select only a subset of the workflows
         selected = [
             ('2018' in key and fragment == "TTbar_13"),
-            ('2021' in key and fragment == "TTbar_14TeV"),
+            ('2021' in key and fragment == "TTbar_14TeV" and 'FS' not in key),
             ('2018' in key and fragment == "ZMM_13"),
-            ('2021' in key and fragment == "ZMM_14"),
+            ('2021' in key and fragment == "ZMM_14" and 'FS' not in key),
             ('2026D88' in key and fragment == "TTbar_14TeV" and "PixelOnly" in self.suffix)
         ]
         result = any(selected) and hasHarvest
@@ -1456,7 +1462,7 @@ def setupPU_(self, step, stepName, stepDict, k, properties):
         # override '-n' setting from PUDataSets in relval_steps.py
         stepDict[stepName][k] = merge([{'-n':'1'}, stepDict[step][k]])
     def condition(self, fragment, stepList, key, hasHarvest):
-        return (fragment=="TTbar_13" or fragment=="TTbar_14TeV") and ('2017' in key or '2018' in key or '2021' in key)
+        return (fragment=="TTbar_13" or fragment=="TTbar_14TeV") and ('2017' in key or '2018' in key or '2021' in key) and ('FS' not in key)
 upgradeWFs['0T'] = UpgradeWorkflow_0T(
     steps = [
         'GenSim',
@@ -1807,88 +1813,47 @@ def condition(self, fragment, stepList, key, hasHarvest):
     offset = 0.9921,
 )
 
-class UpgradeWorkflow_Run3FS(UpgradeWorkflow):
-    def setup_(self, step, stepName, stepDict, k, properties):
-        if 'GenSim' in step:
-            stepDict[stepName][k] = merge([{'-s':'GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,PAT,VALIDATION:@standardValidation,DQM:@standardDQMFS',
-                                            '--fast':'',
-                                            '--era':'Run3_FastSim',
-                                            '--eventcontent':'FEVTDEBUGHLT,MINIAODSIM,DQM',
-                                            '--datatier':'GEN-SIM-DIGI-RECO,MINIAODSIM,DQMIO',
-                                            '--relval':'27000,3000'}, stepDict[step][k]])
-        if 'Digi' in step or 'RecoNano' in step or 'ALCA' in step:
-            stepDict[stepName][k] = None
-        if 'HARVESTNano' in step:
-            stepDict[stepName][k] = merge([{'-s':'HARVESTING:validationHarvesting',
-                                            '--fast':'',
-                                            '--era':'Run3_FastSim',
-                                            '--filein':'file:step1_inDQM.root'}, stepDict[step][k]])
-    def condition(self, fragment, stepList, key, hasHarvest):
-        return '2021' in key
-upgradeWFs['Run3FS'] = UpgradeWorkflow_Run3FS(
-    steps = [
-        'GenSim',
-        'Digi',
-        'RecoNano',
-        'HARVESTNano',
-        'ALCA'
-    ],
-    PU = [],
-    suffix = '_Run3FS',
-    offset = 0.301,
-)
-
 class UpgradeWorkflow_Run3FStrackingOnly(UpgradeWorkflow):
     def setup_(self, step, stepName, stepDict, k, properties):
-        if 'GenSim' in step:
-            stepDict[stepName][k] = merge([{'-s':'GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid,L1,DIGI2RAW,L1Reco,RECO,PAT,VALIDATION:@trackingOnlyValidation',
-                                            '--fast':'',
-                                            '--era':'Run3_FastSim',
-                                            '--eventcontent':'FEVTDEBUGHLT,MINIAODSIM,DQM',
-                                            '--datatier':'GEN-SIM-DIGI-RECO,MINIAODSIM,DQMIO',
-                                            '--relval':'27000,3000'}, stepDict[step][k]])
-        if 'Digi' in step or 'RecoNano' in step or 'ALCA' in step:
-            stepDict[stepName][k] = None
-        if 'HARVESTNano' in step:
+        if 'HARVESTFastRun3' in step:
             stepDict[stepName][k] = merge([{'-s':'HARVESTING:@trackingOnlyValidation+@trackingOnlyDQM',
                                             '--fast':'',
                                             '--era':'Run3_FastSim',
                                             '--filein':'file:step1_inDQM.root'}, stepDict[step][k]])
+        else:
+            stepDict[stepName][k] = merge([stepDict[step][k]])
     def condition(self, fragment, stepList, key, hasHarvest):
-        return '2021' in key
+        return '2021FS' in key
 upgradeWFs['Run3FStrackingOnly'] = UpgradeWorkflow_Run3FStrackingOnly(
     steps = [
-        'GenSim',
-        'Digi',
-        'RecoNano',
-        'HARVESTNano',
-        'ALCA'
+        'FastSimRun3',
+        'HARVESTFastRun3'
+    ],
+    PU = [
+        'FastSimRun3',
+        'HARVESTFastRun3'
     ],
-    PU = [],
     suffix = '_Run3FSTrackingOnly',
     offset = 0.302,
 )
 
 class UpgradeWorkflow_Run3FSMBMixing(UpgradeWorkflow):
     def setup_(self, step, stepName, stepDict, k, properties):
-        if 'GenSim' in step:
+        if 'FastSimRun3' in step:
             stepDict[stepName][k] = merge([{'-s':'GEN,SIM,RECOBEFMIX',
                                             '--fast':'',
                                             '--era':'Run3_FastSim',
                                             '--eventcontent':'FASTPU',
                                             '--datatier':'GEN-SIM-RECO',
                                             '--relval':'27000,3000'}, stepDict[step][k]])
-        if 'Digi' in step or 'RecoNano' in step or 'ALCA' in step or 'HARVESTNano' in step:
+        else:
             stepDict[stepName][k] = None
     def condition(self, fragment, stepList, key, hasHarvest):
-        return '2021' in key and fragment=="MinBias_14TeV"
+        return '2021FS' in key and fragment=="MinBias_14TeV"
 upgradeWFs['Run3FSMBMixing'] = UpgradeWorkflow_Run3FSMBMixing(
     steps = [
-        'GenSim',
-        'Digi',
-        'RecoNano',
-        'HARVESTNano',
-        'ALCA'
+        'FastSimRun3',
+        'HARVESTFastRun3'
     ],
     PU = [],
     suffix = '_Run3FSMBMixing',
@@ -1897,14 +1862,14 @@ def condition(self, fragment, stepList, key, hasHarvest):
 
 class UpgradeWorkflow_DD4hep(UpgradeWorkflow):
     def setup_(self, step, stepName, stepDict, k, properties):
-        if 'Run3' in stepDict[step][k]['--era']:
+        if 'Run3' in stepDict[step][k]['--era'] and 'Fast' not in stepDict[step][k]['--era']:
             stepDict[stepName][k] = merge([{'--geometry': 'DD4hepExtended2021'}, stepDict[step][k]])
         elif 'Phase2' in stepDict[step][k]['--era']:
             dd4hepGeom="DD4hep"
             dd4hepGeom+=stepDict[step][k]['--geometry']
             stepDict[stepName][k] = merge([{'--geometry' : dd4hepGeom, '--procModifiers': 'dd4hep'}, stepDict[step][k]])
     def condition(self, fragment, stepList, key, hasHarvest):
-        return '2021' in key or '2026' in key
+        return ('2021' in key or '2026' in key) and ('FS' not in key)
 upgradeWFs['DD4hep'] = UpgradeWorkflow_DD4hep(
     steps = [
         'GenSim',
@@ -1930,10 +1895,10 @@ def condition(self, fragment, stepList, key, hasHarvest):
 #Keep it for future use in Phase-2, then delete
 class UpgradeWorkflow_DD4hepDB(UpgradeWorkflow):
     def setup_(self, step, stepName, stepDict, k, properties):
-        if 'Run3' in stepDict[step][k]['--era']:
+        if 'Run3' in stepDict[step][k]['--era'] and 'Fast' not in stepDict[step][k]['--era']:
             stepDict[stepName][k] = merge([{'--conditions': 'auto:phase1_2022_realistic', '--geometry': 'DB:Extended'}, stepDict[step][k]])
     def condition(self, fragment, stepList, key, hasHarvest):
-        return '2021' in key
+        return '2021' in key and 'FS' not in key
 upgradeWFs['DD4hepDB'] = UpgradeWorkflow_DD4hepDB(
     steps = [
         'GenSim',
@@ -1957,14 +1922,14 @@ def condition(self, fragment, stepList, key, hasHarvest):
 
 class UpgradeWorkflow_DDDDB(UpgradeWorkflow):
     def setup_(self, step, stepName, stepDict, k, properties):
-        if 'Run3' in stepDict[step][k]['--era']:
+        if 'Run3' in stepDict[step][k]['--era'] and 'Fast' not in stepDict[step][k]['--era']:
             # retain any other eras
             tmp_eras = stepDict[step][k]['--era'].split(',')
             tmp_eras[tmp_eras.index("Run3")] = 'Run3_DDD'
             tmp_eras = ','.join(tmp_eras)
             stepDict[stepName][k] = merge([{'--conditions': 'auto:phase1_2022_realistic_ddd', '--geometry': 'DB:Extended', '--era': tmp_eras}, stepDict[step][k]])
     def condition(self, fragment, stepList, key, hasHarvest):
-        return '2021' in key
+        return '2021' in key and 'FS' not in key
 upgradeWFs['DDDDB'] = UpgradeWorkflow_DDDDB(
     steps = [
         'GenSim',
@@ -2098,14 +2063,25 @@ def condition(self, fragment, stepList, key, hasHarvest):
         'BeamSpot': 'Run3RoundOptics25ns13TeVLowSigmaZ',
         'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano','ALCA'],
     },
+    '2021FS' : {
+        'Geom' : 'DB:Extended',
+        'GT' : 'auto:phase1_2022_realistic',
+        'HLTmenu': '@relval2022',
+        'Era' : 'Run3_FastSim',
+        'BeamSpot': 'Run3RoundOptics25ns13TeVLowSigmaZ',
+        'ScenToRun' : ['FastSimRun3','HARVESTFastRun3'],
+    },
 }
 
 # standard PU sequences
 for key in list(upgradeProperties[2017].keys()):
     upgradeProperties[2017][key+'PU'] = deepcopy(upgradeProperties[2017][key])
-    upgradeProperties[2017][key+'PU']['ScenToRun'] = ['GenSim','DigiPU'] + \
-                                                     (['RecoNanoPU','HARVESTNanoPU'] if '202' in key else ['RecoFakeHLTPU','HARVESTFakeHLTPU']) + \
-                                                     (['Nano'] if 'Nano' in upgradeProperties[2017][key]['ScenToRun'] else [])
+    if "FS" not in key:
+        upgradeProperties[2017][key+'PU']['ScenToRun'] = ['GenSim','DigiPU'] + \
+                                                         (['RecoNanoPU','HARVESTNanoPU'] if '202' in key else ['RecoFakeHLTPU','HARVESTFakeHLTPU']) + \
+                                                         (['Nano'] if 'Nano' in upgradeProperties[2017][key]['ScenToRun'] else [])
+    else:
+        upgradeProperties[2017][key+'PU']['ScenToRun'] = ['FastSimRun3PU','HARVESTFastRun3PU']
 
 upgradeProperties[2026] = {
     '2026D49' : {

Configuration/PyReleaseValidation/scripts/runTheMatrix.py

@@ -93,9 +93,10 @@ def runSelected(opt):
                      11634.911, #2021 DD4hep ttbar reading geometry from XML
                      11634.914, #2021 DDD ttbar reading geometry from the DB
                      11634.0, #2021 ttbar (switching to DD4hep by default)
-                     11634.301, #2021 ttbar fastsim
                      11634.7, #2021 ttbar mkFit
                      11834.0, #2021 ttbar PU
+                     13234.0, #2021 ttbar fastsim
+                     13434.0, #2021 ttbar PU fastsim
                      12434.0, #2023 ttbar
                      23234.0, #2026D49 ttbar (HLT TDR baseline w/ HGCal v11)
                      28234.0, #2026D60 (exercise HF nose)