Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Migration to DD4hep as default for all Run-3 workflows #35761

Merged
merged 11 commits into from
Oct 26, 2021
Merged

Migration to DD4hep as default for all Run-3 workflows #35761

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
97d238c
first commit for dd4hep migration
Oct 21, 2021
2acadd3
update readme
Oct 21, 2021
2a13b76
Update autoCondModifier, Phase2 Era to exclude dd4hep
Oct 21, 2021
35ffe6e
remove RecNan
Oct 21, 2021
d30b618
update hi Run3 workflow
Oct 21, 2021
25b7d98
update Run3 cosmics MC
Oct 21, 2021
71ed565
autoCondModifer to DD4hep
Oct 23, 2021
ee40013
update GT
Oct 25, 2021
82a4a25
revert unexpected change
Oct 25, 2021
c00a854
revert unexpected change in relval_step
Oct 25, 2021
e7329c0
fix comments
Oct 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions Configuration/AlCa/python/autoCond.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
# GlobalTag for MC production with realistic conditions for Phase1 2017 detector
'phase1_2017_realistic' : '121X_mc2017_realistic_v7',
# GlobalTag for MC production with realistic conditions for Phase1 2017 detector, for PP reference run
'phase1_2017_realistic_ppref' : '120X_mc2017_realistic_forppRef5TeV_v2',
'phase1_2017_realistic_ppref' : '121X_mc2017_realistic_forppRef5TeV_v1',
# GlobalTag for MC production (cosmics) with realistic alignment and calibrations for Phase1 2017 detector, Strip tracker in DECO mode
'phase1_2017_cosmics' : '121X_mc2017cosmics_realistic_deco_v7',
# GlobalTag for MC production (cosmics) with realistic alignment and calibrations for Phase1 2017 detector, Strip tracker in PEAK mode
Expand All @@ -66,17 +66,17 @@
# GlobalTag for MC production (cosmics) with realistic conditions for full Phase1 2018 detector, Strip tracker in PEAK mode
'phase1_2018_cosmics_peak' : '121X_upgrade2018cosmics_realistic_peak_v7',
# GlobalTag for MC production with perfectly aligned and calibrated detector for Phase1 2021
'phase1_2021_design' : '121X_mcRun3_2021_design_v12',
'phase1_2021_design' : '121X_mcRun3_2021_design_v13',
# GlobalTag for MC production with realistic conditions for Phase1 2021
'phase1_2021_realistic' : '121X_mcRun3_2021_realistic_v14',
'phase1_2021_realistic' : '121X_mcRun3_2021_realistic_v15',
# GlobalTag for MC production (cosmics) with realistic conditions for Phase1 2021, Strip tracker in DECO mode
'phase1_2021_cosmics' : '121X_mcRun3_2021cosmics_realistic_deco_v14',
'phase1_2021_cosmics' : '121X_mcRun3_2021cosmics_realistic_deco_v15',
# GlobalTag for MC production with realistic conditions for Phase1 2021 detector for Heavy Ion
'phase1_2021_realistic_hi' : '121X_mcRun3_2021_realistic_HI_v14',
'phase1_2021_realistic_hi' : '121X_mcRun3_2021_realistic_HI_v15',
# GlobalTag for MC production with realistic conditions for Phase1 2023
'phase1_2023_realistic' : '121X_mcRun3_2023_realistic_v13',
'phase1_2023_realistic' : '121X_mcRun3_2023_realistic_v14',
# GlobalTag for MC production with realistic conditions for Phase1 2024
'phase1_2024_realistic' : '121X_mcRun3_2024_realistic_v13',
'phase1_2024_realistic' : '121X_mcRun3_2024_realistic_v14',
# GlobalTag for MC production with realistic conditions for Phase2
'phase2_realistic' : '121X_mcRun4_realistic_v7'
}
Expand Down Expand Up @@ -108,9 +108,9 @@
from Configuration.AlCa.autoCondModifiers import autoCondHLTHI
autoCond = autoCondHLTHI(autoCond)

# special GT for 2021 DD4HEP geometry
from Configuration.AlCa.autoCondModifiers import autoCondDD4HEP
autoCond = autoCondDD4HEP(autoCond)
# special GT for 2021 DDD geometry
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor comment, perhaps alca can fix it in a follow-up PR, but at this point is not only 2021, but all Run3 geometries.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @mmusich
I can also fix it together with @tvami comment above, then re-trrigger the test again. But prefer everyone to reivew and comment first.

from Configuration.AlCa.autoCondModifiers import autoCondDDD
autoCond = autoCondDDD(autoCond)

### OLD KEYS ### kept for backward compatibility
# GlobalTag for MC production with perfectly aligned and calibrated detector
Expand Down
72 changes: 36 additions & 36 deletions Configuration/AlCa/python/autoCondModifiers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,43 +31,43 @@ def autoCondHLTHI(autoCond):
autoCond.update(GlobalTagsHLTHI)
return autoCond

def autoCondDD4HEP(autoCond):
def autoCondDDD(autoCond):

GlobalTagsDDHEP = {}
# substitute the DDD geometry tags with DD4HEP ones
CSCRECODIGI_Geometry_dd4hep = ','.join( ['CSCRECODIGI_Geometry_120DD4hepV1' , "CSCRecoDigiParametersRcd", connectionString, "", "2021-09-28 12:00:00.000"] )
CSCRECO_Geometry_dd4hep = ','.join( ['CSCRECO_Geometry_120DD4hepV1' , "CSCRecoGeometryRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
DTRECO_Geometry_dd4hep = ','.join( ['DTRECO_Geometry_120DD4hepV1' , "DTRecoGeometryRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
GEMRECO_Geometry_dd4hep = ','.join( ['GEMRECO_Geometry_120DD4hepV1' , "GEMRecoGeometryRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
XMLFILE_Geometry_dd4hep = ','.join( ['XMLFILE_Geometry_121DD4hepV1_Extended2021_mc', "GeometryFileRcd" , connectionString, "Extended", "2021-09-28 12:00:00.000"] )
HCALParameters_Geometry_dd4hep = ','.join( ['HCALParameters_Geometry_120DD4hepV1' , "HcalParametersRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
TKRECO_Geometry_dd4hep = ','.join( ['TKRECO_Geometry_121DD4hepV1' , "IdealGeometryRecord" , connectionString, "", "2021-09-28 12:00:00.000"] )
CTRECO_Geometry_dd4hep = ','.join( ['CTRECO_Geometry_120DD4hepV1' , "PCaloTowerRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
EBRECO_Geometry_dd4hep = ','.join( ['EBRECO_Geometry_120DD4hepV1' , "PEcalBarrelRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
EERECO_Geometry_dd4hep = ','.join( ['EERECO_Geometry_120DD4hepV1' , "PEcalEndcapRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
EPRECO_Geometry_dd4hep = ','.join( ['EPRECO_Geometry_120DD4hepV1' , "PEcalPreshowerRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
HCALRECO_Geometry_dd4hep = ','.join( ['HCALRECO_Geometry_120DD4hepV1' , "PHcalRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
TKParameters_Geometry_dd4hep = ','.join( ['TKParameters_Geometry_120DD4hepV1' , "PTrackerParametersRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
ZDCRECO_Geometry_dd4hep = ','.join( ['ZDCRECO_Geometry_120DD4hepV1' , "PZdcRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
RPCRECO_Geometry_dd4hep = ','.join( ['RPCRECO_Geometry_120DD4hepV1' , "RPCRecoGeometryRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
GlobalTagsDDD = {}
# substitute the DD4hep geometry tags with DDD ones
CSCRECODIGI_Geometry_ddd = ','.join( ['CSCRECODIGI_Geometry_112YV2' , "CSCRecoDigiParametersRcd", connectionString, "", "2021-09-28 12:00:00.000"] )
CSCRECO_Geometry_ddd = ','.join( ['CSCRECO_Geometry_112YV2' , "CSCRecoGeometryRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
DTRECO_Geometry_ddd = ','.join( ['DTRECO_Geometry_112YV2' , "DTRecoGeometryRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
GEMRECO_Geometry_ddd = ','.join( ['GEMRECO_Geometry_113YV4' , "GEMRecoGeometryRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
XMLFILE_Geometry_ddd = ','.join( ['XMLFILE_Geometry_121YV1_Extended2021_mc', "GeometryFileRcd" , connectionString, "Extended", "2021-09-28 12:00:00.000"] )
HCALParameters_Geometry_ddd = ','.join( ['HCALParameters_Geometry_112YV2' , "HcalParametersRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
TKRECO_Geometry_ddd = ','.join( ['TKRECO_Geometry_120YV2' , "IdealGeometryRecord" , connectionString, "", "2021-09-28 12:00:00.000"] )
CTRECO_Geometry_ddd = ','.join( ['CTRECO_Geometry_112YV2' , "PCaloTowerRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
EBRECO_Geometry_ddd = ','.join( ['EBRECO_Geometry_112YV2' , "PEcalBarrelRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
EERECO_Geometry_ddd = ','.join( ['EERECO_Geometry_112YV2' , "PEcalEndcapRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
EPRECO_Geometry_ddd = ','.join( ['EPRECO_Geometry_112YV2' , "PEcalPreshowerRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
HCALRECO_Geometry_ddd = ','.join( ['HCALRECO_Geometry_112YV2' , "PHcalRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
TKParameters_Geometry_ddd = ','.join( ['TKParameters_Geometry_112YV2' , "PTrackerParametersRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
ZDCRECO_Geometry_ddd = ','.join( ['ZDCRECO_Geometry_112YV2' , "PZdcRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )
RPCRECO_Geometry_ddd = ','.join( ['RPCRECO_Geometry_112YV2' , "RPCRecoGeometryRcd" , connectionString, "", "2021-09-28 12:00:00.000"] )

for key,val in autoCond.items():
if key == 'phase1_2021_realistic': # modification of the DD4HEP relval GT
GlobalTagsDDHEP['phase1_2021_dd4hep'] = (autoCond[key],
CSCRECODIGI_Geometry_dd4hep,
CSCRECO_Geometry_dd4hep,
DTRECO_Geometry_dd4hep,
GEMRECO_Geometry_dd4hep,
XMLFILE_Geometry_dd4hep,
HCALParameters_Geometry_dd4hep,
TKRECO_Geometry_dd4hep,
CTRECO_Geometry_dd4hep,
EBRECO_Geometry_dd4hep,
EERECO_Geometry_dd4hep,
EPRECO_Geometry_dd4hep,
HCALRECO_Geometry_dd4hep,
TKParameters_Geometry_dd4hep,
ZDCRECO_Geometry_dd4hep,
RPCRECO_Geometry_dd4hep)
autoCond.update(GlobalTagsDDHEP)
if 'phase1_202' in key: # modification of the DDD relval GT
GlobalTagsDDD[key+'_ddd'] = (autoCond[key],
CSCRECODIGI_Geometry_ddd,
CSCRECO_Geometry_ddd,
DTRECO_Geometry_ddd,
GEMRECO_Geometry_ddd,
XMLFILE_Geometry_ddd,
HCALParameters_Geometry_ddd,
TKRECO_Geometry_ddd,
CTRECO_Geometry_ddd,
EBRECO_Geometry_ddd,
EERECO_Geometry_ddd,
EPRECO_Geometry_ddd,
HCALRECO_Geometry_ddd,
TKParameters_Geometry_ddd,
ZDCRECO_Geometry_ddd,
RPCRECO_Geometry_ddd)
autoCond.update(GlobalTagsDDD)
return autoCond
3 changes: 2 additions & 1 deletion Configuration/Eras/python/Era_Phase2_cff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,6 @@
from Configuration.Eras.Modifier_phase2_timing_layer_cff import phase2_timing_layer
from Configuration.Eras.Modifier_phase2_trigger_cff import phase2_trigger
from Configuration.Eras.Modifier_ctpps_2021_cff import ctpps_2021
from Configuration.ProcessModifiers.dd4hep_cff import dd4hep

Phase2 = cms.ModifierChain(Run3_noMkFit.copyAndExclude([phase1Pixel,trackingPhase1,ctpps_2021]), phase2_common, phase2_tracker, trackingPhase2PU140, phase2_ecal, phase2_hcal, phase2_hgcal, phase2_muon, phase2_GEM, hcalHardcodeConditions, phase2_timing, phase2_timing_layer, phase2_trigger)
Phase2 = cms.ModifierChain(Run3_noMkFit.copyAndExclude([phase1Pixel,trackingPhase1,ctpps_2021,dd4hep]), phase2_common, phase2_tracker, trackingPhase2PU140, phase2_ecal, phase2_hcal, phase2_hgcal, phase2_muon, phase2_GEM, hcalHardcodeConditions, phase2_timing, phase2_timing_layer, phase2_trigger)
7 changes: 7 additions & 0 deletions Configuration/Eras/python/Era_Run3_DDD_cff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import FWCore.ParameterSet.Config as cms

from Configuration.Eras.Era_Run3_cff import Run3
from Configuration.ProcessModifiers.dd4hep_cff import dd4hep

Run3_DDD = Run3.copyAndExclude([dd4hep])

3 changes: 2 additions & 1 deletion Configuration/Eras/python/Era_Run3_cff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from Configuration.Eras.Modifier_stage2L1Trigger_2021_cff import stage2L1Trigger_2021
from Configuration.Eras.Modifier_ctpps_2018_cff import ctpps_2018
from Configuration.Eras.Modifier_ctpps_2021_cff import ctpps_2021
from Configuration.ProcessModifiers.dd4hep_cff import dd4hep

Run3 = cms.ModifierChain(Run2_2018.copyAndExclude([run2_GEM_2017, ctpps_2018]), run3_common, run3_GEM, run3_HB, run3_HFSL, stage2L1Trigger_2021, ctpps_2021)
Run3 = cms.ModifierChain(Run2_2018.copyAndExclude([run2_GEM_2017, ctpps_2018]), run3_common, run3_GEM, run3_HB, run3_HFSL, stage2L1Trigger_2021, ctpps_2021, dd4hep)

3 changes: 2 additions & 1 deletion Configuration/Eras/python/Era_Run3_dd4hep_cff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@
from Configuration.Eras.Era_Run3_cff import Run3
from Configuration.ProcessModifiers.dd4hep_cff import dd4hep

Run3_dd4hep = cms.ModifierChain(Run3, dd4hep)
#Run3 includes DD4hep as default starting from 12_1_0_pre6
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
#Run3 includes DD4hep as default starting from 12_1_0_pre6
#Run3 includes DD4hep as default starting from 12_1_0_pre5

Run3_dd4hep = Run3

1 change: 1 addition & 0 deletions Configuration/PyReleaseValidation/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ The offsets currently in use are:
* 0.9921: Production-like premixing stage1+stage2
* 0.911: DD4Hep reading geometry from XML
* 0.912: DD4Hep reading geometry from the DB
* 0.914: DDD DB
* 0.101: Phase-2 aging, 1000fb-1
* 0.103: Phase-2 aging, 3000fb-1
* 0.9001: Sonic Triton
4 changes: 2 additions & 2 deletions Configuration/PyReleaseValidation/python/relval_2017.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
# (Patatrack ECAL-only: TTbar - on CPU)
# (Patatrack HCAL-only: TTbar - on CPU)
# 2021 (DD4HEP XML: TTbar, ZMM)
# (DD4HEP DB: TTbar)
# (DDD DB: TTbar, ZMM)
# (ele guns 10, 35, 1000; pho guns 10, 35; mu guns 1, 10, 100, 1000, QCD 3TeV, QCD Flat)
# (ZMM, TTbar, ZEE, MinBias, TTbar PU, TTbar PU premix, ZEE PU, TTbar design, GluGluTo2Jets, GluGluTo2Jets PU)
# (TTbar trackingOnly, pixelTrackingOnly, trackingMkFit, trackdnn)
Expand Down Expand Up @@ -56,7 +56,7 @@
10824.511,
10824.521,
11634.911, 11650.911,
11634.912,
11634.914, 11650.914,
11601.0,11602.0,11603.0,11604.0,11605.0,11606.0,11607.0,11608.0,11609.0,11630.0,11643.0,
11650.0,11634.0,11646.0,11640.0,11834.0,11834.99,11846.0,12034.0,11725.0,11925.0,
11634.1,11634.5,11634.7,11634.91,
Expand Down
31 changes: 29 additions & 2 deletions Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1157,7 +1157,7 @@ def condition(self, fragment, stepList, key, hasHarvest):
class UpgradeWorkflow_DD4hep(UpgradeWorkflow):
def setup_(self, step, stepName, stepDict, k, properties):
if 'Run3' in stepDict[step][k]['--era']:
stepDict[stepName][k] = merge([{'--geometry': 'DD4hepExtended2021', '--procModifiers': 'dd4hep'}, stepDict[step][k]])
stepDict[stepName][k] = merge([{'--geometry': 'DD4hepExtended2021'}, stepDict[step][k]])
elif 'Phase2' in stepDict[step][k]['--era']:
dd4hepGeom="DD4hep"
dd4hepGeom+=stepDict[step][k]['--geometry']
Expand All @@ -1183,10 +1183,12 @@ def condition(self, fragment, stepList, key, hasHarvest):
)
upgradeWFs['DD4hep'].allowReuse = False

#This workflow is now obsolete, it becomes default for Run-3.
#Keep it for future use in Phase-2, then delete
class UpgradeWorkflow_DD4hepDB(UpgradeWorkflow):
def setup_(self, step, stepName, stepDict, k, properties):
if 'Run3' in stepDict[step][k]['--era']:
stepDict[stepName][k] = merge([{'--conditions': 'auto:phase1_2021_dd4hep', '--geometry': 'DB:Extended', '--procModifiers': 'dd4hep'}, stepDict[step][k]])
stepDict[stepName][k] = merge([{'--conditions': 'auto:phase1_2021_realistic', '--geometry': 'DB:Extended'}, stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
return '2021' in key
upgradeWFs['DD4hepDB'] = UpgradeWorkflow_DD4hepDB(
Expand All @@ -1208,6 +1210,31 @@ def condition(self, fragment, stepList, key, hasHarvest):
)
upgradeWFs['DD4hepDB'].allowReuse = False

class UpgradeWorkflow_DDDDB(UpgradeWorkflow):
def setup_(self, step, stepName, stepDict, k, properties):
if 'Run3' in stepDict[step][k]['--era']:
stepDict[stepName][k] = merge([{'--conditions': 'auto:phase1_2021_realistic_ddd', '--geometry': 'DB:Extended', '--era': 'Run3_DDD'}, stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
return '2021' in key
upgradeWFs['DDDDB'] = UpgradeWorkflow_DDDDB(
steps = [
'GenSim',
'GenSimHLBeamSpot',
'GenSimHLBeamSpot14',
'Digi',
'DigiTrigger',
'Reco',
'RecoGlobal',
'HARVEST',
'HARVESTGlobal',
'ALCA',
],
PU = [],
suffix = '_DDDDB',
offset = 0.914,
)
upgradeWFs['DDDDB'].allowReuse = False

class UpgradeWorkflow_SonicTriton(UpgradeWorkflow):
def setup_(self, step, stepName, stepDict, k, properties):
stepDict[stepName][k] = merge([{'--procModifiers': 'allSonicTriton'}, stepDict[step][k]])
Expand Down
5 changes: 3 additions & 2 deletions Configuration/PyReleaseValidation/scripts/runTheMatrix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,14 +89,15 @@ def runSelected(opt):
10824.0, #2018 ttbar
2018.1, #2018 ttbar fastsim
11634.911, #2021 DD4hep ttbar reading geometry from XML
11634.912, #2021 DD4hep ttbar reading geometry from the DB
11634.0, #2021 ttbar
11634.914, #2021 DDD ttbar reading geometry from the DB
11634.0, #2021 ttbar (switching to DD4hep by default)
11634.7, #2021 ttbar mkFit
12434.0, #2023 ttbar
23234.0, #2026D49 ttbar (HLT TDR baseline w/ HGCal v11)
28234.0, #2026D60 (exercise HF nose)
34634.0, #2026D76 ttbar (2021 new baseline)
34834.999, #2026D76 ttbar premixing stage1+stage2, PU50
38634.0, #2026D86 ttbar
25202.0, #2016 ttbar UP15 PU
250202.181, #2018 ttbar stage1 + stage2 premix
],
Expand Down
1 change: 1 addition & 0 deletions Configuration/StandardSequences/python/Eras.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def __init__(self):
'Run3_noMkFit',
'Run3_pp_on_PbPb',
'Run3_dd4hep',
'Run3_DDD',
'Phase2',
'Phase2C9',
'Phase2C10',
Expand Down