Merge branch 'master' into TestPCL_forSiPixelLA

dmwm · Mar 11, 2022 · 9afe564 · 9afe564
2 parents 0d1df57 + 2abca2a
commit 9afe564
Show file tree

Hide file tree

Showing 8 changed files with 69 additions and 53 deletions.
diff --git a/CHANGES b/CHANGES
@@ -1,3 +1,9 @@
+3.0.2 to 3.0.3:
+  - Make Express and Promt memory configurable (#4638)
+  - Adding deployment ID to db and Tier0Config (#4632)
+  - Setting a 46h soft timeout for PromptReco jobs (#4600)
+  - Fix new AlCa datasets for collisions (#4626)
+
 2.2.4 to 3.0.2:
   - Merge pull request #4624 from germanfgv/ProdPilotBeam
   - Production configuration for Pilot Beam test 2021

diff --git a/bin/00_deploy_prod.sh b/bin/00_deploy_prod.sh
@@ -4,9 +4,9 @@ BASE_DIR=/data/tier0
 DEPLOY_DIR=$BASE_DIR/srv/wmagent
 SPEC_DIR=$BASE_DIR/admin/Specs
 
-TIER0_VERSION=3.0.1
+TIER0_VERSION=3.0.3
 TIER0_ARCH=slc7_amd64_gcc630
-DEPLOY_TAG=HG2110b
+DEPLOY_TAG=HG2201e
 
 function echo_header {
 	echo ''
@@ -134,7 +134,7 @@ echo "config.RetryManager.PauseAlgo.default.coolOffTime = {'create': 10, 'job':
 
 # Twiking Rucio configuration
 sed -i "s+config.RucioInjector.containerDiskRuleParams.*+config.RucioInjector.containerDiskRuleParams = {}+" ./config/tier0/config.py
-
+echo "config.RucioInjector.blockRuleParams = {}" >> ./config/tier0/config.py
 #
 # Set output datasets status to VALID in DBS
 #

diff --git a/bin/00_deploy_replay.sh b/bin/00_deploy_replay.sh
@@ -4,9 +4,9 @@ BASE_DIR=/data/tier0
 DEPLOY_DIR=$BASE_DIR/srv/wmagent
 SPEC_DIR=$BASE_DIR/admin/Specs
 
-TIER0_VERSION=3.0.1
+TIER0_VERSION=3.0.3
 TIER0_ARCH=slc7_amd64_gcc630
-DEPLOY_TAG=HG2110b
+DEPLOY_TAG=HG2201e
 
 function echo_header {
     echo ''
@@ -130,7 +130,7 @@ echo "config.RetryManager.PauseAlgo.default.coolOffTime = {'create': 10, 'job':
 # Twiking Rucio configuration
 sed -i "s+config.RucioInjector.listTiersToInject.*+config.RucioInjector.listTiersToInject = ['AOD', 'MINIAOD', 'NANOAOD', 'NANOAODSIM', 'RAW', 'FEVT', 'USER', 'ALCARECO', 'ALCAPROMPT', 'DQMIO','RAW-RECO']+" ./config/tier0/config.py
 sed -i "s+config.RucioInjector.containerDiskRuleParams.*+config.RucioInjector.containerDiskRuleParams = {'lifetime': 7 * 24 * 60 * 60}+" ./config/tier0/config.py
-
+echo "config.RucioInjector.blockRuleParams = {'lifetime': 7 * 24 * 60 * 60}" >> ./config/tier0/config.py
 
 #
 # Set output datasets status to VALID in DBS

diff --git a/bin/00_patches.sh b/bin/00_patches.sh
@@ -16,5 +16,5 @@ DEPLOY_DIR=$BASE_DIR/srv/wmagent
 #wget -nv https://github.com/dmwm/T0/pull/4597.patch -O - | patch -f -d $DEPLOY_DIR/current/apps/t0/lib/python*/site-packages/ -p 3
 
 #Patches on top of 3.0.1
-wget -nv https://github.com/dmwm/WMCore/pull/10801.patch -O - | patch -f -d $DEPLOY_DIR/current/apps/t0/lib/python3*/site-packages/ -p 3
+#wget -nv https://github.com/dmwm/WMCore/pull/10801.patch -O - | patch -f -d $DEPLOY_DIR/current/apps/t0/lib/python3*/site-packages/ -p 3
 wget -nv https://github.com/dmwm/T0/pull/4611.patch -O - | patch -f -d $DEPLOY_DIR/current/apps/t0/lib/python3*/site-packages/ -p 3
diff --git a/etc/ProdOfflineConfiguration.py b/etc/ProdOfflineConfiguration.py
@@ -33,7 +33,7 @@
 setConfigVersion(tier0Config, "replace with real version")
 
 # Set the min run number:
-setInjectMinRun(tier0Config, 345500)
+setInjectMinRun(tier0Config, 347611)
 
 # Set the max run number:
 setInjectMaxRun(tier0Config, 9999999)
@@ -55,7 +55,7 @@
 #  Data type
 #  Processing site (where jobs run)
 #  PhEDEx locations
-setAcquisitionEra(tier0Config, "Commissioning2021")
+setAcquisitionEra(tier0Config, "Commissioning2022")
 setBaseRequestPriority(tier0Config, 250000)
 setBackfill(tier0Config, None)
 setBulkDataType(tier0Config, "data")
@@ -94,7 +94,7 @@
 
 # Defaults for CMSSW version
 defaultCMSSWVersion = {
-    'default': "CMSSW_12_0_3_patch2"
+    'default': "CMSSW_12_2_1_patch1"
 }
 
 # Configure ScramArch
@@ -127,9 +127,9 @@
 }
 
 # Defaults for GlobalTag
-expressGlobalTag = "120X_dataRun3_Express_v2"
-promptrecoGlobalTag = "120X_dataRun3_Prompt_v2"
-alcap0GlobalTag = "120X_dataRun3_Prompt_v2"
+expressGlobalTag = "122X_dataRun3_Express_v3"
+promptrecoGlobalTag = "122X_dataRun3_Prompt_v3"
+alcap0GlobalTag = "122X_dataRun3_Prompt_v3"
 
 # Mandatory for CondDBv2
 globalTagConnect = "frontier://PromptProd/CMS_CONDITIONS"
@@ -167,7 +167,8 @@
     "CMSSW_12_0_2_patch1" : defaultCMSSWVersion['default'],
     "CMSSW_12_0_2_patch2" : defaultCMSSWVersion['default'],
     "CMSSW_12_0_3" : defaultCMSSWVersion['default'],
-    "CMSSW_12_0_3_patch1" : defaultCMSSWVersion['default']
+    "CMSSW_12_0_3_patch1" : defaultCMSSWVersion['default'],
+    "CMSSW_12_2_1" : defaultCMSSWVersion['default']
     }
 
 expressVersionOverride = {
@@ -192,7 +193,8 @@
     "CMSSW_12_0_2_patch1" : defaultCMSSWVersion['default'],
     "CMSSW_12_0_2_patch2" : defaultCMSSWVersion['default'],
     "CMSSW_12_0_3" : defaultCMSSWVersion['default'],
-    "CMSSW_12_0_3_patch1" : defaultCMSSWVersion['default']
+    "CMSSW_12_0_3_patch1" : defaultCMSSWVersion['default'],
+    "CMSSW_12_2_1" : defaultCMSSWVersion['default']
     }
 
 #set default repack settings for bulk streams
@@ -229,6 +231,7 @@
            blockCloseDelay=24 * 3600,
            timePerEvent=5,
            sizePerEvent=1500,
+           maxMemoryperCore=2000,
            scenario=ppScenario)
 
 #############################
@@ -259,6 +262,7 @@
                  blockCloseDelay=1200,
                  timePerEvent=4,
                  sizePerEvent=1700,
+                 maxMemoryperCore=2000,
                  versionOverride=expressVersionOverride)
 
 addExpressConfig(tier0Config, "ExpressCosmics",
@@ -268,7 +272,7 @@
                  write_dqm=True,
                  alca_producers=["SiStripPCLHistos", "SiStripCalZeroBias", "TkAlCosmics0T",
                                  "SiPixelCalZeroBias",
-                                 "PromptCalibProdSiStrip", "PromptCalibProdSiPixel"
+                                 "PromptCalibProdSiStrip"
                                 ],
                  reco_version=defaultCMSSWVersion,
                  multicore=numberOfCores,
@@ -284,6 +288,7 @@
                  blockCloseDelay=1200,
                  timePerEvent=4, #I have to get some stats to set this properly
                  sizePerEvent=1700, #I have to get some stats to set this properly
+                 maxMemoryperCore=2000,
                  versionOverride=expressVersionOverride)
 
 addExpressConfig(tier0Config, "HLTMonitor",
@@ -307,6 +312,7 @@
                  blockCloseDelay=1200,
                  timePerEvent=4, #I have to get some stats to set this properly
                  sizePerEvent=1700, #I have to get some stats to set this properly
+                 maxMemoryperCore=2000,
                  versionOverride=expressVersionOverride)
 
 addExpressConfig(tier0Config, "Calibration",
@@ -329,6 +335,7 @@
                  timePerEvent=4,
                  sizePerEvent=1700,
                  versionOverride=expressVersionOverride,
+                 maxMemoryperCore=2000,
                  archivalNode="T0_CH_CERN_MSS",
                  dataType="data",
                  tape_node="T1_US_FNAL_MSS")
@@ -354,6 +361,7 @@
                  timePerEvent=4,
                  sizePerEvent=1700,
                  versionOverride=expressVersionOverride,
+                 maxMemoryperCore=2000,
                  diskNode="T2_CH_CERN")
 
 addExpressConfig(tier0Config, "ALCALUMIPIXELSEXPRESS",
@@ -377,6 +385,7 @@
                  timePerEvent=4,
                  sizePerEvent=1700,
                  versionOverride=expressVersionOverride,
+                 maxMemoryperCore=2000,
                  archivalNode=None,
                  tapeNode=None,
                  diskNode=None)
@@ -410,6 +419,7 @@
                  blockCloseDelay=1200,
                  timePerEvent=4,
                  sizePerEvent=1700,
+                 maxMemoryperCore=2000,
                  versionOverride=expressVersionOverride)
 
 addExpressConfig(tier0Config, "HIExpressAlignment",
@@ -434,6 +444,7 @@
                  timePerEvent=4,
                  sizePerEvent=1700,
                  versionOverride=expressVersionOverride,
+                 maxMemoryperCore=2000,
                  diskNode="T2_CH_CERN")
 
 ###################################
@@ -660,7 +671,7 @@
                tape_node="T1_UK_RAL_MSS",
                disk_node="T1_UK_RAL_Disk",
                alca_producers=["HcalCalIsoTrkFilter", "HcalCalIsolatedBunchFilter"],
-               dqm_sequences=["@common", "@jetmet", "@hcal"],
+               dqm_sequences=["@common", "@jetmet", "@L1TMon", "@hcal"],
                physics_skims=["JetHTJetPlusHOFilter", "LogError", "LogErrorMonitor"],
                timePerEvent=5.7,
                sizePerEvent=2250,
@@ -676,7 +687,7 @@
                tape_node="T1_DE_KIT_MSS",
                disk_node="T1_DE_KIT_Disk",
                alca_producers=["HcalCalNoise"],
-               dqm_sequences=["@common", "@jetmet", "@hcal"],
+               dqm_sequences=["@common", "@jetmet", "@L1TMon", "@hcal"],
                physics_skims=["EXOMONOPOLE", "HighMET", "LogError", "LogErrorMonitor"],
                scenario=ppScenario)
 
@@ -802,7 +813,7 @@
                raw_to_disk=True,
                write_dqm=True,
                alca_producers=["TkAlMinBias", "SiStripCalMinBias", "HcalCalIsoTrk", "HcalCalIsolatedBunchSelector"],
-               dqm_sequences=["@common", "@hcal"],
+               dqm_sequences=["@common", "@L1TMon", "@hcal"],
                physics_skims=["EcalActivity", "LogError", "LogErrorMonitor"],
                timePerEvent=12,
                sizePerEvent=4000,
@@ -817,7 +828,7 @@
                write_aod=True,
                write_miniaod=True,
                write_reco=False,
-               dqm_sequences=["@common", "@hcal"],
+               dqm_sequences=["@common", "@L1TMon", "@hcal"],
                alca_producers=["HcalCalMinBias"],
                physics_skims=["LogError", "LogErrorMonitor"],
                timePerEvent=4.2,
@@ -958,7 +969,7 @@
                write_dqm=True,
                write_miniaod=True,
                write_aod=True,
-               dqm_sequences=["@common", "@ecal", "@jetmet", "@hcal", "@L1TEgamma"],
+               dqm_sequences=["@common", "@ecal", "@jetmet", "@L1TMon", "@hcal", "@L1TEgamma"],
                alca_producers=["TkAlMinBias"],
                physics_skims=["LogError", "LogErrorMonitor"],
                scenario=ppScenario)
@@ -1025,7 +1036,7 @@
                write_dqm=True,
                tape_node="T1_RU_JINR_MSS",
                disk_node="T1_RU_JINR_Disk",
-               dqm_sequences=["@commonSiStripZeroBias", "@ecal", "@hcal", "@muon", "@jetmet"],
+               dqm_sequences=["@commonSiStripZeroBias", "@ecal", "@L1TMon", "@hcal", "@muon", "@jetmet"],
                timePerEvent=1,
                alca_producers=["SiStripCalZeroBias", "SiStripCalMinBias", "TkAlMinBias", "HcalCalHO", 
                                "HcalCalIterativePhiSym", "HcalCalHBHEMuonFilter", "HcalCalIsoTrkFilter"],
@@ -1098,7 +1109,7 @@
                raw_to_disk=True,
                write_reco=False,
                write_dqm=True,
-               dqm_sequences=["@commonSiStripZeroBias", "@ecal", "@hcal", "@muon", "@jetmet"],
+               dqm_sequences=["@commonSiStripZeroBias", "@ecal", "@L1TMon", "@hcal", "@muon", "@jetmet"],
                alca_producers=["SiStripCalZeroBias", "TkAlMinBias", "LumiPixelsMinBias", "SiStripCalMinBias", "AlCaPCCZeroBiasFromRECO"],
                physics_skims=["LogError", "LogErrorMonitor"],
                timePerEvent=3.5,
@@ -1262,7 +1273,7 @@
     addDataset(tier0Config, dataset,
                do_reco=True,
                write_dqm=True,
-               dqm_sequences=["@common", "@ecal", "@egamma", "@hcal", "@jetmet"],
+               dqm_sequences=["@common", "@ecal", "@egamma", "@L1TMon", "@hcal", "@jetmet"],
                scenario=ppScenario)
 
 DATASETS = ["HeavyFlavor", "SingleTrack"]
@@ -1282,7 +1293,7 @@
     addDataset(tier0Config, dataset,
                do_reco=True,
                write_dqm=True,
-               dqm_sequences=["@commonSiStripZeroBias", "@ecal", "@hcal", "@muon"],
+               dqm_sequences=["@commonSiStripZeroBias", "@ecal", "@L1TMon", "@hcal", "@muon"],
                alca_producers=["SiStripCalZeroBias", "TkAlMinBias", "LumiPixelsMinBias", "SiStripCalMinBias", "AlCaPCCZeroBiasFromRECO"],
                timePerEvent=3.5,
                sizePerEvent=1500,
@@ -1549,7 +1560,7 @@
 addDataset(tier0Config, "PAEGJet1",
            do_reco=True,
            write_dqm=True,
-           dqm_sequences=["@common", "@ecal", "@egamma", "@hcal", "@jetmet"],
+           dqm_sequences=["@common", "@ecal", "@egamma", "@L1TMon", "@hcal", "@jetmet"],
            physics_skims=["PAZEE"],
            scenario=hiScenario)
 
@@ -1616,7 +1627,7 @@
                do_reco=True,
                write_dqm=True,
                alca_producers=["LumiPixelsMinBias"],
-               dqm_sequences=["@commonSiStripZeroBias", "@ecal", "@hcal", "@muon"],
+               dqm_sequences=["@commonSiStripZeroBias", "@ecal", "@L1TMon", "@hcal", "@muon"],
                scenario=hiScenario)
 
 addDataset(tier0Config, "PADoubleMuOpen",

diff --git a/etc/ReplayOfflineConfiguration.py b/etc/ReplayOfflineConfiguration.py
@@ -119,7 +119,7 @@
 hiTestppScenario = "ppEra_Run2_2018"
 
 # Procesing version number replays
-dt = 128
+dt = 212
 defaultProcVersion = dt
 expressProcVersion = dt
 alcarawProcVersion = dt
@@ -161,7 +161,10 @@
     "CMSSW_12_0_1" : defaultCMSSWVersion['default'],
     "CMSSW_12_0_2" : defaultCMSSWVersion['default'],
     "CMSSW_12_0_2_patch1" : defaultCMSSWVersion['default'],
-    "CMSSW_12_0_2_patch2" : defaultCMSSWVersion['default']
+    "CMSSW_12_0_2_patch2" : defaultCMSSWVersion['default'],
+    "CMSSW_12_0_3" : defaultCMSSWVersion['default'],
+    "CMSSW_12_0_3_patch1" : defaultCMSSWVersion['default'],
+    "CMSSW_12_2_1" : defaultCMSSWVersion['default']
     }
 
 expressVersionOverride = {
@@ -181,9 +184,11 @@
     "CMSSW_12_0_0" : defaultCMSSWVersion['default'],
     "CMSSW_12_0_1" : defaultCMSSWVersion['default'],
     "CMSSW_12_0_2" : defaultCMSSWVersion['default'],
-    "CMSSW_12_0_2" : defaultCMSSWVersion['default'],
     "CMSSW_12_0_2_patch1" : defaultCMSSWVersion['default'],
-    "CMSSW_12_0_2_patch2" : defaultCMSSWVersion['default']
+    "CMSSW_12_0_2_patch2" : defaultCMSSWVersion['default'],
+    "CMSSW_12_0_3" : defaultCMSSWVersion['default'],
+    "CMSSW_12_0_3_patch1" : defaultCMSSWVersion['default'],
+    "CMSSW_12_2_1" : defaultCMSSWVersion['default']
     }
 
 #set default repack settings for bulk streams
@@ -220,6 +225,7 @@
            blockCloseDelay=1200,
            timePerEvent=5,
            sizePerEvent=1500,
+           maxMemoryperCore=2000,
            scenario=ppScenario)
 
 #############################
@@ -250,6 +256,7 @@
                  blockCloseDelay=1200,
                  timePerEvent=4,
                  sizePerEvent=1700,
+                 maxMemoryperCore=2000,
                  versionOverride=expressVersionOverride)
 
 #######################

diff --git a/src/python/T0/RunConfig/RunConfigAPI.py b/src/python/T0/RunConfig/RunConfigAPI.py
@@ -616,18 +616,10 @@ def configureRunStream(tier0Config, run, stream, specDirectory, dqmUploadProxy):
             specArguments['TimePerEvent'] = streamConfig.Express.TimePerEvent
             specArguments['SizePerEvent'] = streamConfig.Express.SizePerEvent
 
-            if streamConfig.Express.Scenario == "HeavyIonsRun2":
-                baseMemory = 3000
-                perCoreMemory = 1300
-            else:
-                baseMemory = 3500
-                perCoreMemory = 1050
-
-            specArguments['Memory'] = baseMemory + perCoreMemory
-
+            specArguments['Memory'] = streamConfig.Express.MaxMemoryperCore
             if streamConfig.Express.Multicore:
                 specArguments['Multicore'] = streamConfig.Express.Multicore
-                specArguments['Memory'] += (streamConfig.Express.Multicore - 1) * perCoreMemory
+                specArguments['Memory'] += (streamConfig.Express.Multicore - 1) * streamConfig.Express.MaxMemoryperCore
 
             specArguments['Requestor'] = "Tier0"
             specArguments['RequestName'] = workflowName
@@ -1047,20 +1039,10 @@ def releasePromptReco(tier0Config, specDirectory, dqmUploadProxy):
                 specArguments['TimePerEvent'] = datasetConfig.TimePerEvent
                 specArguments['SizePerEvent'] = datasetConfig.SizePerEvent
 
-                if datasetConfig.Scenario == "HeavyIonsRun2":
-                    baseMemory = 3000
-                    perCoreMemory = 1300
-                else:
-                    baseMemory = 3500
-                    perCoreMemory = 1050
-
-                specArguments['Memory'] = baseMemory + perCoreMemory
-
+                specArguments['Memory'] = datasetConfig.MaxMemoryperCore
                 if datasetConfig.Multicore:
                     specArguments['Multicore'] = datasetConfig.Multicore
-                    specArguments['Memory'] += (datasetConfig.Multicore - 1) * perCoreMemory
-
-                specArguments['Memory'] += len(datasetConfig.PhysicsSkims) * 100
+                    specArguments['Memory'] += (datasetConfig.Multicore - 1) * datasetConfig.MaxMemoryperCore
 
                 specArguments['Requestor'] = "Tier0"
                 specArguments['RequestName'] = workflowName