From a4f64e51e0a64e840da49de68ce2f44ee6eb2ed9 Mon Sep 17 00:00:00 2001 From: German Giraldo Date: Wed, 18 Sep 2024 16:56:17 +0200 Subject: [PATCH 1/4] Fix nThread usage and adds streamer input option --- .../DataProcessing/test/RunExpressProcessing.py | 14 ++++++++++---- Configuration/DataProcessing/test/run_CfgTest_9.sh | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Configuration/DataProcessing/test/RunExpressProcessing.py b/Configuration/DataProcessing/test/RunExpressProcessing.py index 4540096ff4ac4..4b1bee62c892e 100644 --- a/Configuration/DataProcessing/test/RunExpressProcessing.py +++ b/Configuration/DataProcessing/test/RunExpressProcessing.py @@ -31,6 +31,7 @@ def __init__(self): self.inputLFN = None self.alcaRecos = None self.nThreads = None + self.dat = False def __call__(self): if self.scenario == None: @@ -90,9 +91,11 @@ def __call__(self): if self.alcaRecos: kwds['skims'] = self.alcaRecos - if self.nThreads: - kwds['nThreads'] = self.nThreads + kwds['nThreads'] = int(self.nThreads) + + if self.dat: + kwds['inputSource'] = 'DAT' process = scenario.expressProcessing(self.globalTag, **kwds) @@ -135,7 +138,7 @@ def __call__(self): if __name__ == '__main__': valid = ["scenario=", "raw", "reco", "fevt", "dqm", "dqmio", "no-output", - "global-tag=", "lfn=", 'alcarecos=', "nThreads="] + "global-tag=", "lfn=", "dat", 'alcarecos=', "nThreads="] usage = \ """ RunExpressProcessing.py @@ -149,6 +152,7 @@ def __call__(self): --no-output (create config with no output, overrides other settings) --global-tag=GlobalTag --lfn=/store/input/lfn + --dat (to enable streamer files as input) --alcarecos=plus_seprated_list --nThreads=Number_of_cores_or_Threads_used @@ -156,7 +160,7 @@ def __call__(self): python RunExpressProcessing.py --scenario cosmics --global-tag GLOBALTAG --lfn /store/whatever --fevt --dqmio --alcarecos=TkAlCosmics0T+SiStripCalZeroBias -python RunExpressProcessing.py --scenario pp --global-tag GLOBALTAG --lfn /store/whatever --fevt --dqmio --alcarecos=TkAlMinBias+SiStripCalZeroBias +python RunExpressProcessing.py --scenario pp --global-tag GLOBALTAG --lfn /store/whatever --daq --fevt --dqmio --alcarecos=TkAlMinBias+SiStripCalZeroBias """ try: @@ -192,5 +196,7 @@ def __call__(self): expressinator.alcaRecos = [ x for x in arg.split('+') if len(x) > 0 ] if opt == "--nThreads": expressinator.nThreads = arg + if opt == "--dat": + expressinator.dat = True expressinator() diff --git a/Configuration/DataProcessing/test/run_CfgTest_9.sh b/Configuration/DataProcessing/test/run_CfgTest_9.sh index 9e22ea7810316..191255740cb45 100755 --- a/Configuration/DataProcessing/test/run_CfgTest_9.sh +++ b/Configuration/DataProcessing/test/run_CfgTest_9.sh @@ -11,6 +11,8 @@ function die { echo $1: status $2 ; exit $2; } function runTest { echo $1 ; python3 $1 || die "Failure for configuration: $1" $?; } runTest "${SCRAM_TEST_PATH}/RunExpressProcessing.py --scenario AlCaTestEnable --global-tag GLOBALTAG --lfn /store/whatever --alcareco PromptCalibProdEcalPedestals " +runTest "${SCRAM_TEST_PATH}/RunExpressProcessing.py --scenario AlCaTestEnable --global-tag GLOBALTAG --lfn /store/whatever --alcareco PromptCalibProdEcalPedestals --dat" +runTest "${SCRAM_TEST_PATH}/RunExpressProcessing.py --scenario AlCaTestEnable --global-tag GLOBALTAG --lfn /store/whatever --alcareco PromptCalibProdEcalPedestals --nThreads=8" runTest "${SCRAM_TEST_PATH}/RunAlcaSkimming.py --scenario AlCaTestEnable --lfn=/store/whatever --global-tag GLOBALTAG --skims PromptCalibProdEcalPedestals" runTest "${SCRAM_TEST_PATH}/RunAlcaHarvesting.py --scenario AlCaTestEnable --lfn /store/whatever --dataset /A/B/C --global-tag GLOBALTAG --alcapromptdataset=PromptCalibProdEcalPedestals" From 0ff9be1a590a26b2e0b71a648f803875067ece3b Mon Sep 17 00:00:00 2001 From: German Giraldo Date: Fri, 20 Sep 2024 16:06:27 +0200 Subject: [PATCH 2/4] Cast number of threads to int before passing it to ConfigBuilder --- Configuration/DataProcessing/test/RunPromptReco.py | 2 +- Configuration/DataProcessing/test/run_CfgTest_7.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Configuration/DataProcessing/test/RunPromptReco.py b/Configuration/DataProcessing/test/RunPromptReco.py index 947a62c078e68..919c00a1a4f1e 100644 --- a/Configuration/DataProcessing/test/RunPromptReco.py +++ b/Configuration/DataProcessing/test/RunPromptReco.py @@ -109,7 +109,7 @@ def __call__(self): kwds['repacked'] = self.isRepacked if self.nThreads: - kwds['nThreads'] = self.nThreads + kwds['nThreads'] = int(self.nThreads) process = scenario.promptReco(self.globalTag, **kwds) diff --git a/Configuration/DataProcessing/test/run_CfgTest_7.sh b/Configuration/DataProcessing/test/run_CfgTest_7.sh index 86360b330f11d..5d7486202c3d5 100755 --- a/Configuration/DataProcessing/test/run_CfgTest_7.sh +++ b/Configuration/DataProcessing/test/run_CfgTest_7.sh @@ -14,6 +14,7 @@ declare -a arr=("ppEra_Run2_2018" "ppEra_Run2_2018_highBetaStar" "ppEra_Run2_201 for scenario in "${arr[@]}" do runTest "${SCRAM_TEST_PATH}/RunPromptReco.py --scenario $scenario --reco --aod --miniaod --dqmio --global-tag GLOBALTAG --lfn=/store/whatever --alcareco TkAlMinBias+SiStripCalMinBias" + runTest "${SCRAM_TEST_PATH}/RunPromptReco.py --scenario $scenario --reco --aod --miniaod --dqmio --global-tag GLOBALTAG --lfn=/store/whatever --alcareco TkAlMinBias+SiStripCalMinBias --nThreads=8" runTest "${SCRAM_TEST_PATH}/RunPromptReco.py --scenario $scenario --reco --aod --dqmio --global-tag GLOBALTAG --lfn=/store/whatever --alcareco TkAlMinBias+SiStripCalMinBias --PhysicsSkim=@SingleMuon" done From eed11a54688fed384069ae1d8e45353763c39f58 Mon Sep 17 00:00:00 2001 From: German Giraldo Date: Mon, 23 Sep 2024 15:15:50 +0200 Subject: [PATCH 3/4] Unify skim syntax in AlCa test scripts --- Configuration/DataProcessing/test/RunAlcaHarvesting.py | 2 +- Configuration/DataProcessing/test/RunAlcaSkimming.py | 2 +- Configuration/DataProcessing/test/run_CfgTest_10.sh | 2 +- Configuration/DataProcessing/test/run_CfgTest_11.sh | 2 +- Configuration/DataProcessing/test/run_CfgTest_12.sh | 4 ++-- Configuration/DataProcessing/test/run_CfgTest_2.sh | 2 +- Configuration/DataProcessing/test/run_CfgTest_6.sh | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Configuration/DataProcessing/test/RunAlcaHarvesting.py b/Configuration/DataProcessing/test/RunAlcaHarvesting.py index 10ddfd7529744..4695ad58df987 100644 --- a/Configuration/DataProcessing/test/RunAlcaHarvesting.py +++ b/Configuration/DataProcessing/test/RunAlcaHarvesting.py @@ -140,7 +140,7 @@ def __call__(self): if opt == "--dataset" : harvester.dataset = arg if opt == "--workflows": - harvester.workflows = [ x for x in arg.split(',') if len(x) > 0 ] + harvester.workflows = [ x for x in arg.split('+') if len(x) > 0 ] if opt == "--alcapromptdataset": harvester.alcapromptdataset = arg diff --git a/Configuration/DataProcessing/test/RunAlcaSkimming.py b/Configuration/DataProcessing/test/RunAlcaSkimming.py index 15dcb72cb8a11..0670e5b2b2125 100644 --- a/Configuration/DataProcessing/test/RunAlcaSkimming.py +++ b/Configuration/DataProcessing/test/RunAlcaSkimming.py @@ -124,7 +124,7 @@ def __call__(self): if opt == "--lfn" : skimmer.inputLFN = arg if opt == "--skims": - skimmer.skims = [ x for x in arg.split(',') if len(x) > 0 ] + skimmer.skims = [ x for x in arg.split('+') if len(x) > 0 ] if opt == "--global-tag": skimmer.globalTag = arg diff --git a/Configuration/DataProcessing/test/run_CfgTest_10.sh b/Configuration/DataProcessing/test/run_CfgTest_10.sh index b278e1490c621..635142426be2d 100755 --- a/Configuration/DataProcessing/test/run_CfgTest_10.sh +++ b/Configuration/DataProcessing/test/run_CfgTest_10.sh @@ -11,6 +11,6 @@ function die { echo $1: status $2 ; exit $2; } function runTest { echo $1 ; python3 $1 || die "Failure for configuration: $1" $?; } runTest "${SCRAM_TEST_PATH}/RunExpressProcessing.py --scenario AlCaLumiPixels --global-tag GLOBALTAG --lfn /store/whatever --alcareco AlCaPCCRandom+PromptCalibProdLumiPCC" -runTest "${SCRAM_TEST_PATH}/RunAlcaSkimming.py --scenario AlCaLumiPixels --lfn=/store/whatever --global-tag GLOBALTAG --skims AlCaPCCRandom,PromptCalibProdLumiPCC" +runTest "${SCRAM_TEST_PATH}/RunAlcaSkimming.py --scenario AlCaLumiPixels --lfn=/store/whatever --global-tag GLOBALTAG --skims AlCaPCCRandom+PromptCalibProdLumiPCC" runTest "${SCRAM_TEST_PATH}/RunAlcaHarvesting.py --scenario AlCaLumiPixels --lfn /store/whatever --dataset /A/B/C --global-tag GLOBALTAG --alcapromptdataset=PromptCalibProdLumiPCC" diff --git a/Configuration/DataProcessing/test/run_CfgTest_11.sh b/Configuration/DataProcessing/test/run_CfgTest_11.sh index 539c3756e5b6d..9cfe83bfde8e8 100755 --- a/Configuration/DataProcessing/test/run_CfgTest_11.sh +++ b/Configuration/DataProcessing/test/run_CfgTest_11.sh @@ -11,6 +11,6 @@ function die { echo $1: status $2 ; exit $2; } function runTest { echo $1 ; python3 $1 || die "Failure for configuration: $1" $?; } runTest "${SCRAM_TEST_PATH}/RunExpressProcessing.py --scenario AlCaLumiPixels_Run3 --global-tag GLOBALTAG --lfn /store/whatever --alcareco AlCaPCCRandom+PromptCalibProdLumiPCC" -runTest "${SCRAM_TEST_PATH}/RunAlcaSkimming.py --scenario AlCaLumiPixels_Run3 --lfn=/store/whatever --global-tag GLOBALTAG --skims AlCaPCCRandom,PromptCalibProdLumiPCC" +runTest "${SCRAM_TEST_PATH}/RunAlcaSkimming.py --scenario AlCaLumiPixels_Run3 --lfn=/store/whatever --global-tag GLOBALTAG --skims AlCaPCCRandom+PromptCalibProdLumiPCC" runTest "${SCRAM_TEST_PATH}/RunAlcaHarvesting.py --scenario AlCaLumiPixels_Run3 --lfn /store/whatever --dataset /A/B/C --global-tag GLOBALTAG --alcapromptdataset=PromptCalibProdLumiPCC" diff --git a/Configuration/DataProcessing/test/run_CfgTest_12.sh b/Configuration/DataProcessing/test/run_CfgTest_12.sh index 746ecdd6a4f2e..7b1189784e3c0 100755 --- a/Configuration/DataProcessing/test/run_CfgTest_12.sh +++ b/Configuration/DataProcessing/test/run_CfgTest_12.sh @@ -14,8 +14,8 @@ declare -a arr=("trackingOnlyEra_Run2_2018" "trackingOnlyEra_Run2_2018_highBetaS for scenario in "${arr[@]}" do runTest "${SCRAM_TEST_PATH}/RunExpressProcessing.py --scenario $scenario --global-tag GLOBALTAG --lfn /store/whatever --alcarecos=TkAlMinBias+PromptCalibProdBeamSpotHP" - runTest "${SCRAM_TEST_PATH}/RunAlcaSkimming.py --scenario $scenario --lfn /store/whatever --global-tag GLOBALTAG --skims TkAlMinBias,PromptCalibProdBeamSpotHP" - runTest "${SCRAM_TEST_PATH}/RunAlcaSkimming.py --scenario $scenario --lfn /store/whatever --global-tag GLOBALTAG --skims TkAlMinBias,PromptCalibProdBeamSpotHPLowPU" + runTest "${SCRAM_TEST_PATH}/RunAlcaSkimming.py --scenario $scenario --lfn /store/whatever --global-tag GLOBALTAG --skims TkAlMinBias+PromptCalibProdBeamSpotHP" + runTest "${SCRAM_TEST_PATH}/RunAlcaSkimming.py --scenario $scenario --lfn /store/whatever --global-tag GLOBALTAG --skims TkAlMinBias+PromptCalibProdBeamSpotHPLowPU" runTest "${SCRAM_TEST_PATH}/RunAlcaHarvesting.py --scenario $scenario --lfn /store/whatever --dataset /A/B/C --global-tag GLOBALTAG --alcapromptdataset=PromptCalibProdBeamSpotHP" runTest "${SCRAM_TEST_PATH}/RunAlcaHarvesting.py --scenario $scenario --lfn /store/whatever --dataset /A/B/C --global-tag GLOBALTAG --alcapromptdataset=PromptCalibProdBeamSpotHPLowPU" done diff --git a/Configuration/DataProcessing/test/run_CfgTest_2.sh b/Configuration/DataProcessing/test/run_CfgTest_2.sh index 10ec8c3fcfe97..4c6caf69609be 100755 --- a/Configuration/DataProcessing/test/run_CfgTest_2.sh +++ b/Configuration/DataProcessing/test/run_CfgTest_2.sh @@ -15,6 +15,6 @@ for scenario in "${arr[@]}" do runTest "${SCRAM_TEST_PATH}/RunExpressProcessing.py --scenario $scenario --global-tag GLOBALTAG --lfn /store/whatever --fevt --dqmio --alcareco TkAlMinBias+SiStripCalMinBias " runTest "${SCRAM_TEST_PATH}/RunVisualizationProcessing.py --scenario $scenario --lfn /store/whatever --global-tag GLOBALTAG --fevt" - runTest "${SCRAM_TEST_PATH}/RunAlcaHarvesting.py --scenario $scenario --lfn /store/whatever --dataset /A/B/C --global-tag GLOBALTAG --workflows=BeamSpotByRun,BeamSpotByLumi,SiStripQuality" + runTest "${SCRAM_TEST_PATH}/RunAlcaHarvesting.py --scenario $scenario --lfn /store/whatever --dataset /A/B/C --global-tag GLOBALTAG --workflows=BeamSpotByRun+BeamSpotByLumi+SiStripQuality" done diff --git a/Configuration/DataProcessing/test/run_CfgTest_6.sh b/Configuration/DataProcessing/test/run_CfgTest_6.sh index a2cddf2baa042..675cea0bf6b68 100755 --- a/Configuration/DataProcessing/test/run_CfgTest_6.sh +++ b/Configuration/DataProcessing/test/run_CfgTest_6.sh @@ -13,7 +13,7 @@ function runTest { echo $1 ; python3 $1 || die "Failure for configuration: $1" $ declare -a arr=("AlCaLumiPixels" "cosmicsEra_Run2_2018" "ppEra_Run2_2018" "ppEra_Run2_2018_highBetaStar" "ppEra_Run2_2018_pp_on_AA" "cosmicsHybridEra_Run2_2018" "cosmicsEra_Run3" "ppEra_Run3" "AlCaLumiPixels_Run3") for scenario in "${arr[@]}" do - runTest "${SCRAM_TEST_PATH}/RunAlcaSkimming.py --scenario $scenario --lfn=/store/whatever --global-tag GLOBALTAG --skims SiStripCalZeroBias,SiStripCalMinBias,PromptCalibProd" + runTest "${SCRAM_TEST_PATH}/RunAlcaSkimming.py --scenario $scenario --lfn=/store/whatever --global-tag GLOBALTAG --skims SiStripCalZeroBias+SiStripCalMinBias+PromptCalibProd" runTest "${SCRAM_TEST_PATH}/RunDQMHarvesting.py --scenario $scenario --lfn /store/whatever --run 12345 --dataset /A/B/C --global-tag GLOBALTAG" done From fdd933864804983ed106ff582bd49077f97bc504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Germ=C3=A1n=20Felipe=20Giraldo=20Villa?= Date: Mon, 23 Sep 2024 23:22:50 +0200 Subject: [PATCH 4/4] Fix typo in RunExpressProcessing.py docstring Co-authored-by: Marco Musich --- Configuration/DataProcessing/test/RunExpressProcessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Configuration/DataProcessing/test/RunExpressProcessing.py b/Configuration/DataProcessing/test/RunExpressProcessing.py index 4b1bee62c892e..6672301909dfa 100644 --- a/Configuration/DataProcessing/test/RunExpressProcessing.py +++ b/Configuration/DataProcessing/test/RunExpressProcessing.py @@ -160,7 +160,7 @@ def __call__(self): python RunExpressProcessing.py --scenario cosmics --global-tag GLOBALTAG --lfn /store/whatever --fevt --dqmio --alcarecos=TkAlCosmics0T+SiStripCalZeroBias -python RunExpressProcessing.py --scenario pp --global-tag GLOBALTAG --lfn /store/whatever --daq --fevt --dqmio --alcarecos=TkAlMinBias+SiStripCalZeroBias +python RunExpressProcessing.py --scenario pp --global-tag GLOBALTAG --lfn /store/whatever --dat --fevt --dqmio --alcarecos=TkAlMinBias+SiStripCalZeroBias """ try: