From a5f83902cb9c22564d054bfc975ee22fe9d14555 Mon Sep 17 00:00:00 2001 From: Andrea Bocci Date: Sat, 11 Jul 2020 00:11:51 +0200 Subject: [PATCH] Implement HCAL-only workflows on GPU (cms-patatrack#505) Implement the HCAL-only on GPU workflow in runTheMatrix.py, 11634.522 . Add customisations for profiling the HCAL-only workflows. --- .../python/customizeHcalOnlyForProfiling.py | 60 ++++++++++++++++++ .../Configuration/python/hcalLocalReco_cff.py | 20 ++++++ .../python/hbheRecHitProducerGPUTask_cff.py | 63 +++++++++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 RecoLocalCalo/Configuration/python/customizeHcalOnlyForProfiling.py create mode 100644 RecoLocalCalo/HcalRecProducers/python/hbheRecHitProducerGPUTask_cff.py diff --git a/RecoLocalCalo/Configuration/python/customizeHcalOnlyForProfiling.py b/RecoLocalCalo/Configuration/python/customizeHcalOnlyForProfiling.py new file mode 100644 index 0000000000000..b3a2548791ae5 --- /dev/null +++ b/RecoLocalCalo/Configuration/python/customizeHcalOnlyForProfiling.py @@ -0,0 +1,60 @@ +import FWCore.ParameterSet.Config as cms + +# Customise the HCAL-only reconstruction to run on GPU +# +# Currently, this means: +# - running the unpacker on CPU, converting the digis into SoA format and copying them to GPU; +# - running the HBHE local reconstruction, including MAHI, on GPU. +def customizeHcalOnlyForProfilingGPUOnly(process): + + process.consumer = cms.EDAnalyzer("GenericConsumer", + eventProducts = cms.untracked.vstring('hbheRecHitProducerGPU') + ) + + process.consume_step = cms.EndPath(process.consumer) + + process.schedule = cms.Schedule(process.raw2digi_step, process.reconstruction_step, process.consume_step) + + return process + + +# Customise the HCAL-only reconstruction to run on GPU, and copy the data to the host +# +# Currently, this means: +# - running the unpacker on CPU, converting the digis into SoA format and copying them to GPU; +# - running the HBHE local reconstruction, including MAHI, on GPU; +# - copying the rechits to CPU and converting them to legacy format. +# +# (this is equivalent to customizeHcalOnlyForProfiling, as the copy and conversion is done by the same module) +def customizeHcalOnlyForProfilingGPUWithHostCopy(process): + + process.consumer = cms.EDAnalyzer("GenericConsumer", + eventProducts = cms.untracked.vstring('hbheprereco') + ) + + process.consume_step = cms.EndPath(process.consumer) + + process.schedule = cms.Schedule(process.raw2digi_step, process.reconstruction_step, process.consume_step) + + return process + + +# Customise the HCAL-only reconstruction to run on GPU, copy the data to the host, and convert to legacy format +# +# Currently, this means: +# - running the unpacker on CPU, converting the digis into SoA format and copying them to GPU; +# - running the HBHE local reconstruction, including MAHI, on GPU; +# - copying the rechits to CPU and converting them to legacy format. +# +# The same customisation can be also used on the CPU workflow, running up to the rechits on CPU. +def customizeHcalOnlyForProfiling(process): + + process.consumer = cms.EDAnalyzer("GenericConsumer", + eventProducts = cms.untracked.vstring('hbheprereco') + ) + + process.consume_step = cms.EndPath(process.consumer) + + process.schedule = cms.Schedule(process.raw2digi_step, process.reconstruction_step, process.consume_step) + + return process diff --git a/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py b/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py index ddcab0252afcf..2eff3a1eb5e40 100644 --- a/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py +++ b/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py @@ -48,6 +48,26 @@ from Configuration.ProcessModifiers.run2_HECollapse_2018_cff import run2_HECollapse_2018 run2_HECollapse_2018.toReplaceWith(hcalLocalRecoTask, _collapse_hcalLocalRecoTask) +# Run 3 HCAL workflow on GPU +from Configuration.ProcessModifiers.gpu_cff import gpu + +from RecoLocalCalo.HcalRecProducers.hbheRecHitProducerGPUTask_cff import * +_hcalLocalRecoTaskGPU = hcalLocalRecoTask.copy() +_hcalLocalRecoTaskGPU.add(hbheRecHitProducerGPUTask) +gpu.toReplaceWith(hcalLocalRecoTask, _hcalLocalRecoTaskGPU) + +_hcalOnlyLocalRecoTaskGPU = hcalOnlyLocalRecoTask.copy() +_hcalOnlyLocalRecoTaskGPU.add(hbheRecHitProducerGPUTask) +gpu.toReplaceWith(hcalOnlyLocalRecoTask, _hcalOnlyLocalRecoTaskGPU) + +from RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi import hcalCPURecHitsProducer as _hcalCPURecHitsProducer +gpu.toReplaceWith(hbheprereco, _hcalCPURecHitsProducer.clone( + recHitsM0LabelIn = "hbheRecHitProducerGPU", + recHitsM0LabelOut = "", + recHitsLegacyLabelOut = "" +)) +#--- + _phase2_hcalLocalRecoTask = hcalLocalRecoTask.copy() _phase2_hcalLocalRecoTask.remove(hbheprereco) diff --git a/RecoLocalCalo/HcalRecProducers/python/hbheRecHitProducerGPUTask_cff.py b/RecoLocalCalo/HcalRecProducers/python/hbheRecHitProducerGPUTask_cff.py new file mode 100644 index 0000000000000..d2d3dac166469 --- /dev/null +++ b/RecoLocalCalo/HcalRecProducers/python/hbheRecHitProducerGPUTask_cff.py @@ -0,0 +1,63 @@ +import FWCore.ParameterSet.Config as cms + +# Run 3 HCAL workflow on GPU + +# EventSetup modules used by HBHERecHitProducerGPU +from RecoLocalCalo.HcalRecProducers.hcalGainsGPUESProducer_cfi import hcalGainsGPUESProducer +from RecoLocalCalo.HcalRecProducers.hcalGainWidthsGPUESProducer_cfi import hcalGainWidthsGPUESProducer +from RecoLocalCalo.HcalRecProducers.hcalLUTCorrsGPUESProducer_cfi import hcalLUTCorrsGPUESProducer +from RecoLocalCalo.HcalRecProducers.hcalConvertedPedestalsGPUESProducer_cfi import hcalConvertedPedestalsGPUESProducer +from RecoLocalCalo.HcalRecProducers.hcalConvertedEffectivePedestalsGPUESProducer_cfi import hcalConvertedEffectivePedestalsGPUESProducer +hcalConvertedEffectivePedestalsGPUESProducer.label0 = "withTopoEff" + +from RecoLocalCalo.HcalRecProducers.hcalConvertedPedestalWidthsGPUESProducer_cfi import hcalConvertedPedestalWidthsGPUESProducer +from RecoLocalCalo.HcalRecProducers.hcalConvertedEffectivePedestalWidthsGPUESProducer_cfi import hcalConvertedEffectivePedestalWidthsGPUESProducer +hcalConvertedEffectivePedestalWidthsGPUESProducer.label0 = "withTopoEff" +hcalConvertedEffectivePedestalWidthsGPUESProducer.label1 = "withTopoEff" + +from RecoLocalCalo.HcalRecProducers.hcalQIECodersGPUESProducer_cfi import hcalQIECodersGPUESProducer +from RecoLocalCalo.HcalRecProducers.hcalRecoParamsWithPulseShapesGPUESProducer_cfi import hcalRecoParamsWithPulseShapesGPUESProducer +from RecoLocalCalo.HcalRecProducers.hcalRespCorrsGPUESProducer_cfi import hcalRespCorrsGPUESProducer +from RecoLocalCalo.HcalRecProducers.hcalTimeCorrsGPUESProducer_cfi import hcalTimeCorrsGPUESProducer +from RecoLocalCalo.HcalRecProducers.hcalQIETypesGPUESProducer_cfi import hcalQIETypesGPUESProducer +from RecoLocalCalo.HcalRecProducers.hcalSiPMParametersGPUESProducer_cfi import hcalSiPMParametersGPUESProducer +from RecoLocalCalo.HcalRecProducers.hcalSiPMCharacteristicsGPUESProducer_cfi import hcalSiPMCharacteristicsGPUESProducer + +# convert the HBHE digis into SoA format, and copy them from CPU to GPU +from EventFilter.HcalRawToDigi.hcalDigisProducerGPU_cfi import hcalDigisProducerGPU as _hcalDigisProducerGPU +hcalDigisGPU = _hcalDigisProducerGPU.clone( + digisLabelF01HE = "", + digisLabelF5HB = "", + digisLabelF3HB = "" +) + +# run the HCAL local reconstruction (MAHI) on GPU +from RecoLocalCalo.HcalRecProducers.hbheRecHitProducerGPU_cfi import hbheRecHitProducerGPU as _hbheRecHitProducerGPU +hbheRecHitProducerGPU = _hbheRecHitProducerGPU.clone( + digisLabelF01HE = "hcalDigisGPU", + digisLabelF5HB = "hcalDigisGPU", + digisLabelF3HB = "hcalDigisGPU", + recHitsLabelM0HBHE = "" +) + +# Tasks and Sequences +hbheRecHitProducerGPUTask = cms.Task( + hcalGainsGPUESProducer, + hcalGainWidthsGPUESProducer, + hcalLUTCorrsGPUESProducer, + hcalConvertedPedestalsGPUESProducer, + hcalConvertedEffectivePedestalsGPUESProducer, + hcalConvertedPedestalWidthsGPUESProducer, + hcalConvertedEffectivePedestalWidthsGPUESProducer, + hcalQIECodersGPUESProducer, + hcalRecoParamsWithPulseShapesGPUESProducer, + hcalRespCorrsGPUESProducer, + hcalTimeCorrsGPUESProducer, + hcalQIETypesGPUESProducer, + hcalSiPMParametersGPUESProducer, + hcalSiPMCharacteristicsGPUESProducer, + hcalDigisGPU, + hbheRecHitProducerGPU +) + +hbheRecHitProducerGPUSequence = cms.Sequence(hbheRecHitProducerGPUTask)