Implement HCAL-only workflows on GPU (#505)

Implement the HCAL-only on GPU workflow in runTheMatrix.py, 11634.522 .
Add customisations for profiling the HCAL-only workflows.

RecoLocalCalo/Configuration/python/customizeHcalOnlyForProfiling.py

@@ -0,0 +1,60 @@
+import FWCore.ParameterSet.Config as cms
+
+# Customise the HCAL-only reconstruction to run on GPU
+#
+# Currently, this means:
+#   - running the unpacker on CPU, converting the digis into SoA format and copying them to GPU;
+#   - running the HBHE local reconstruction, including MAHI, on GPU.
+def customizeHcalOnlyForProfilingGPUOnly(process):
+
+  process.consumer = cms.EDAnalyzer("GenericConsumer",
+      eventProducts = cms.untracked.vstring('hbheRecHitProducerGPU')
+  )
+
+  process.consume_step = cms.EndPath(process.consumer)
+
+  process.schedule = cms.Schedule(process.raw2digi_step, process.reconstruction_step, process.consume_step)
+
+  return process
+
+
+# Customise the HCAL-only reconstruction to run on GPU, and copy the data to the host
+#
+# Currently, this means:
+#   - running the unpacker on CPU, converting the digis into SoA format and copying them to GPU;
+#   - running the HBHE local reconstruction, including MAHI, on GPU;
+#   - copying the rechits to CPU and converting them to legacy format.
+#
+# (this is equivalent to customizeHcalOnlyForProfiling, as the copy and conversion is done by the same module)
+def customizeHcalOnlyForProfilingGPUWithHostCopy(process):
+
+  process.consumer = cms.EDAnalyzer("GenericConsumer",
+      eventProducts = cms.untracked.vstring('hbheprereco')
+  )
+
+  process.consume_step = cms.EndPath(process.consumer)
+
+  process.schedule = cms.Schedule(process.raw2digi_step, process.reconstruction_step, process.consume_step)
+
+  return process
+
+
+# Customise the HCAL-only reconstruction to run on GPU, copy the data to the host, and convert to legacy format
+#
+# Currently, this means:
+#   - running the unpacker on CPU, converting the digis into SoA format and copying them to GPU;
+#   - running the HBHE local reconstruction, including MAHI, on GPU;
+#   - copying the rechits to CPU and converting them to legacy format.
+#
+# The same customisation can be also used on the CPU workflow, running up to the rechits on CPU.
+def customizeHcalOnlyForProfiling(process):
+
+  process.consumer = cms.EDAnalyzer("GenericConsumer",
+      eventProducts = cms.untracked.vstring('hbheprereco')
+  )
+
+  process.consume_step = cms.EndPath(process.consumer)
+
+  process.schedule = cms.Schedule(process.raw2digi_step, process.reconstruction_step, process.consume_step)
+
+  return process

RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py

@@ -48,6 +48,26 @@
 from Configuration.ProcessModifiers.run2_HECollapse_2018_cff import run2_HECollapse_2018
 run2_HECollapse_2018.toReplaceWith(hcalLocalRecoTask, _collapse_hcalLocalRecoTask)
 
+# Run 3 HCAL workflow on GPU
+from Configuration.ProcessModifiers.gpu_cff import gpu
+
+from RecoLocalCalo.HcalRecProducers.hbheRecHitProducerGPUTask_cff import *
+_hcalLocalRecoTaskGPU = hcalLocalRecoTask.copy()
+_hcalLocalRecoTaskGPU.add(hbheRecHitProducerGPUTask)
+gpu.toReplaceWith(hcalLocalRecoTask, _hcalLocalRecoTaskGPU)
+
+_hcalOnlyLocalRecoTaskGPU = hcalOnlyLocalRecoTask.copy()
+_hcalOnlyLocalRecoTaskGPU.add(hbheRecHitProducerGPUTask)
+gpu.toReplaceWith(hcalOnlyLocalRecoTask, _hcalOnlyLocalRecoTaskGPU)
+
+from RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi import hcalCPURecHitsProducer as _hcalCPURecHitsProducer
+gpu.toReplaceWith(hbheprereco, _hcalCPURecHitsProducer.clone(
+    recHitsM0LabelIn = "hbheRecHitProducerGPU",
+    recHitsM0LabelOut = "",
+    recHitsLegacyLabelOut = ""
+))
+#---
+
 _phase2_hcalLocalRecoTask = hcalLocalRecoTask.copy()
 _phase2_hcalLocalRecoTask.remove(hbheprereco)
 

RecoLocalCalo/HcalRecProducers/python/hbheRecHitProducerGPUTask_cff.py

@@ -0,0 +1,63 @@
+import FWCore.ParameterSet.Config as cms
+
+# Run 3 HCAL workflow on GPU
+
+# EventSetup modules used by HBHERecHitProducerGPU
+from RecoLocalCalo.HcalRecProducers.hcalGainsGPUESProducer_cfi import hcalGainsGPUESProducer
+from RecoLocalCalo.HcalRecProducers.hcalGainWidthsGPUESProducer_cfi import hcalGainWidthsGPUESProducer
+from RecoLocalCalo.HcalRecProducers.hcalLUTCorrsGPUESProducer_cfi import hcalLUTCorrsGPUESProducer
+from RecoLocalCalo.HcalRecProducers.hcalConvertedPedestalsGPUESProducer_cfi import hcalConvertedPedestalsGPUESProducer
+from RecoLocalCalo.HcalRecProducers.hcalConvertedEffectivePedestalsGPUESProducer_cfi import hcalConvertedEffectivePedestalsGPUESProducer
+hcalConvertedEffectivePedestalsGPUESProducer.label0 = "withTopoEff"
+
+from RecoLocalCalo.HcalRecProducers.hcalConvertedPedestalWidthsGPUESProducer_cfi import hcalConvertedPedestalWidthsGPUESProducer
+from RecoLocalCalo.HcalRecProducers.hcalConvertedEffectivePedestalWidthsGPUESProducer_cfi import hcalConvertedEffectivePedestalWidthsGPUESProducer
+hcalConvertedEffectivePedestalWidthsGPUESProducer.label0 = "withTopoEff"
+hcalConvertedEffectivePedestalWidthsGPUESProducer.label1 = "withTopoEff"
+
+from RecoLocalCalo.HcalRecProducers.hcalQIECodersGPUESProducer_cfi import hcalQIECodersGPUESProducer
+from RecoLocalCalo.HcalRecProducers.hcalRecoParamsWithPulseShapesGPUESProducer_cfi import hcalRecoParamsWithPulseShapesGPUESProducer
+from RecoLocalCalo.HcalRecProducers.hcalRespCorrsGPUESProducer_cfi import hcalRespCorrsGPUESProducer
+from RecoLocalCalo.HcalRecProducers.hcalTimeCorrsGPUESProducer_cfi import hcalTimeCorrsGPUESProducer
+from RecoLocalCalo.HcalRecProducers.hcalQIETypesGPUESProducer_cfi import hcalQIETypesGPUESProducer
+from RecoLocalCalo.HcalRecProducers.hcalSiPMParametersGPUESProducer_cfi import hcalSiPMParametersGPUESProducer
+from RecoLocalCalo.HcalRecProducers.hcalSiPMCharacteristicsGPUESProducer_cfi import hcalSiPMCharacteristicsGPUESProducer
+
+# convert the HBHE digis into SoA format, and copy them from CPU to GPU
+from EventFilter.HcalRawToDigi.hcalDigisProducerGPU_cfi import hcalDigisProducerGPU as _hcalDigisProducerGPU
+hcalDigisGPU = _hcalDigisProducerGPU.clone(
+    digisLabelF01HE = "",
+    digisLabelF5HB = "",
+    digisLabelF3HB = ""
+)
+
+# run the HCAL local reconstruction (MAHI) on GPU
+from RecoLocalCalo.HcalRecProducers.hbheRecHitProducerGPU_cfi import hbheRecHitProducerGPU as _hbheRecHitProducerGPU
+hbheRecHitProducerGPU = _hbheRecHitProducerGPU.clone(
+    digisLabelF01HE = "hcalDigisGPU",
+    digisLabelF5HB = "hcalDigisGPU",
+    digisLabelF3HB = "hcalDigisGPU",
+    recHitsLabelM0HBHE = ""
+)
+
+# Tasks and Sequences
+hbheRecHitProducerGPUTask = cms.Task(
+    hcalGainsGPUESProducer,
+    hcalGainWidthsGPUESProducer,
+    hcalLUTCorrsGPUESProducer,
+    hcalConvertedPedestalsGPUESProducer,
+    hcalConvertedEffectivePedestalsGPUESProducer,
+    hcalConvertedPedestalWidthsGPUESProducer,
+    hcalConvertedEffectivePedestalWidthsGPUESProducer,
+    hcalQIECodersGPUESProducer,
+    hcalRecoParamsWithPulseShapesGPUESProducer,
+    hcalRespCorrsGPUESProducer,
+    hcalTimeCorrsGPUESProducer,
+    hcalQIETypesGPUESProducer,
+    hcalSiPMParametersGPUESProducer,
+    hcalSiPMCharacteristicsGPUESProducer,
+    hcalDigisGPU,
+    hbheRecHitProducerGPU
+)
+
+hbheRecHitProducerGPUSequence = cms.Sequence(hbheRecHitProducerGPUTask)