Pixel Alpaka Migration: Local Reconstruction [III]

CUDADataFormats/SiPixelDigi/interface/SiPixelDigisCUDA.h

@@ -7,29 +7,18 @@
 #include "HeterogeneousCore/CUDAUtilities/interface/host_unique_ptr.h"
 #include "HeterogeneousCore/CUDAUtilities/interface/cudaCompat.h"
 #include "CUDADataFormats/Common/interface/PortableDeviceCollection.h"
+#include "DataFormats/SiPixelDigiSoA/interface/SiPixelDigisSoA.h"
 #include "DataFormats/SoATemplate/interface/SoALayout.h"
 
-GENERATE_SOA_LAYOUT(SiPixelDigisSoALayout,
-                    SOA_COLUMN(int32_t, clus),
-                    SOA_COLUMN(uint32_t, pdigi),
-                    SOA_COLUMN(uint32_t, rawIdArr),
-                    SOA_COLUMN(uint16_t, adc),
-                    SOA_COLUMN(uint16_t, xx),
-                    SOA_COLUMN(uint16_t, yy),
-                    SOA_COLUMN(uint16_t, moduleId))
-
-using SiPixelDigisCUDASOA = SiPixelDigisSoALayout<>;
-using SiPixelDigisCUDASOAView = SiPixelDigisCUDASOA::View;
-using SiPixelDigisCUDASOAConstView = SiPixelDigisCUDASOA::ConstView;
-
 // TODO: The class is created via inheritance of the PortableDeviceCollection.
 // This is generally discouraged, and should be done via composition.
 // See: https://github.com/cms-sw/cmssw/pull/40465#discussion_r1067364306
-class SiPixelDigisCUDA : public cms::cuda::PortableDeviceCollection<SiPixelDigisSoALayout<>> {
+class SiPixelDigisCUDA : public cms::cuda::PortableDeviceCollection<SiPixelDigisSoA> {
 public:
   SiPixelDigisCUDA() = default;
   explicit SiPixelDigisCUDA(size_t maxFedWords, cudaStream_t stream)
-      : PortableDeviceCollection<SiPixelDigisSoALayout<>>(maxFedWords + 1, stream) {}
+      : PortableDeviceCollection<SiPixelDigisSoA>(maxFedWords + 1, stream) {}
+
   ~SiPixelDigisCUDA() = default;
 
   SiPixelDigisCUDA(SiPixelDigisCUDA &&) = default;

CalibTracker/Records/interface/SiPixelGainCalibrationForHLTSoARcd.h

@@ -0,0 +1,14 @@
+#ifndef CalibTracker_Records_SiPixelGainCalibrationForHLTSoARcd_h
+#define CalibTracker_Records_SiPixelGainCalibrationForHLTSoARcd_h
+
+#include "CondFormats/DataRecord/interface/SiPixelGainCalibrationForHLTRcd.h"
+#include "FWCore/Framework/interface/DependentRecordImplementation.h"
+#include "FWCore/Framework/interface/EventSetupRecordImplementation.h"
+#include "Geometry/Records/interface/TrackerDigiGeometryRecord.h"
+
+class SiPixelGainCalibrationForHLTSoARcd
+    : public edm::eventsetup::DependentRecordImplementation<
+          SiPixelGainCalibrationForHLTSoARcd,
+          edm::mpl::Vector<SiPixelGainCalibrationForHLTRcd, TrackerDigiGeometryRecord>> {};
+
+#endif  // CalibTracker_Records_SiPixelGainCalibrationForHLTSoARcd_h

CalibTracker/Records/interface/SiPixelMappingSoARecord.h

@@ -0,0 +1,17 @@
+#ifndef CalibTracker_Records_interface_SiPixelMappingSoARecord_h
+#define CalibTracker_Records_interface_SiPixelMappingSoARecord_h
+
+#include "CondFormats/DataRecord/interface/SiPixelFedCablingMapRcd.h"
+#include "CondFormats/DataRecord/interface/SiPixelGainCalibrationForHLTRcd.h"
+#include "CondFormats/DataRecord/interface/SiPixelQualityRcd.h"
+#include "FWCore/Framework/interface/DependentRecordImplementation.h"
+#include "Geometry/Records/interface/TrackerDigiGeometryRecord.h"
+
+class SiPixelMappingSoARecord
+    : public edm::eventsetup::DependentRecordImplementation<SiPixelMappingSoARecord,
+                                                            edm::mpl::Vector<SiPixelGainCalibrationForHLTRcd,
+                                                                             SiPixelQualityRcd,
+                                                                             SiPixelFedCablingMapRcd,
+                                                                             TrackerDigiGeometryRecord>> {};
+
+#endif  // CalibTracker_Records_interface_SiPixelMappingSoARecord_h

CalibTracker/Records/src/SiPixelGainCalibrationForHLTSoARcd.cc

@@ -0,0 +1,5 @@
+#include "CalibTracker/Records/interface/SiPixelGainCalibrationForHLTSoARcd.h"
+#include "FWCore/Framework/interface/eventsetuprecord_registration_macro.h"
+#include "FWCore/Utilities/interface/typelookup.h"
+
+EVENTSETUP_RECORD_REG(SiPixelGainCalibrationForHLTSoARcd);

CalibTracker/Records/src/SiPixelMappingSoARcd.cc

@@ -0,0 +1,5 @@
+#include "CalibTracker/Records/interface/SiPixelMappingSoARecord.h"
+#include "FWCore/Framework/interface/eventsetuprecord_registration_macro.h"
+#include "FWCore/Utilities/interface/typelookup.h"
+
+EVENTSETUP_RECORD_REG(SiPixelMappingSoARecord);

CalibTracker/SiPixelESProducers/plugins/BuildFile.xml

@@ -1,4 +1,3 @@
-<use name="cuda"/>
 <use name="CalibTracker/Records"/>
 <use name="CalibTracker/SiPixelESProducers"/>
 <use name="CalibTracker/StandaloneTrackerTopology"/>
@@ -11,6 +10,14 @@
 <use name="Geometry/TrackerGeometryBuilder"/>
 <use name="MagneticField/Engine"/>
 <use name="RecoTracker/Record"/>
+
 <library file="*.cc" name="CalibTrackerSiPixelESProducersPlugins">
+  <use name="cuda"/>
+  <flags EDM_PLUGIN="1"/>
+</library>
+
+<library file="alpaka/*.cc" name="CalibTrackerSiPixelESProducersPluginsPortable">
+  <use name="alpaka"/>
+  <flags ALPAKA_BACKENDS="1"/>
   <flags EDM_PLUGIN="1"/>
 </library>

CalibTracker/SiPixelESProducers/plugins/alpaka/SiPixelCablingSoAESProducer.cc

@@ -0,0 +1,140 @@
+#include "CalibTracker/Records/interface/SiPixelMappingSoARecord.h"
+#include "CondFormats/DataRecord/interface/SiPixelFedCablingMapRcd.h"
+#include "CondFormats/DataRecord/interface/SiPixelQualityRcd.h"
+#include "CondFormats/SiPixelObjects/interface/SiPixelFedCablingMap.h"
+#include "CondFormats/SiPixelObjects/interface/SiPixelFedCablingTree.h"
+#include "CondFormats/SiPixelObjects/interface/SiPixelMappingHost.h"
+#include "CondFormats/SiPixelObjects/interface/SiPixelQuality.h"
+#include "CondFormats/SiPixelObjects/interface/alpaka/SiPixelMappingDevice.h"
+#include "DataFormats/SiPixelClusterSoA/interface/ClusteringConstants.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
+#include "FWCore/Utilities/interface/ESGetToken.h"
+#include "Geometry/Records/interface/TrackerDigiGeometryRecord.h"
+#include "Geometry/TrackerGeometryBuilder/interface/TrackerGeometry.h"
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/ESProducer.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/memory.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  using namespace cms::alpakatools;
+
+  class SiPixelCablingSoAESProducer : public ESProducer {
+  public:
+    SiPixelCablingSoAESProducer(edm::ParameterSet const& iConfig)
+        : ESProducer(iConfig), useQuality_(iConfig.getParameter<bool>("UseQualityInfo")) {
+      auto cc = setWhatProduced(this);
+      cablingMapToken_ = cc.consumes(edm::ESInputTag{"", iConfig.getParameter<std::string>("CablingMapLabel")});
+      if (useQuality_) {
+        qualityToken_ = cc.consumes();
+      }
+      geometryToken_ = cc.consumes();
+    }
+
+    static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
+      edm::ParameterSetDescription desc;
+      desc.add<std::string>("CablingMapLabel", "")->setComment("CablingMap label");
+      desc.add<bool>("UseQualityInfo", false);
+      descriptions.addWithDefaultLabel(desc);
+    }
+
+    std::optional<SiPixelMappingHost> produce(const SiPixelMappingSoARecord& iRecord) {
+      auto cablingMap = iRecord.getTransientHandle(cablingMapToken_);
+      const SiPixelQuality* quality = nullptr;
+      if (useQuality_) {
+        auto qualityInfo = iRecord.getTransientHandle(qualityToken_);
+        quality = qualityInfo.product();
+      }
+
+      auto geom = iRecord.getTransientHandle(geometryToken_);
+      SiPixelMappingHost product(pixelgpudetails::MAX_SIZE, cms::alpakatools::host());
+      std::vector<unsigned int> const& fedIds = cablingMap->fedIds();
+      std::unique_ptr<SiPixelFedCablingTree> const& cabling = cablingMap->cablingTree();
+
+      unsigned int startFed = fedIds.front();
+      unsigned int endFed = fedIds.back();
+
+      sipixelobjects::CablingPathToDetUnit path;
+      int index = 1;
+
+      auto mapView = product.view();
+
+      mapView.hasQuality() = useQuality_;
+      for (unsigned int fed = startFed; fed <= endFed; fed++) {
+        for (unsigned int link = 1; link <= pixelgpudetails::MAX_LINK; link++) {
+          for (unsigned int roc = 1; roc <= pixelgpudetails::MAX_ROC; roc++) {
+            path = {fed, link, roc};
+            const sipixelobjects::PixelROC* pixelRoc = cabling->findItem(path);
+            mapView[index].fed() = fed;
+            mapView[index].link() = link;
+            mapView[index].roc() = roc;
+            if (pixelRoc != nullptr) {
+              mapView[index].rawId() = pixelRoc->rawId();
+              mapView[index].rocInDet() = pixelRoc->idInDetUnit();
+              mapView[index].modToUnpDefault() = false;
+              if (quality != nullptr)
+                mapView[index].badRocs() = quality->IsRocBad(pixelRoc->rawId(), pixelRoc->idInDetUnit());
+              else
+                mapView[index].badRocs() = false;
+            } else {  // store some dummy number
+              mapView[index].rawId() = pixelClustering::invalidModuleId;
+              mapView[index].rocInDet() = pixelClustering::invalidModuleId;
+              mapView[index].badRocs() = true;
+              mapView[index].modToUnpDefault() = true;
+            }
+            index++;
+          }
+        }
+      }  // end of FED loop
+      // Given FedId, Link and idinLnk; use the following formula
+      // to get the rawId and idinDU
+      // index = (FedID-1200) * MAX_LINK* MAX_ROC + (Link-1)* MAX_ROC + idinLnk;
+      // where, MAX_LINK = 48, MAX_ROC = 8
+      // FedID varies between 1200 to 1338 (In total 108 FED's)
+      // Link varies between 1 to 48
+      // idinLnk varies between 1 to 8
+
+      auto trackerGeom = iRecord.getTransientHandle(geometryToken_);
+
+      for (int i = 1; i < index; i++) {
+        if (mapView[i].rawId() == pixelClustering::invalidModuleId) {
+          mapView[i].moduleId() = pixelClustering::invalidModuleId;
+        } else {
+          auto gdet = trackerGeom->idToDetUnit(mapView[i].rawId());
+          if (!gdet) {
+            LogDebug("SiPixelCablingSoAESProducer") << " Not found: " << mapView[i].rawId() << std::endl;
+            continue;
+          }
+          mapView[i].moduleId() = gdet->index();
+        }
+        LogDebug("SiPixelCablingSoAESProducer")
+            << "----------------------------------------------------------------------------" << std::endl;
+        LogDebug("SiPixelCablingSoAESProducer") << i << std::setw(20) << mapView[i].fed() << std::setw(20)
+                                                << mapView[i].link() << std::setw(20) << mapView[i].roc() << std::endl;
+        LogDebug("SiPixelCablingSoAESProducer")
+            << i << std::setw(20) << mapView[i].rawId() << std::setw(20) << mapView[i].rocInDet() << std::setw(20)
+            << mapView[i].moduleId() << std::endl;
+        LogDebug("SiPixelCablingSoAESProducer")
+            << i << std::setw(20) << mapView[i].badRocs() << std::setw(20) << std::endl;
+        LogDebug("SiPixelCablingSoAESProducer")
+            << "----------------------------------------------------------------------------" << std::endl;
+      }
+
+      mapView.size() = index - 1;
+
+      return product;
+    }
+
+  private:
+    edm::ESGetToken<SiPixelFedCablingMap, SiPixelFedCablingMapRcd> cablingMapToken_;
+    edm::ESGetToken<SiPixelQuality, SiPixelQualityRcd> qualityToken_;
+    edm::ESGetToken<TrackerGeometry, TrackerDigiGeometryRecord> geometryToken_;
+    const bool useQuality_;
+  };
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/ModuleFactory.h"
+DEFINE_FWK_EVENTSETUP_ALPAKA_MODULE(SiPixelCablingSoAESProducer);

CalibTracker/SiPixelESProducers/plugins/alpaka/SiPixelGainCalibrationForHLTSoAESProducer.cc

@@ -0,0 +1,128 @@
+#include "CondFormats/SiPixelObjects/interface/SiPixelGainCalibrationForHLTHost.h"
+#include "CalibTracker/Records/interface/SiPixelGainCalibrationForHLTSoARcd.h"
+#include "CondFormats/DataRecord/interface/SiPixelGainCalibrationForHLTRcd.h"
+#include "CondFormats/SiPixelObjects/interface/SiPixelGainCalibrationForHLT.h"
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+#include "FWCore/Framework/interface/ESProducer.h"
+#include "FWCore/Framework/interface/EventSetup.h"
+#include "FWCore/Framework/interface/ESHandle.h"
+#include "FWCore/Framework/interface/ModuleFactory.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "Geometry/TrackerGeometryBuilder/interface/TrackerGeometry.h"
+#include "Geometry/Records/interface/TrackerDigiGeometryRecord.h"
+#include "Geometry/CommonDetUnit/interface/GeomDetType.h"
+
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/ESProducer.h"
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/ModuleFactory.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/memory.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+
+#include <memory>
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  class SiPixelGainCalibrationForHLTSoAESProducer : public ESProducer {
+  public:
+    explicit SiPixelGainCalibrationForHLTSoAESProducer(const edm::ParameterSet& iConfig);
+    std::unique_ptr<SiPixelGainCalibrationForHLTHost> produce(const SiPixelGainCalibrationForHLTSoARcd& iRecord);
+
+    static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
+
+  private:
+    edm::ESGetToken<SiPixelGainCalibrationForHLT, SiPixelGainCalibrationForHLTRcd> gainsToken_;
+    edm::ESGetToken<TrackerGeometry, TrackerDigiGeometryRecord> geometryToken_;
+  };
+
+  SiPixelGainCalibrationForHLTSoAESProducer::SiPixelGainCalibrationForHLTSoAESProducer(const edm::ParameterSet& iConfig)
+      : ESProducer(iConfig) {
+    auto cc = setWhatProduced(this);
+    gainsToken_ = cc.consumes();
+    geometryToken_ = cc.consumes();
+  }
+
+  void SiPixelGainCalibrationForHLTSoAESProducer::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
+    edm::ParameterSetDescription desc;
+    descriptions.addWithDefaultLabel(desc);
+  }
+
+  std::unique_ptr<SiPixelGainCalibrationForHLTHost> SiPixelGainCalibrationForHLTSoAESProducer::produce(
+      const SiPixelGainCalibrationForHLTSoARcd& iRecord) {
+    auto const& gains = iRecord.get(gainsToken_);
+    auto const& geom = iRecord.get(geometryToken_);
+
+    auto product = std::make_unique<SiPixelGainCalibrationForHLTHost>(gains.data().size(), cms::alpakatools::host());
+
+    // bizzarre logic (looking for fist strip-det) don't ask
+    auto const& dus = geom.detUnits();
+    unsigned int n_detectors = dus.size();
+    for (unsigned int i = 1; i < 7; ++i) {
+      const auto offset = geom.offsetDU(GeomDetEnumerators::tkDetEnum[i]);
+      if (offset != dus.size() && dus[offset]->type().isTrackerStrip()) {
+        if (n_detectors > offset)
+          n_detectors = offset;
+      }
+    }
+
+    LogDebug("SiPixelGainCalibrationForHLTSoA")
+        << "caching calibs for " << n_detectors << " pixel detectors of size " << gains.data().size() << '\n'
+        << "sizes " << sizeof(char) << ' ' << sizeof(uint8_t) << ' ' << sizeof(siPixelGainsSoA::DecodingStructure);
+
+    for (size_t i = 0; i < gains.data().size(); i = i + 2) {
+      product->view().v_pedestals()[i / 2].gain = gains.data()[i];
+      product->view().v_pedestals()[i / 2].ped = gains.data()[i + 1];
+    }
+
+    //std::copy here
+    // do not read back from the (possibly write-combined) memory buffer
+    auto minPed = gains.getPedLow();
+    auto maxPed = gains.getPedHigh();
+    auto minGain = gains.getGainLow();
+    auto maxGain = gains.getGainHigh();
+    auto nBinsToUseForEncoding = 253;
+
+    // we will simplify later (not everything is needed....)
+    product->view().minPed() = minPed;
+    product->view().maxPed() = maxPed;
+    product->view().minGain() = minGain;
+    product->view().maxGain() = maxGain;
+
+    product->view().numberOfRowsAveragedOver() = 80;
+    product->view().nBinsToUseForEncoding() = nBinsToUseForEncoding;
+    product->view().deadFlag() = 255;
+    product->view().noisyFlag() = 254;
+
+    product->view().pedPrecision() = static_cast<float>(maxPed - minPed) / nBinsToUseForEncoding;
+    product->view().gainPrecision() = static_cast<float>(maxGain - minGain) / nBinsToUseForEncoding;
+
+    LogDebug("SiPixelGainCalibrationForHLTSoA")
+        << "precisions g " << product->view().pedPrecision() << ' ' << product->view().gainPrecision();
+
+    // fill the index map
+    auto const& ind = gains.getIndexes();
+    LogDebug("SiPixelGainCalibrationForHLTSoA") << ind.size() << " " << n_detectors;
+
+    for (auto i = 0U; i < n_detectors; ++i) {
+      auto p = std::lower_bound(
+          ind.begin(), ind.end(), dus[i]->geographicalId().rawId(), SiPixelGainCalibrationForHLT::StrictWeakOrdering());
+      assert(p != ind.end() && p->detid == dus[i]->geographicalId());
+      assert(p->iend <= gains.data().size());
+      assert(p->iend >= p->ibegin);
+      assert(0 == p->ibegin % 2);
+      assert(0 == p->iend % 2);
+      assert(p->ibegin != p->iend);
+      assert(p->ncols > 0);
+
+      product->view().modStarts()[i] = p->ibegin;
+      product->view().modEnds()[i] = p->iend;
+      product->view().modCols()[i] = p->ncols;
+
+      if (ind[i].detid != dus[i]->geographicalId())
+        LogDebug("SiPixelGainCalibrationForHLTSoA") << ind[i].detid << "!=" << dus[i]->geographicalId();
+    }
+
+    return product;
+  }
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+DEFINE_FWK_EVENTSETUP_ALPAKA_MODULE(SiPixelGainCalibrationForHLTSoAESProducer);

CondFormats/SiPixelObjects/BuildFile.xml

@@ -1,3 +1,4 @@
+<use name="alpaka"/>
 <use name="DataFormats/DetId"/>
 <use name="DataFormats/SiPixelDetId"/>
 <use name="CalibFormats/SiPixelObjects"/>
@@ -12,6 +13,9 @@
 <use name="FWCore/MessageLogger"/>
 <use name="Geometry/CommonDetUnit"/>
 <use name="Geometry/CommonTopologies"/>
+<use name="HeterogeneousCore/AlpakaInterface"/>
+<use name="HeterogeneousCore/AlpakaCore"/>
+<flags ALPAKA_BACKENDS="1"/>
 <export>
   <lib name="1"/>
 </export>

CondFormats/SiPixelObjects/interface/SiPixelGainCalibrationForHLTHost.h

@@ -0,0 +1,9 @@
+#ifndef CondFormats_SiPixelObjects_SiPixelGainCalibrationForHLTHost_h
+#define CondFormats_SiPixelObjects_SiPixelGainCalibrationForHLTHost_h
+
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "CondFormats/SiPixelObjects/interface/SiPixelGainCalibrationForHLTLayout.h"
+
+using SiPixelGainCalibrationForHLTHost = PortableHostCollection<SiPixelGainCalibrationForHLTSoA>;
+
+#endif  // CondFormats_SiPixelObjects_SiPixelGainCalibrationForHLTHost_h