Skip to content

Commit

Permalink
Merge pull request #43294 from PixelTracksAlpaka/alpaka_port_configs
Browse files Browse the repository at this point in the history
Pixel Alpaka Migration: Configs and Fixes [VII]
  • Loading branch information
cmsbuild authored Jan 30, 2024
2 parents ff42400 + c426124 commit 2e8c612
Show file tree
Hide file tree
Showing 49 changed files with 1,795 additions and 219 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import FWCore.ParameterSet.Config as cms

# This modifier chain is for turning on DQM modules used for alpaka device/host validation for pixels

alpakaValidationPixel = cms.Modifier()

11 changes: 11 additions & 0 deletions Configuration/ProcessModifiers/python/alpakaValidation_cff.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import FWCore.ParameterSet.Config as cms

from Configuration.ProcessModifiers.alpaka_cff import *
from Configuration.ProcessModifiers.alpakaValidationPixel_cff import *

# This modifier chain is for turning on DQM modules used for alpaka device/host validation

alpakaValidation = cms.ModifierChain(
alpaka,
alpakaValidationPixel
)
Original file line number Diff line number Diff line change
Expand Up @@ -896,6 +896,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
# - HLT on CPU
# - Pixel-only reconstruction on CPU, with DQM and validation
# - harvesting

upgradeWFs['PatatrackPixelOnlyCPU'] = PatatrackWorkflow(
digi = {
# the HLT menu is already set up for using GPUs if available and if the "gpu" modifier is enabled
Expand Down Expand Up @@ -1062,8 +1063,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
upgradeWFs['PatatrackECALOnlyAlpaka'] = PatatrackWorkflow(
digi = {
# customize the ECAL Local Reco part of the HLT menu for Alpaka
'--procModifiers': 'alpaka',
'--customise' : 'HLTrigger/Configuration/customizeHLTforAlpaka.customizeHLTforAlpakaEcalLocalReco'
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_ecalOnly,RECO:reconstruction_ecalOnly,VALIDATION:@ecalOnlyValidation,DQM:@ecalOnly',
Expand Down Expand Up @@ -1536,6 +1536,53 @@ def setup_(self, step, stepName, stepDict, k, properties):
offset = 0.597,
)


# Alpaka workflows

upgradeWFs['PatatrackPixelOnlyAlpaka'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
'--procModifiers': 'alpaka'
},
harvest = {
'-s': 'HARVESTING:@trackingOnlyValidation+@pixelTrackingOnlyDQM'
},
suffix = 'Patatrack_PixelOnlyAlpaka',
offset = 0.402,
)

upgradeWFs['PatatrackPixelOnlyAlpakaValidation'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
'--procModifiers': 'alpakaValidation'
},
harvest = {
'-s': 'HARVESTING:@trackingOnlyValidation+@pixelTrackingOnlyDQM'
},
suffix = 'Patatrack_PixelOnlyAlpaka_Validation',
offset = 0.403,
)

upgradeWFs['PatatrackPixelOnlyAlpakaProfiling'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly',
'--procModifiers': 'alpaka',
'--customise' : 'RecoTracker/Configuration/customizePixelOnlyForProfiling.customizePixelOnlyForProfilingGPUOnly'
},
harvest = None,
suffix = 'Patatrack_PixelOnlyAlpaka_Profiling',
offset = 0.404,
)

# end of Patatrack workflows

class UpgradeWorkflow_ProdLike(UpgradeWorkflow):
Expand Down Expand Up @@ -2741,31 +2788,31 @@ def condition(self, fragment, stepList, key, hasHarvest):
},
'2022HI' : {
'Geom' : 'DB:Extended',
'GT':'auto:phase1_2022_realistic_hi',
'GT':'auto:phase1_2022_realistic_hi',
'HLTmenu': '@fake2',
'Era':'Run3_pp_on_PbPb',
'BeamSpot': 'DBrealistic',
'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano','ALCA'],
},
'2022HIRP' : {
'Geom' : 'DB:Extended',
'GT':'auto:phase1_2022_realistic_hi',
'GT':'auto:phase1_2022_realistic_hi',
'HLTmenu': '@fake2',
'Era':'Run3_pp_on_PbPb_approxSiStripClusters',
'BeamSpot': 'DBrealistic',
'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano','ALCA'],
},
'2023HI' : {
'Geom' : 'DB:Extended',
'GT':'auto:phase1_2023_realistic_hi',
'GT':'auto:phase1_2023_realistic_hi',
'HLTmenu': '@fake2',
'Era':'Run3_pp_on_PbPb',
'BeamSpot': 'DBrealistic',
'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano','ALCA'],
},
'2023HIRP' : {
'Geom' : 'DB:Extended',
'GT':'auto:phase1_2023_realistic_hi',
'GT':'auto:phase1_2023_realistic_hi',
'HLTmenu': '@fake2',
'Era':'Run3_pp_on_PbPb_approxSiStripClusters',
'BeamSpot': 'DBrealistic',
Expand Down
26 changes: 13 additions & 13 deletions DQM/SiPixelHeterogeneous/plugins/SiPixelCompareTrackSoAAlpaka.cc
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ void SiPixelCompareTrackSoAAlpaka<T>::analyze(const edm::Event& iEvent, const ed

float ptHost = tsoaHost.view()[it].pt();
float etaHost = tsoaHost.view()[it].eta();
float phiHost = helper::phi(tsoaHost.view(), it);
float zipHost = helper::zip(tsoaHost.view(), it);
float tipHost = helper::tip(tsoaHost.view(), it);
float phiHost = reco::phi(tsoaHost.view(), it);
float zipHost = reco::zip(tsoaHost.view(), it);
float tipHost = reco::tip(tsoaHost.view(), it);

if (!(ptHost > 0.))
continue;
Expand All @@ -193,7 +193,7 @@ void SiPixelCompareTrackSoAAlpaka<T>::analyze(const edm::Event& iEvent, const ed

for (auto gid : looseTrkidxDevice) {
float etaDevice = tsoaDevice.view()[gid].eta();
float phiDevice = helper::phi(tsoaDevice.view(), gid);
float phiDevice = reco::phi(tsoaDevice.view(), gid);
float dr2 = reco::deltaR2(etaHost, phiHost, etaDevice, phiDevice);
if (dr2 > dr2cut_)
continue; // this is arbitrary
Expand All @@ -210,22 +210,22 @@ void SiPixelCompareTrackSoAAlpaka<T>::analyze(const edm::Event& iEvent, const ed
nLooseAndAboveTracksHost_matchedDevice++;

hchi2_->Fill(tsoaHost.view()[it].chi2(), tsoaDevice.view()[closestTkidx].chi2());
hCharge_->Fill(reco::charge<T>(tsoaHost.view(), it), reco::charge<T>(tsoaDevice.view(), closestTkidx));
hCharge_->Fill(reco::charge(tsoaHost.view(), it), reco::charge(tsoaDevice.view(), closestTkidx));
hnHits_->Fill(helper::nHits(tsoaHost.view(), it), helper::nHits(tsoaDevice.view(), closestTkidx));
hnLayers_->Fill(tsoaHost.view()[it].nLayers(), tsoaDevice.view()[closestTkidx].nLayers());
hpt_->Fill(tsoaHost.view()[it].pt(), tsoaDevice.view()[closestTkidx].pt());
hptLogLog_->Fill(tsoaHost.view()[it].pt(), tsoaDevice.view()[closestTkidx].pt());
heta_->Fill(etaHost, tsoaDevice.view()[closestTkidx].eta());
hphi_->Fill(phiHost, helper::phi(tsoaDevice.view(), closestTkidx));
hz_->Fill(zipHost, helper::zip(tsoaDevice.view(), closestTkidx));
htip_->Fill(tipHost, helper::tip(tsoaDevice.view(), closestTkidx));
hphi_->Fill(phiHost, reco::phi(tsoaDevice.view(), closestTkidx));
hz_->Fill(zipHost, reco::zip(tsoaDevice.view(), closestTkidx));
htip_->Fill(tipHost, reco::tip(tsoaDevice.view(), closestTkidx));
hptdiffMatched_->Fill(ptHost - tsoaDevice.view()[closestTkidx].pt());
hCurvdiffMatched_->Fill((reco::charge<T>(tsoaHost.view(), it) / tsoaHost.view()[it].pt()) -
(reco::charge<T>(tsoaDevice.view(), closestTkidx) / tsoaDevice.view()[closestTkidx].pt()));
hCurvdiffMatched_->Fill((reco::charge(tsoaHost.view(), it) / tsoaHost.view()[it].pt()) -
(reco::charge(tsoaDevice.view(), closestTkidx) / tsoaDevice.view()[closestTkidx].pt()));
hetadiffMatched_->Fill(etaHost - tsoaDevice.view()[closestTkidx].eta());
hphidiffMatched_->Fill(reco::deltaPhi(phiHost, helper::phi(tsoaDevice.view(), closestTkidx)));
hzdiffMatched_->Fill(zipHost - helper::zip(tsoaDevice.view(), closestTkidx));
htipdiffMatched_->Fill(tipHost - helper::tip(tsoaDevice.view(), closestTkidx));
hphidiffMatched_->Fill(reco::deltaPhi(phiHost, reco::phi(tsoaDevice.view(), closestTkidx)));
hzdiffMatched_->Fill(zipHost - reco::zip(tsoaDevice.view(), closestTkidx));
htipdiffMatched_->Fill(tipHost - reco::tip(tsoaDevice.view(), closestTkidx));
hpt_eta_tkAllHostMatched_->Fill(etaHost, tsoaHost.view()[it].pt()); //matched to gpu
hphi_z_tkAllHostMatched_->Fill(etaHost, zipHost);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,35 @@
from DQM.SiPixelHeterogeneous.siPixelPhase2MonitorTrackSoA_cfi import *
from DQM.SiPixelHeterogeneous.siPixelHIonPhase1MonitorTrackSoA_cfi import *
from DQM.SiPixelHeterogeneous.siPixelMonitorVertexSoA_cfi import *
# Alpaka Modules
from Configuration.ProcessModifiers.alpaka_cff import alpaka
from DQM.SiPixelHeterogeneous.siPixelPhase1MonitorRecHitsSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelPhase2MonitorRecHitsSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelHIonPhase1MonitorRecHitsSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelPhase1MonitorTrackSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelPhase2MonitorTrackSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelHIonPhase1MonitorTrackSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelMonitorVertexSoAAlpaka_cfi import *

# Run-3 sequence
monitorpixelSoASource = cms.Sequence(siPixelPhase1MonitorRecHitsSoA * siPixelPhase1MonitorTrackSoA * siPixelMonitorVertexSoA)

# Run-3 Alpaka sequence
monitorpixelSoASourceAlpaka = cms.Sequence(siPixelPhase1MonitorRecHitsSoAAlpaka * siPixelPhase1MonitorTrackSoAAlpaka * siPixelMonitorVertexSoAAlpaka)
alpaka.toReplaceWith(monitorpixelSoASource, monitorpixelSoASourceAlpaka)
# Phase-2 sequence
from Configuration.Eras.Modifier_phase2_tracker_cff import phase2_tracker
_monitorpixelSoARecHitsSource = cms.Sequence(siPixelPhase2MonitorRecHitsSoA * siPixelPhase2MonitorTrackSoA * siPixelMonitorVertexSoA)
phase2_tracker.toReplaceWith(monitorpixelSoASource, _monitorpixelSoARecHitsSource)
(phase2_tracker & ~alpaka).toReplaceWith(monitorpixelSoASource, _monitorpixelSoARecHitsSource)
_monitorpixelSoARecHitsSourceAlpaka = cms.Sequence(siPixelPhase2MonitorRecHitsSoAAlpaka * siPixelPhase2MonitorTrackSoAAlpaka * siPixelMonitorVertexSoAAlpaka)
(phase2_tracker & alpaka).toReplaceWith(monitorpixelSoASource, _monitorpixelSoARecHitsSourceAlpaka)

# HIon Phase 1 sequence
from Configuration.ProcessModifiers.pp_on_AA_cff import pp_on_AA

_monitorpixelSoARecHitsSourceHIon = cms.Sequence(siPixelHIonPhase1MonitorRecHitsSoA * siPixelHIonPhase1MonitorTrackSoA * siPixelMonitorVertexSoA)
(pp_on_AA & ~phase2_tracker).toReplaceWith(monitorpixelSoASource, _monitorpixelSoARecHitsSourceHIon)
_monitorpixelSoARecHitsSourceHIonAlpaka = cms.Sequence(siPixelHIonPhase1MonitorRecHitsSoAAlpaka * siPixelHIonPhase1MonitorTrackSoAAlpaka * siPixelMonitorVertexSoAAlpaka)
(pp_on_AA & ~phase2_tracker & alpaka).toReplaceWith(monitorpixelSoASource, _monitorpixelSoARecHitsSourceHIonAlpaka)

#Define the sequence for GPU vs CPU validation
#This should run:- individual monitor for the 2 collections + comparison module
Expand All @@ -33,6 +48,14 @@
from DQM.SiPixelHeterogeneous.siPixelCompareVertexSoA_cfi import *
from DQM.SiPixelHeterogeneous.siPixelPhase1RawDataErrorComparator_cfi import *
from DQM.SiPixelPhase1Common.SiPixelPhase1RawData_cfi import *
#Alpaka
from DQM.SiPixelHeterogeneous.siPixelPhase1CompareRecHitsSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelPhase2CompareRecHitsSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelHIonPhase1CompareRecHitsSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelPhase1CompareTrackSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelPhase2CompareTrackSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelHIonPhase1CompareTrackSoAAlpaka_cfi import *
from DQM.SiPixelHeterogeneous.siPixelCompareVertexSoAAlpaka_cfi import *

# digi errors
SiPixelPhase1RawDataConfForCPU = copy.deepcopy(SiPixelPhase1RawDataConf)
Expand Down Expand Up @@ -126,6 +149,43 @@
topFolderName = 'SiPixelHeterogeneous/PixelVertexSoAGPU',
)

### Alpaka

# PixelRecHits: monitor of CPUSerial product (Alpaka backend: 'serial_sync')
siPixelRecHitsSoAMonitorSerial = siPixelPhase1MonitorRecHitsSoAAlpaka.clone(
pixelHitsSrc = cms.InputTag( 'siPixelRecHitsPreSplittingAlpakaSerial' ),
TopFolderName = cms.string( 'SiPixelHeterogeneous/PixelRecHitsSerial' )
)

# PixelRecHits: monitor of Device product (Alpaka backend: '')
siPixelRecHitsSoAMonitorDevice = siPixelPhase1MonitorRecHitsSoAAlpaka.clone(
pixelHitsSrc = cms.InputTag( 'siPixelRecHitsPreSplittingAlpaka' ),
TopFolderName = cms.string( 'SiPixelHeterogeneous/PixelRecHitsDevice' )
)

# PixelTracks: monitor of CPUSerial product (Alpaka backend: 'serial_sync')
siPixelTrackSoAMonitorSerial = siPixelPhase1MonitorTrackSoAAlpaka.clone(
pixelTrackSrc = cms.InputTag('pixelTracksAlpakaSerial'),
topFolderName = cms.string('SiPixelHeterogeneous/PixelTrackSerial')
)

# PixelTracks: monitor of CPUSerial product (Alpaka backend: 'serial_sync')
siPixelTrackSoAMonitorDevice = siPixelPhase1MonitorTrackSoAAlpaka.clone(
pixelTrackSrc = cms.InputTag('pixelTracksAlpaka'),
topFolderName = cms.string('SiPixelHeterogeneous/PixelTrackDevice')
)

# PixelVertices: monitor of CPUSerial product (Alpaka backend: 'serial_sync')
siPixelVertexSoAMonitorSerial = siPixelMonitorVertexSoAAlpaka.clone(
pixelVertexSrc = cms.InputTag("pixelVerticesAlpakaSerial"),
topFolderName = cms.string('SiPixelHeterogeneous/PixelVertexSerial')
)

siPixelVertexSoAMonitorDevice = siPixelMonitorVertexSoAAlpaka.clone(
pixelVertexSrc = cms.InputTag("pixelVerticesAlpaka"),
topFolderName = cms.string('SiPixelHeterogeneous/PixelVertexDevice')
)

# Run-3 sequence
monitorpixelSoACompareSource = cms.Sequence(siPixelPhase1MonitorRawDataACPU *
siPixelPhase1MonitorRawDataAGPU *
Expand All @@ -139,6 +199,17 @@
siPixelMonitorVertexSoAGPU *
siPixelCompareVertexSoA *
siPixelPhase1RawDataErrorComparator)
# and the Alpaka version
monitorpixelSoACompareSourceAlpaka = cms.Sequence(
siPixelRecHitsSoAMonitorSerial *
siPixelRecHitsSoAMonitorDevice *
siPixelPhase1CompareRecHitsSoAAlpaka *
siPixelTrackSoAMonitorSerial *
siPixelTrackSoAMonitorDevice *
siPixelPhase1CompareTrackSoAAlpaka *
siPixelVertexSoAMonitorSerial *
siPixelVertexSoAMonitorDevice *
siPixelCompareVertexSoAAlpaka )

# Phase-2 sequence
_monitorpixelSoACompareSource = cms.Sequence(siPixelPhase2MonitorRecHitsSoACPU *
Expand Down Expand Up @@ -166,3 +237,6 @@

from Configuration.ProcessModifiers.gpuValidationPixel_cff import gpuValidationPixel
gpuValidationPixel.toReplaceWith(monitorpixelSoASource, monitorpixelSoACompareSource)

from Configuration.ProcessModifiers.alpakaValidationPixel_cff import alpakaValidationPixel
(alpakaValidationPixel & ~gpuValidationPixel).toReplaceWith(monitorpixelSoASource, monitorpixelSoACompareSourceAlpaka)
58 changes: 52 additions & 6 deletions DataFormats/TrackSoA/interface/TracksSoA.h
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
#ifndef DataFormats_Track_interface_TrackLayout_h
#define DataFormats_Track_interface_TrackLayout_h
#ifndef DataFormats_TrackSoA_interface_TracksSoA_h
#define DataFormats_TrackSoA_interface_TracksSoA_h

#include <alpaka/alpaka.hpp>

#include <Eigen/Core>

#include "HeterogeneousCore/AlpakaInterface/interface/OneToManyAssoc.h"
#include "Geometry/CommonTopologies/interface/SimplePixelTopology.h"
#include "DataFormats/SoATemplate/interface/SoALayout.h"
Expand Down Expand Up @@ -29,6 +32,7 @@ namespace reco {
SOA_COLUMN(int8_t, nLayers),
SOA_COLUMN(float, eta),
SOA_COLUMN(float, pt),
// state at the beam spot: {phi, tip, 1/pt, cotan(theta), zip}
SOA_EIGEN_COLUMN(Vector5f, state),
SOA_EIGEN_COLUMN(Vector15f, covariance),
SOA_SCALAR(int, nTracks),
Expand All @@ -43,14 +47,56 @@ namespace reco {
template <typename TrackerTraits>
using TrackSoAConstView = typename reco::TrackSoA<TrackerTraits>::template Layout<>::ConstView;

template <typename TrackerTraits>
ALPAKA_FN_HOST_ACC ALPAKA_FN_INLINE static constexpr float charge(const TrackSoAConstView<TrackerTraits> &tracks,
int32_t i) {
/* Implement a type trait to identify the specialisations of TrackSoAConstView<TrackerTraits>
*
* This is done explicitly for all possible pixel topologies, because we did not find a way
* to use template deduction with a partial specialisation.
*/
template <typename T>
struct IsTrackSoAConstView : std::false_type {};
template <>
struct IsTrackSoAConstView<TrackSoAConstView<pixelTopology::Phase1>> : std::true_type {};
template <>
struct IsTrackSoAConstView<TrackSoAView<pixelTopology::Phase1>> : std::true_type {};
template <>
struct IsTrackSoAConstView<TrackSoAConstView<pixelTopology::Phase2>> : std::true_type {};
template <>
struct IsTrackSoAConstView<TrackSoAView<pixelTopology::Phase2>> : std::true_type {};
template <>
struct IsTrackSoAConstView<TrackSoAConstView<pixelTopology::HIonPhase1>> : std::true_type {};
template <>
struct IsTrackSoAConstView<TrackSoAView<pixelTopology::HIonPhase1>> : std::true_type {};

template <typename T>
constexpr bool isTrackSoAConstView = IsTrackSoAConstView<T>::value;

template <typename ConstView, typename = std::enable_if_t<isTrackSoAConstView<ConstView>>>
ALPAKA_FN_HOST_ACC ALPAKA_FN_INLINE constexpr float charge(ConstView const& tracks, int32_t i) {
//was: std::copysign(1.f, tracks[i].state()(2)). Will be constexpr with C++23
float v = tracks[i].state()(2);
return float((0.0f < v) - (v < 0.0f));
}

template <typename ConstView, typename = std::enable_if_t<isTrackSoAConstView<ConstView>>>
ALPAKA_FN_HOST_ACC ALPAKA_FN_INLINE constexpr float phi(ConstView const& tracks, int32_t i) {
return tracks[i].state()(0);
}

template <typename ConstView, typename = std::enable_if_t<isTrackSoAConstView<ConstView>>>
ALPAKA_FN_HOST_ACC ALPAKA_FN_INLINE constexpr float tip(ConstView const& tracks, int32_t i) {
return tracks[i].state()(1);
}

template <typename ConstView, typename = std::enable_if_t<isTrackSoAConstView<ConstView>>>
ALPAKA_FN_HOST_ACC ALPAKA_FN_INLINE constexpr float zip(ConstView const& tracks, int32_t i) {
return tracks[i].state()(4);
}

template <typename ConstView, typename = std::enable_if_t<isTrackSoAConstView<ConstView>>>
ALPAKA_FN_HOST_ACC ALPAKA_FN_INLINE constexpr bool isTriplet(ConstView const& tracks, int32_t i) {
return tracks[i].nLayers() == 3;
}

} // namespace reco

#endif
#endif // DataFormats_TrackSoA_interface_TracksSoA_h
Loading

0 comments on commit 2e8c612

Please sign in to comment.