Skip to content

Commit

Permalink
Introduce Cluster Charge Cut, optimize Histogram (bucket sorting) (#171)
Browse files Browse the repository at this point in the history
  • Loading branch information
VinInn authored and fwyzard committed Nov 6, 2020
1 parent 3e69838 commit cbe13d0
Show file tree
Hide file tree
Showing 21 changed files with 456 additions and 207 deletions.
2 changes: 2 additions & 0 deletions CalibTracker/SiPixelESProducers/BuildFile.xml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
<use name="DataFormats/SiPixelDigi"/>
<use name="CalibTracker/Records"/>
<use name="MagneticField/VolumeBasedEngine"/>
<use name="HeterogeneousCore/CUDACore"/>
<use name="boost"/>
<use name="cuda-api-wrappers"/>
<export>
<lib name="1"/>
</export>
2 changes: 2 additions & 0 deletions CalibTracker/SiPixelESProducers/plugins/BuildFile.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
<use name="Geometry/Records"/>
<use name="Geometry/TrackerGeometryBuilder"/>
<use name="CalibTracker/SiPixelESProducers"/>
<use name="HeterogeneousCore/CUDACore"/>
<use name="cuda-api-wrappers"/>
<library file="*.cc" name="CalibTrackerSiPixelESProducersPlugins">
<flags EDM_PLUGIN="1"/>
</library>
3 changes: 2 additions & 1 deletion Configuration/StandardSequences/python/RawToDigi_cff.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@
RawToDigi_pixelOnly = cms.Sequence(siPixelDigis)

scalersRawToDigi.scalersInputTag = 'rawDataCollector'
siPixelDigis.InputLabel = 'rawDataCollector'
from Configuration.ProcessModifiers.gpu_cff import gpu
(~gpu).toModify(siPixelDigis, InputLabel = 'rawDataCollector')
#false by default anyways ecalDigis.DoRegional = False
ecalDigis.InputLabel = 'rawDataCollector'
ecalPreshowerDigis.sourceTag = 'rawDataCollector'
Expand Down
230 changes: 131 additions & 99 deletions DQM/Integration/python/clients/beampixel_dqm_sourceclient-live_cfg.py

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -114,45 +114,36 @@
# Heavy-Ion run
if (process.runType.getRunType() == process.runType.hi_run):
process.DQMStore.referenceFileName = "/dqmdata/dqm/reference/l1t_reference_hi.root"
process.onlineMetaDataDigis.onlineMetaDataInputLabel = cms.InputTag("rawDataRepacker")
process.onlineMetaDataRawToDigi.onlineMetaDataInputLabel = cms.InputTag("rawDataRepacker")
process.castorDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.ctppsDiamondRawToDigi.rawDataTag = cms.InputTag("rawDataRepacker")
process.ctppsPixelDigis.inputLabel = cms.InputTag("rawDataRepacker")
process.ctppsPixelDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.ecalDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.ecalPreshowerDigis.sourceTag = cms.InputTag("rawDataRepacker")
process.hcalDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.muonCSCDigis.InputObjects = cms.InputTag("rawDataRepacker")
process.muonDTDigis.inputLabel = cms.InputTag("rawDataRepacker")
process.muonRPCDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.muonGEMDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.scalersRawToDigi.scalersInputTag = cms.InputTag("rawDataRepacker")
process.siPixelDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.siStripDigis.ProductLabel = cms.InputTag("rawDataRepacker")
process.tcdsDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.tcdsRawToDigi.InputLabel = cms.InputTag("rawDataRepacker")
process.totemRPRawToDigi.rawDataTag = cms.InputTag("rawDataRepacker")
process.totemTriggerRawToDigi.rawDataTag = cms.InputTag("rawDataRepacker")
process.totemTimingRawToDigi.rawDataTag = cms.InputTag("rawDataRepacker")
process.csctfDigis.producer = cms.InputTag("rawDataRepacker")
process.dttfDigis.DTTF_FED_Source = cms.InputTag("rawDataRepacker")
process.gctDigis.inputLabel = cms.InputTag("rawDataRepacker")
process.gtDigis.DaqGtInputTag = cms.InputTag("rawDataRepacker")
process.twinMuxStage2Digis.DTTM7_FED_Source = cms.InputTag("rawDataRepacker")
process.RPCTwinMuxRawToDigi.inputTag = cms.InputTag("rawDataRepacker")
process.bmtfDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.omtfStage2Digis.inputLabel = cms.InputTag("rawDataRepacker")
process.emtfStage2Digis.InputLabel = cms.InputTag("rawDataRepacker")
process.gmtStage2Digis.InputLabel = cms.InputTag("rawDataRepacker")
process.caloLayer1Digis.InputLabel = cms.InputTag("rawDataRepacker")
process.caloStage1Digis.InputLabel = cms.InputTag("rawDataRepacker")
process.caloStage2Digis.InputLabel = cms.InputTag("rawDataRepacker")
process.gtStage2Digis.InputLabel = cms.InputTag("rawDataRepacker")
process.l1tStage2CaloLayer1.fedRawDataLabel = cms.InputTag("rawDataRepacker")
process.l1tStage2uGMTZeroSupp.rawData = cms.InputTag("rawDataRepacker")
process.l1tStage2uGMTZeroSuppFatEvts.rawData = cms.InputTag("rawDataRepacker")
process.l1tStage2BmtfZeroSupp.rawData = cms.InputTag("rawDataRepacker")
process.l1tStage2BmtfZeroSuppFatEvts.rawData = cms.InputTag("rawDataRepacker")
process.selfFatEventFilter.rawInput = cms.InputTag("rawDataRepacker")

#--------------------------------------------------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,37 +110,30 @@

# Heavy-Ion run
if (process.runType.getRunType() == process.runType.hi_run):
process.onlineMetaDataDigis.onlineMetaDataInputLabel = cms.InputTag("rawDataRepacker")
process.onlineMetaDataRawToDigi.onlineMetaDataInputLabel = cms.InputTag("rawDataRepacker")
process.castorDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.ctppsDiamondRawToDigi.rawDataTag = cms.InputTag("rawDataRepacker")
process.ctppsPixelDigis.inputLabel = cms.InputTag("rawDataRepacker")
process.ctppsPixelDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.ecalDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.ecalPreshowerDigis.sourceTag = cms.InputTag("rawDataRepacker")
process.hcalDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.muonCSCDigis.InputObjects = cms.InputTag("rawDataRepacker")
process.muonDTDigis.inputLabel = cms.InputTag("rawDataRepacker")
process.muonRPCDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.muonGEMDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.scalersRawToDigi.scalersInputTag = cms.InputTag("rawDataRepacker")
process.siPixelDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.siStripDigis.ProductLabel = cms.InputTag("rawDataRepacker")
process.tcdsDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.tcdsRawToDigi.InputLabel = cms.InputTag("rawDataRepacker")
process.totemRPRawToDigi.rawDataTag = cms.InputTag("rawDataRepacker")
process.totemTriggerRawToDigi.rawDataTag = cms.InputTag("rawDataRepacker")
process.totemTimingRawToDigi.rawDataTag = cms.InputTag("rawDataRepacker")
process.csctfDigis.producer = cms.InputTag("rawDataRepacker")
process.dttfDigis.DTTF_FED_Source = cms.InputTag("rawDataRepacker")
process.gctDigis.inputLabel = cms.InputTag("rawDataRepacker")
process.gtDigis.DaqGtInputTag = cms.InputTag("rawDataRepacker")
process.twinMuxStage2Digis.DTTM7_FED_Source = cms.InputTag("rawDataRepacker")
process.RPCTwinMuxRawToDigi.inputTag = cms.InputTag("rawDataRepacker")
process.bmtfDigis.InputLabel = cms.InputTag("rawDataRepacker")
process.omtfStage2Digis.inputLabel = cms.InputTag("rawDataRepacker")
process.emtfStage2Digis.InputLabel = cms.InputTag("rawDataRepacker")
process.gmtStage2Digis.InputLabel = cms.InputTag("rawDataRepacker")
process.caloLayer1Digis.InputLabel = cms.InputTag("rawDataRepacker")
process.caloStage1Digis.InputLabel = cms.InputTag("rawDataRepacker")
process.caloStage2Digis.InputLabel = cms.InputTag("rawDataRepacker")
process.simHcalTriggerPrimitiveDigis.InputTagFEDRaw = cms.InputTag("rawDataRepacker")
Expand Down
8 changes: 8 additions & 0 deletions EventFilter/SiPixelRawToDigi/python/SiPixelRawToDigi_cfi.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import FWCore.ParameterSet.Config as cms
import EventFilter.SiPixelRawToDigi.siPixelRawToDigi_cfi
import RecoLocalTracker.SiPixelClusterizer.siPixelDigiHeterogeneousConverter_cfi

siPixelDigis = EventFilter.SiPixelRawToDigi.siPixelRawToDigi_cfi.siPixelRawToDigi.clone()
siPixelDigis.Timing = cms.untracked.bool(False)
Expand All @@ -20,3 +21,10 @@

from Configuration.Eras.Modifier_phase1Pixel_cff import phase1Pixel
phase1Pixel.toModify(siPixelDigis, UsePhase1=True)

_siPixelDigis_gpu = RecoLocalTracker.SiPixelClusterizer.siPixelDigiHeterogeneousConverter_cfi.siPixelDigiHeterogeneousConverter.clone()
_siPixelDigis_gpu.includeErrors = cms.bool(True)

from Configuration.ProcessModifiers.gpu_cff import gpu
gpu.toReplaceWith(siPixelDigis, _siPixelDigis_gpu)

Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,15 @@ namespace phase1PixelTopology {

constexpr uint32_t numPixsInModule = uint32_t(numRowsInModule)* uint32_t(numColsInModule);

constexpr uint32_t numberOfModules = 1856;

constexpr uint32_t layerStart[11] = {0,96,320,672,1184,1296,1408,1520,1632,1744,1856};
constexpr char const * layerName[10] = {"BL1","BL2","BL3","BL4",
"E+1", "E+2", "E+3",
"E-1", "E-2", "E-3"
};


// this is for the ROC n<512 (upgrade 1024)
constexpr inline
uint16_t divu52(uint16_t n) {
Expand Down
4 changes: 4 additions & 0 deletions RecoLocalTracker/Configuration/python/RecoLocalTracker_cff.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
striptrackerlocalreco = cms.Sequence(siStripZeroSuppression*siStripClusters*siStripMatchedRecHits)
trackerlocalreco = cms.Sequence(pixeltrackerlocalreco*striptrackerlocalreco*clusterSummaryProducer)

from Configuration.ProcessModifiers.gpu_cff import gpu
from RecoLocalTracker.SiPixelRecHits.siPixelRecHitHeterogeneous_cfi import siPixelRecHitHeterogeneous as _siPixelRecHitHeterogeneous
gpu.toReplaceWith(siPixelRecHitsPreSplitting, _siPixelRecHitHeterogeneous)

from RecoLocalTracker.SiPhase2Clusterizer.phase2TrackerClusterizer_cfi import *
from RecoLocalTracker.Phase2TrackerRecHits.Phase2StripCPEGeometricESProducer_cfi import *

Expand Down
11 changes: 10 additions & 1 deletion RecoLocalTracker/SiPixelClusterizer/plugins/BuildFile.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,16 @@
<use name="DataFormats/SiPixelDetId"/>
<use name="DataFormats/SiPixelCluster"/>
<use name="boost_serialization"/>
<use name="RecoLocalTracker/SiPixelClusterizer"/>
<use name="RecoTracker/Record"/>
<use name="CalibTracker/SiPixelESProducers"/>
<library file="*.cc" name="RecoLocalTrackerSiPixelClusterizerPlugins">
<use name="EventFilter/SiPixelRawToDigi"/>
<use name="HeterogeneousCore/Producer"/>
<use name="HeterogeneousCore/Product"/>
<use name="HeterogeneousCore/CUDACore"/>
<use name="cuda"/>
<use name="cuda-api-wrappers"/>
<use name="cub"/>
<library file="*.cc *.cu" name="RecoLocalTrackerSiPixelClusterizerPlugins">
<flags EDM_PLUGIN="1"/>
</library>
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include "HeterogeneousCore/CUDAUtilities/interface/cudaCheck.h"
#include "RecoLocalTracker/SiPixelClusterizer/plugins/gpuCalibPixel.h"
#include "RecoLocalTracker/SiPixelClusterizer/plugins/gpuClustering.h"
#include "RecoLocalTracker/SiPixelClusterizer/plugins/gpuClusterChargeCut.h"
#include "RecoLocalTracker/SiPixelClusterizer/interface/SiPixelFedCablingMapGPU.h"

// local includes
Expand Down Expand Up @@ -687,11 +688,11 @@ namespace pixelgpudetails {
cudaCheck(cudaMemcpyAsync(adc_h, adc_d, wordCounter*sizeof(uint16_t), cudaMemcpyDefault, stream.id()));
}

/*
std::cout
#ifdef GPU_DEBUG
std::cout
<< "CUDA countModules kernel launch with " << blocks
<< " blocks of " << threadsPerBlock << " threads\n";
*/
#endif

cudaCheck(cudaMemsetAsync(moduleStart_d, 0x00, sizeof(uint32_t), stream.id()));

Expand All @@ -703,10 +704,10 @@ namespace pixelgpudetails {

threadsPerBlock = 256;
blocks = MaxNumModules;
/*
#ifdef GPU_DEBUG
std::cout << "CUDA findClus kernel launch with " << blocks
<< " blocks of " << threadsPerBlock << " threads\n";
*/
#endif
cudaCheck(cudaMemsetAsync(clusInModule_d, 0, (MaxNumModules)*sizeof(uint32_t), stream.id()));
findClus<<<blocks, threadsPerBlock, 0, stream.id()>>>(
moduleInd_d,
Expand All @@ -717,6 +718,18 @@ namespace pixelgpudetails {
wordCounter);
cudaCheck(cudaGetLastError());

// apply charge cut
clusterChargeCut<<<blocks, threadsPerBlock, 0, stream.id()>>>(
moduleInd_d,
adc_d,
moduleStart_d,
clusInModule_d, moduleId_d,
clus_d,
wordCounter);
cudaCheck(cudaGetLastError());



// count the module start indices already here (instead of
// rechits) so that the number of clusters/hits can be made
// available in the rechit producer without additional points of
Expand Down
97 changes: 97 additions & 0 deletions RecoLocalTracker/SiPixelClusterizer/plugins/gpuClusterChargeCut.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#ifndef RecoLocalTracker_SiPixelClusterizer_plugins_gpuClusterChargeCut_h
#define RecoLocalTracker_SiPixelClusterizer_plugins_gpuClusterChargeCut_h

#include <cassert>
#include <cstdint>
#include <cstdio>

#include "gpuClusteringConstants.h"

#include "HeterogeneousCore/CUDAUtilities/interface/prefixScan.h"

namespace gpuClustering {

__global__ void clusterChargeCut(
uint16_t * __restrict__ id, // module id of each pixel (modified if bad cluster)
uint16_t const * __restrict__ adc, // charge of each pixel
uint32_t const * __restrict__ moduleStart, // index of the first pixel of each module
uint32_t * __restrict__ nClustersInModule, // modified: number of clusters found in each module
uint32_t const * __restrict__ moduleId, // module id of each module
int32_t * __restrict__ clusterId, // modified: cluster id of each pixel
int numElements)
{

if (blockIdx.x >= moduleStart[0])
return;

auto firstPixel = moduleStart[1 + blockIdx.x];
auto thisModuleId = id[firstPixel];
assert(thisModuleId < MaxNumModules);
assert(thisModuleId==moduleId[blockIdx.x]);

auto nclus = nClustersInModule[thisModuleId];
if (nclus==0) return;

assert(nclus<=MaxNumClustersPerModules);

#ifdef GPU_DEBUG
if (thisModuleId % 100 == 1)
if (threadIdx.x == 0)
printf("start clusterizer for module %d in block %d\n", thisModuleId, blockIdx.x);
#endif

auto first = firstPixel + threadIdx.x;

__shared__ int32_t charge[MaxNumClustersPerModules];
for (int i=threadIdx.x; i<nclus; i += blockDim.x) {
charge[i]=0;
}
__syncthreads();

for (int i = first; i < numElements; i += blockDim.x) {
if (id[i] == InvId) continue; // not valid
if (id[i] != thisModuleId) break; // end of module
atomicAdd(&charge[clusterId[i]], adc[i]);
}
__syncthreads();

auto chargeCut = thisModuleId<96 ? 2000 : 4000; // move in constants (calib?)
__shared__ uint8_t ok[MaxNumClustersPerModules];
__shared__ uint16_t newclusId[MaxNumClustersPerModules];
for (int i=threadIdx.x; i<nclus; i += blockDim.x) {
newclusId[i] = ok[i] = charge[i]>chargeCut ? 1 : 0;
}

__syncthreads();

// renumber
__shared__ uint16_t ws[32];
blockPrefixScan(newclusId, nclus, ws);

assert(nclus>=newclusId[nclus-1]);

if(nclus==newclusId[nclus-1]) return;

nClustersInModule[thisModuleId] = newclusId[nclus-1];
__syncthreads();

// mark bad cluster again
for (int i=threadIdx.x; i<nclus; i += blockDim.x) {
if (0==ok[i]) newclusId[i]=InvId+1;
}
__syncthreads();

// reassign id
for (int i = first; i < numElements; i += blockDim.x) {
if (id[i] == InvId) continue; // not valid
if (id[i] != thisModuleId) break; // end of module
clusterId[i] = newclusId[clusterId[i]]-1;
if(clusterId[i]==InvId) id[i] = InvId;
}

//done
}


} // namespace
#endif
Loading

0 comments on commit cbe13d0

Please sign in to comment.