Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set max channels separately EE and EB for ECAL #517

12 changes: 6 additions & 6 deletions EventFilter/EcalRawToDigi/plugins/DeclsForKernels.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ namespace ecal {
};

struct ConfigurationParameters {
uint32_t maxChannels;
uint32_t maxChannelsEE;
uint32_t maxChannelsEB;
};

struct OutputDataCPU {
Expand All @@ -40,11 +41,10 @@ namespace ecal {
// FIXME: we should separate max channels parameter for eb and ee
fwyzard marked this conversation as resolved.
Show resolved Hide resolved
// FIXME: replace hardcoded values
void allocate(ConfigurationParameters const &config, cudaStream_t cudaStream) {
digisEB.data = cms::cuda::make_device_unique<uint16_t[]>(config.maxChannels, cudaStream);
digisEE.data = cms::cuda::make_device_unique<uint16_t[]>(config.maxChannels, cudaStream);
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe the missing *10 here is also why in the past we were not able to validate fully the MC workflow?
... btw, not it should be fixed.


digisEB.ids = cms::cuda::make_device_unique<uint32_t[]>(config.maxChannels, cudaStream);
digisEE.ids = cms::cuda::make_device_unique<uint32_t[]>(config.maxChannels, cudaStream);
digisEB.data = cms::cuda::make_device_unique<uint16_t[]>(config.maxChannelsEB, cudaStream);
digisEE.data = cms::cuda::make_device_unique<uint16_t[]>(config.maxChannelsEE, cudaStream);
digisEB.ids = cms::cuda::make_device_unique<uint32_t[]>(config.maxChannelsEB, cudaStream);
digisEE.ids = cms::cuda::make_device_unique<uint32_t[]>(config.maxChannelsEE, cudaStream);
}
};

Expand Down
6 changes: 4 additions & 2 deletions EventFilter/EcalRawToDigi/plugins/EcalRawToDigiGPU.cc
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ void EcalRawToDigiGPU::fillDescriptions(edm::ConfigurationDescriptions& confDesc
for (uint32_t i = 0; i < 54; ++i)
feds[i] = i + 601;
desc.add<std::vector<int>>("FEDs", feds);
desc.add<uint32_t>("maxChannels", 20000);
desc.add<uint32_t>("maxChannelsEB", 61200);
desc.add<uint32_t>("maxChannelsEE", 14648);
desc.add<std::string>("digisLabelEB", "ebDigisGPU");
desc.add<std::string>("digisLabelEE", "eeDigisGPU");

Expand All @@ -61,7 +62,8 @@ EcalRawToDigiGPU::EcalRawToDigiGPU(const edm::ParameterSet& ps)
digisEBToken_{produces<OutputProduct>(ps.getParameter<std::string>("digisLabelEB"))},
digisEEToken_{produces<OutputProduct>(ps.getParameter<std::string>("digisLabelEE"))},
fedsToUnpack_{ps.getParameter<std::vector<int>>("FEDs")} {
config_.maxChannels = ps.getParameter<uint32_t>("maxChannels");
config_.maxChannelsEB = ps.getParameter<uint32_t>("maxChannelsEB");
config_.maxChannelsEE = ps.getParameter<uint32_t>("maxChannelsEE");
}

EcalRawToDigiGPU::~EcalRawToDigiGPU() {}
Expand Down
6 changes: 4 additions & 2 deletions HLTrigger/Configuration/python/customizeHLTforPatatrack.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,8 @@ def customise_gpu_ecal(process):
),
digisLabelEB = cms.string("ebDigis"),
digisLabelEE = cms.string("eeDigis"),
maxChannels = cms.uint32(20000)
maxChannelsEB = cms.uint32(61200),
maxChannelsEE = cms.uint32(14648),
)

process.hltEcalDigis = cms.EDProducer("EcalCPUDigisProducer",
Expand Down Expand Up @@ -378,7 +379,8 @@ def customise_gpu_ecal(process):
uncalibrecHitsInLabelEE = cms.InputTag("hltEcalUncalibRecHitGPU","EcalUncalibRecHitsEE"),
recHitsLabelEB = cms.string("EcalRecHitsEB"),
recHitsLabelEE = cms.string("EcalRecHitsEE"),
maxNumberHits = cms.uint32(20000),
maxNumberHitsEB = cms.uint32(61200),
maxNumberHitsEE = cms.uint32(14648),
ChannelStatusToBeExcluded = cms.vstring(
"kDAC",
"kNoisy",
Expand Down
100 changes: 40 additions & 60 deletions RecoLocalCalo/EcalRecProducers/plugins/DeclsForKernels.h
Original file line number Diff line number Diff line change
Expand Up @@ -91,41 +91,43 @@ namespace ecal {

bool shouldRunTimingComputation;

uint32_t maxNumberHits;
uint32_t maxNumberHitsEB;
uint32_t maxNumberHitsEE;
};

struct EventOutputDataGPU {
UncalibratedRecHit<::calo::common::DevStoragePolicy> recHitsEB, recHitsEE;

void allocate(ConfigurationParameters const& configParameters, cudaStream_t cudaStream) {
auto const size = configParameters.maxNumberHits;
auto const sizeEB = configParameters.maxNumberHitsEB;
recHitsEB.amplitudesAll =
cms::cuda::make_device_unique<reco::ComputationScalarType[]>(size * EcalDataFrame::MAXSAMPLES, cudaStream);
recHitsEB.amplitude = cms::cuda::make_device_unique<reco::StorageScalarType[]>(size, cudaStream);
recHitsEB.chi2 = cms::cuda::make_device_unique<reco::StorageScalarType[]>(size, cudaStream);
recHitsEB.pedestal = cms::cuda::make_device_unique<reco::StorageScalarType[]>(size, cudaStream);
cms::cuda::make_device_unique<reco::ComputationScalarType[]>(sizeEB * EcalDataFrame::MAXSAMPLES, cudaStream);
recHitsEB.amplitude = cms::cuda::make_device_unique<reco::StorageScalarType[]>(sizeEB, cudaStream);
recHitsEB.chi2 = cms::cuda::make_device_unique<reco::StorageScalarType[]>(sizeEB, cudaStream);
recHitsEB.pedestal = cms::cuda::make_device_unique<reco::StorageScalarType[]>(sizeEB, cudaStream);

if (configParameters.shouldRunTimingComputation) {
recHitsEB.jitter = cms::cuda::make_device_unique<reco::StorageScalarType[]>(size, cudaStream);
recHitsEB.jitterError = cms::cuda::make_device_unique<reco::StorageScalarType[]>(size, cudaStream);
recHitsEB.jitter = cms::cuda::make_device_unique<reco::StorageScalarType[]>(sizeEB, cudaStream);
recHitsEB.jitterError = cms::cuda::make_device_unique<reco::StorageScalarType[]>(sizeEB, cudaStream);
}

recHitsEB.did = cms::cuda::make_device_unique<uint32_t[]>(size, cudaStream);
recHitsEB.flags = cms::cuda::make_device_unique<uint32_t[]>(size, cudaStream);
recHitsEB.did = cms::cuda::make_device_unique<uint32_t[]>(sizeEB, cudaStream);
recHitsEB.flags = cms::cuda::make_device_unique<uint32_t[]>(sizeEB, cudaStream);

auto const sizeEE = configParameters.maxNumberHitsEE;
recHitsEE.amplitudesAll =
cms::cuda::make_device_unique<reco::ComputationScalarType[]>(size * EcalDataFrame::MAXSAMPLES, cudaStream);
recHitsEE.amplitude = cms::cuda::make_device_unique<reco::StorageScalarType[]>(size, cudaStream);
recHitsEE.chi2 = cms::cuda::make_device_unique<reco::StorageScalarType[]>(size, cudaStream);
recHitsEE.pedestal = cms::cuda::make_device_unique<reco::StorageScalarType[]>(size, cudaStream);
cms::cuda::make_device_unique<reco::ComputationScalarType[]>(sizeEE * EcalDataFrame::MAXSAMPLES, cudaStream);
recHitsEE.amplitude = cms::cuda::make_device_unique<reco::StorageScalarType[]>(sizeEE, cudaStream);
recHitsEE.chi2 = cms::cuda::make_device_unique<reco::StorageScalarType[]>(sizeEE, cudaStream);
recHitsEE.pedestal = cms::cuda::make_device_unique<reco::StorageScalarType[]>(sizeEE, cudaStream);

if (configParameters.shouldRunTimingComputation) {
recHitsEE.jitter = cms::cuda::make_device_unique<reco::StorageScalarType[]>(size, cudaStream);
recHitsEE.jitterError = cms::cuda::make_device_unique<reco::StorageScalarType[]>(size, cudaStream);
recHitsEE.jitter = cms::cuda::make_device_unique<reco::StorageScalarType[]>(sizeEE, cudaStream);
recHitsEE.jitterError = cms::cuda::make_device_unique<reco::StorageScalarType[]>(sizeEE, cudaStream);
}

recHitsEE.did = cms::cuda::make_device_unique<uint32_t[]>(size, cudaStream);
recHitsEE.flags = cms::cuda::make_device_unique<uint32_t[]>(size, cudaStream);
recHitsEE.did = cms::cuda::make_device_unique<uint32_t[]>(sizeEE, cudaStream);
recHitsEE.flags = cms::cuda::make_device_unique<uint32_t[]>(sizeEE, cudaStream);
}
};

Expand Down Expand Up @@ -169,64 +171,40 @@ namespace ecal {
constexpr auto smlength = getLength<SampleMatrix>();
constexpr auto pmlength = getLength<PulseMatrixType>();
constexpr auto bxvlength = getLength<BXVectorType>();
auto const size = configParameters.maxNumberHits;
auto const size = configParameters.maxNumberHitsEB + configParameters.maxNumberHitsEE;

#define MYMALLOC(var, size) var = cms::cuda::make_device_unique<decltype(var)::element_type[]>(size, cudaStream)

MYMALLOC(samples, size * svlength);
//cudaCheck(cudaMalloc((void**)&samples, size * sizeof(SampleVector)));
MYMALLOC(gainsNoise, size * sgvlength);
//cudaCheck(cudaMalloc((void**)&gainsNoise, size * sizeof(SampleGainVector)));

MYMALLOC(noisecov, size * smlength);
//cudaCheck(cudaMalloc((void**)&noisecov, size * sizeof(SampleMatrix)));
MYMALLOC(pulse_matrix, size * pmlength);
//cudaCheck(cudaMalloc((void**)&pulse_matrix, size * sizeof(PulseMatrixType)));
MYMALLOC(activeBXs, size * bxvlength);
//cudaCheck(cudaMalloc((void**)&activeBXs, size * sizeof(BXVectorType)));
MYMALLOC(acState, size);
//cudaCheck(cudaMalloc((void**)&acState, size * sizeof(char)));

MYMALLOC(hasSwitchToGain6, size);
//cudaCheck(cudaMalloc((void**)&hasSwitchToGain6, size * sizeof(bool)));
MYMALLOC(hasSwitchToGain1, size);
//cudaCheck(cudaMalloc((void**)&hasSwitchToGain1, size * sizeof(bool)));
MYMALLOC(isSaturated, size);
//cudaCheck(cudaMalloc((void**)&isSaturated, size * sizeof(bool)));

if (configParameters.shouldRunTimingComputation) {
MYMALLOC(sample_values, size * svlength);
//cudaCheck(cudaMalloc((void**)&sample_values, size * sizeof(SampleVector)));
MYMALLOC(sample_value_errors, size * svlength);
//cudaCheck(cudaMalloc((void**)&sample_value_errors, size * sizeof(SampleVector)));
MYMALLOC(useless_sample_values, size * EcalDataFrame::MAXSAMPLES);
//cudaCheck(cudaMalloc((void**)&useless_sample_values, size * sizeof(bool) * EcalDataFrame::MAXSAMPLES));
MYMALLOC(chi2sNullHypot, size);
//cudaCheck(cudaMalloc((void**)&chi2sNullHypot, size * sizeof(SampleVector::Scalar)));
MYMALLOC(sum0sNullHypot, size);
//cudaCheck(cudaMalloc((void**)&sum0sNullHypot, size * sizeof(SampleVector::Scalar)));
MYMALLOC(sumAAsNullHypot, size);
//cudaCheck(cudaMalloc((void**)&sumAAsNullHypot, size * sizeof(SampleVector::Scalar)));
MYMALLOC(pedestal_nums, size);
//cudaCheck(cudaMalloc((void**)&pedestal_nums, size * sizeof(char)));

MYMALLOC(tMaxAlphaBetas, size);
//cudaCheck(cudaMalloc((void**)&tMaxAlphaBetas, size * sizeof(SampleVector::Scalar)));
MYMALLOC(tMaxErrorAlphaBetas, size);
//cudaCheck(cudaMalloc((void**)&tMaxErrorAlphaBetas, size * sizeof(SampleVector::Scalar)));
MYMALLOC(accTimeMax, size);
//cudaCheck(cudaMalloc((void**)&accTimeMax, size * sizeof(SampleVector::Scalar)));
MYMALLOC(accTimeWgt, size);
//cudaCheck(cudaMalloc((void**)&accTimeWgt, size * sizeof(SampleVector::Scalar)));
MYMALLOC(ampMaxAlphaBeta, size);
//cudaCheck(cudaMalloc((void**)&ampMaxAlphaBeta, size * sizeof(SampleVector::Scalar)));
MYMALLOC(ampMaxError, size);
//cudaCheck(cudaMalloc((void**)&ampMaxError, size * sizeof(SampleVector::Scalar)));
MYMALLOC(timeMax, size);
//cudaCheck(cudaMalloc((void**)&timeMax, size * sizeof(SampleVector::Scalar)));
MYMALLOC(timeError, size);
//cudaCheck(cudaMalloc((void**)&timeError, size * sizeof(SampleVector::Scalar)));
MYMALLOC(tcState, size);
//cudaCheck(cudaMalloc((void**)&tcState, size * sizeof(TimeComputationState)));
}
fwyzard marked this conversation as resolved.
Show resolved Hide resolved
}
};
Expand Down Expand Up @@ -295,8 +273,9 @@ namespace ecal {
uint32_t expanded_v_DB_reco_flagsSize;

uint32_t flagmask;
uint32_t maxNumberHits;

uint32_t maxNumberHitsEB;
uint32_t maxNumberHitsEE;

//
// bool shouldRunTimingComputation;
};
Expand All @@ -307,20 +286,21 @@ namespace ecal {
void allocate(ConfigurationParameters const& configParameters, cudaStream_t cudaStream) {
// void allocate(uint32_t size) {
//---- configParameters -> needed only to decide if to save the timing information or not
auto const size = configParameters.maxNumberHits;
recHitsEB.energy = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(size, cudaStream);
recHitsEB.time = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(size, cudaStream);
recHitsEB.chi2 = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(size, cudaStream);
recHitsEB.flagBits = cms::cuda::make_device_unique<uint32_t[]>(size, cudaStream);
recHitsEB.extra = cms::cuda::make_device_unique<uint32_t[]>(size, cudaStream);
recHitsEB.did = cms::cuda::make_device_unique<uint32_t[]>(size, cudaStream);

recHitsEE.energy = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(size, cudaStream);
recHitsEE.time = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(size, cudaStream);
recHitsEE.chi2 = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(size, cudaStream);
recHitsEE.flagBits = cms::cuda::make_device_unique<uint32_t[]>(size, cudaStream);
recHitsEE.extra = cms::cuda::make_device_unique<uint32_t[]>(size, cudaStream);
recHitsEE.did = cms::cuda::make_device_unique<uint32_t[]>(size, cudaStream);
auto const sizeEB = configParameters.maxNumberHitsEB;
recHitsEB.energy = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(sizeEB, cudaStream);
recHitsEB.time = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(sizeEB, cudaStream);
recHitsEB.chi2 = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(sizeEB, cudaStream);
recHitsEB.flagBits = cms::cuda::make_device_unique<uint32_t[]>(sizeEB, cudaStream);
recHitsEB.extra = cms::cuda::make_device_unique<uint32_t[]>(sizeEB, cudaStream);
recHitsEB.did = cms::cuda::make_device_unique<uint32_t[]>(sizeEB, cudaStream);

auto const sizeEE = configParameters.maxNumberHitsEE;
recHitsEE.energy = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(sizeEE, cudaStream);
recHitsEE.time = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(sizeEE, cudaStream);
recHitsEE.chi2 = cms::cuda::make_device_unique<::ecal::reco::StorageScalarType[]>(sizeEE, cudaStream);
recHitsEE.flagBits = cms::cuda::make_device_unique<uint32_t[]>(sizeEE, cudaStream);
recHitsEE.extra = cms::cuda::make_device_unique<uint32_t[]>(sizeEE, cudaStream);
recHitsEE.did = cms::cuda::make_device_unique<uint32_t[]>(sizeEE, cudaStream);
}
};

Expand Down
12 changes: 7 additions & 5 deletions RecoLocalCalo/EcalRecProducers/plugins/EcalRecHitProducerGPU.cc
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ void EcalRecHitProducerGPU::fillDescriptions(edm::ConfigurationDescriptions& con
desc.add<double>("EBLaserMAX", 30.0);
desc.add<double>("EELaserMAX", 30.0);

desc.add<uint32_t>("maxNumberHits", 20000);
desc.add<uint32_t>("maxNumberHitsEB", 61200);
desc.add<uint32_t>("maxNumberHitsEE", 14648);
}

EcalRecHitProducerGPU::EcalRecHitProducerGPU(const edm::ParameterSet& ps) {
Expand All @@ -121,8 +122,9 @@ EcalRecHitProducerGPU::EcalRecHitProducerGPU(const edm::ParameterSet& ps) {
configParameters_.EELaserMAX = ps.getParameter<double>("EELaserMAX");

// max number of digis to allocate for
configParameters_.maxNumberHits = ps.getParameter<uint32_t>("maxNumberHits");

configParameters_.maxNumberHitsEB = ps.getParameter<uint32_t>("maxNumberHitsEB");
configParameters_.maxNumberHitsEE = ps.getParameter<uint32_t>("maxNumberHitsEE");

flagmask_ = 0;
flagmask_ |= 0x1 << EcalRecHit::kNeighboursRecovered;
flagmask_ |= 0x1 << EcalRecHit::kTowerRecovered;
Expand Down Expand Up @@ -163,8 +165,8 @@ void EcalRecHitProducerGPU::acquire(edm::Event const& event,
nee_ = eeUncalibRecHits.size;
// std::cout << " [EcalRecHitProducerGPU::acquire] neb_:nee_ = " << neb_ << " : " << nee_ << std::endl;

if ((neb_ + nee_) > configParameters_.maxNumberHits) {
edm::LogError("EcalRecHitProducerGPU") << "max number of channels exceeded. See options 'maxNumberHits' ";
if ((neb_ > configParameters_.maxNumberHitsEB) || (nee_ > configParameters_.maxNumberHitsEE)) {
edm::LogError("EcalRecHitProducerGPU") << "max number of channels exceeded. See options 'maxNumberHitsEB and maxNumberHitsEE' ";
}

int nchannelsEB = ebUncalibRecHits.size; // --> offsetForInput, first EB and then EE
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ void EcalUncalibRecHitProducerGPU::fillDescriptions(edm::ConfigurationDescriptio
desc.add<double>("outOfTimeThresholdGain61mEE", 1000);
desc.add<double>("amplitudeThresholdEB", 10);
desc.add<double>("amplitudeThresholdEE", 10);
desc.add<uint32_t>("maxNumberHits", 20000); //---- AM TEST
desc.add<uint32_t>("maxNumberHitsEB", 61200);
desc.add<uint32_t>("maxNumberHitsEE", 14648);
desc.add<std::vector<uint32_t>>("kernelMinimizeThreads", {32, 1, 1});
// ---- default false or true? It was set to true, but at HLT it is false
desc.add<bool>("shouldRunTimingComputation", false);
Expand Down Expand Up @@ -135,8 +136,9 @@ EcalUncalibRecHitProducerGPU::EcalUncalibRecHitProducerGPU(const edm::ParameterS
auto amplitudeThreshEE = ps.getParameter<double>("amplitudeThresholdEE");

// max number of digis to allocate for
configParameters_.maxNumberHits = ps.getParameter<uint32_t>("maxNumberHits");

configParameters_.maxNumberHitsEB = ps.getParameter<uint32_t>("maxNumberHitsEB");
configParameters_.maxNumberHitsEE = ps.getParameter<uint32_t>("maxNumberHitsEE");

// switch to run timing computation kernels
configParameters_.shouldRunTimingComputation = ps.getParameter<bool>("shouldRunTimingComputation");

Expand Down Expand Up @@ -200,8 +202,8 @@ void EcalUncalibRecHitProducerGPU::acquire(edm::Event const& event,
neb_ = ebDigis.size;
nee_ = eeDigis.size;

if ((neb_ + nee_) > configParameters_.maxNumberHits) {
edm::LogError("EcalUncalibRecHitProducerGPU") << "max number of channels exceeded. See options 'maxNumberHits' ";
if ((neb_ > configParameters_.maxNumberHitsEB) || (nee_ > configParameters_.maxNumberHitsEE)) {
edm::LogError("EcalUncalibRecHitProducerGPU") << "max number of channels exceeded. See options 'maxNumberHitsEB and maxNumberHitsEE' ";
}

// conditions
Expand Down
4 changes: 2 additions & 2 deletions RecoLocalCalo/EcalRecProducers/python/ecalRecHitGPU_cfi.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
recHitsLabelEB = cms.string("EcalRecHitsEB"),
recHitsLabelEE = cms.string("EcalRecHitsEE"),

maxNumberHits = cms.uint32(20000), # FIXME AM

maxNumberHitsEB = cms.uint32(61200),
maxNumberHitsEE = cms.uint32(14648),

#EErechitCollection = cms.string('EcalRecHitsEE'),
#EEuncalibRecHitCollection = cms.InputTag("ecalMultiFitUncalibRecHit","EcalUncalibRecHitsEE"),
Expand Down