EcalRecProducers/plugins/EcalUncalibRecHitProducerGPU.cc

0001 #include "CUDADataFormats/EcalRecHitSoA/interface/EcalUncalibratedRecHit.h"
0002 #include "CondFormats/DataRecord/interface/EcalGainRatiosRcd.h"
0003 #include "CondFormats/DataRecord/interface/EcalPedestalsRcd.h"
0004 #include "CondFormats/DataRecord/interface/EcalPulseCovariancesRcd.h"
0005 #include "CondFormats/DataRecord/interface/EcalPulseShapesRcd.h"
0006 #include "CondFormats/DataRecord/interface/EcalSampleMaskRcd.h"
0007 #include "CondFormats/DataRecord/interface/EcalSamplesCorrelationRcd.h"
0008 #include "CondFormats/DataRecord/interface/EcalTimeBiasCorrectionsRcd.h"
0009 #include "CondFormats/DataRecord/interface/EcalTimeCalibConstantsRcd.h"
0010 #include "CondFormats/DataRecord/interface/EcalTimeOffsetConstantRcd.h"
0011 #include "CondFormats/EcalObjects/interface/EcalGainRatiosGPU.h"
0012 #include "CondFormats/EcalObjects/interface/EcalMultifitParametersGPU.h"
0013 #include "CondFormats/EcalObjects/interface/EcalPedestalsGPU.h"
0014 #include "CondFormats/EcalObjects/interface/EcalPulseCovariancesGPU.h"
0015 #include "CondFormats/EcalObjects/interface/EcalPulseShapesGPU.h"
0016 #include "CondFormats/EcalObjects/interface/EcalSamplesCorrelationGPU.h"
0017 #include "CondFormats/EcalObjects/interface/EcalTimeBiasCorrectionsGPU.h"
0018 #include "CondFormats/EcalObjects/interface/EcalTimeCalibConstantsGPU.h"
0019 #include "CondFormats/EcalObjects/interface/EcalTimeOffsetConstant.h"
0020 #include "DataFormats/EcalDigi/interface/EcalDigiCollections.h"
0021 #include "FWCore/Framework/interface/Event.h"
0022 #include "FWCore/Framework/interface/EventSetup.h"
0023 #include "FWCore/Framework/interface/MakerMacros.h"
0024 #include "FWCore/Framework/interface/stream/EDProducer.h"
0025 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0026 #include "HeterogeneousCore/CUDACore/interface/JobConfigurationGPURecord.h"
0027 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0028 #include "HeterogeneousCore/CUDAUtilities/interface/cudaCheck.h"
0029
0030 #include "DeclsForKernels.h"
0031 #include "EcalUncalibRecHitMultiFitAlgoGPU.h"
0032
0033 class EcalUncalibRecHitProducerGPU : public edm::stream::EDProducer<> {
0034 public:
0035   explicit EcalUncalibRecHitProducerGPU(edm::ParameterSet const& ps);
0036   ~EcalUncalibRecHitProducerGPU() override;
0037   static void fillDescriptions(edm::ConfigurationDescriptions&);
0038
0039 private:
0040   void produce(edm::Event&, edm::EventSetup const&) override;
0041
0042 private:
0043   using InputProduct = cms::cuda::Product<ecal::DigisCollection<calo::common::DevStoragePolicy>>;
0044   const edm::EDGetTokenT<InputProduct> digisTokenEB_, digisTokenEE_;
0045   using OutputProduct = cms::cuda::Product<ecal::UncalibratedRecHit<calo::common::DevStoragePolicy>>;
0046   const edm::EDPutTokenT<OutputProduct> recHitsTokenEB_, recHitsTokenEE_;
0047
0048   // conditions tokens
0049   const edm::ESGetToken<EcalPedestalsGPU, EcalPedestalsRcd> pedestalsToken_;
0050   const edm::ESGetToken<EcalGainRatiosGPU, EcalGainRatiosRcd> gainRatiosToken_;
0051   const edm::ESGetToken<EcalPulseShapesGPU, EcalPulseShapesRcd> pulseShapesToken_;
0052   const edm::ESGetToken<EcalPulseCovariancesGPU, EcalPulseCovariancesRcd> pulseCovariancesToken_;
0053   const edm::ESGetToken<EcalSamplesCorrelationGPU, EcalSamplesCorrelationRcd> samplesCorrelationToken_;
0054   const edm::ESGetToken<EcalTimeBiasCorrectionsGPU, EcalTimeBiasCorrectionsRcd> timeBiasCorrectionsToken_;
0055   const edm::ESGetToken<EcalTimeCalibConstantsGPU, EcalTimeCalibConstantsRcd> timeCalibConstantsToken_;
0056   const edm::ESGetToken<EcalSampleMask, EcalSampleMaskRcd> sampleMaskToken_;
0057   const edm::ESGetToken<EcalTimeOffsetConstant, EcalTimeOffsetConstantRcd> timeOffsetConstantToken_;
0058   const edm::ESGetToken<EcalMultifitParametersGPU, JobConfigurationGPURecord> multifitParametersToken_;
0059
0060   // configuration parameters
0061   ecal::multifit::ConfigurationParameters configParameters_;
0062 };
0063
0064 void EcalUncalibRecHitProducerGPU::fillDescriptions(edm::ConfigurationDescriptions& confDesc) {
0065   edm::ParameterSetDescription desc;
0066
0067   desc.add<edm::InputTag>("digisLabelEB", edm::InputTag("ecalRawToDigiGPU", "ebDigis"));
0068   desc.add<edm::InputTag>("digisLabelEE", edm::InputTag("ecalRawToDigiGPU", "eeDigis"));
0069
0070   desc.add<std::string>("recHitsLabelEB", "EcalUncalibRecHitsEB");
0071   desc.add<std::string>("recHitsLabelEE", "EcalUncalibRecHitsEE");
0072
0073   desc.add<double>("EBtimeFitLimits_Lower", 0.2);
0074   desc.add<double>("EBtimeFitLimits_Upper", 1.4);
0075   desc.add<double>("EEtimeFitLimits_Lower", 0.2);
0076   desc.add<double>("EEtimeFitLimits_Upper", 1.4);
0077   desc.add<double>("EBtimeConstantTerm", .6);
0078   desc.add<double>("EEtimeConstantTerm", 1.0);
0079   desc.add<double>("EBtimeNconst", 28.5);
0080   desc.add<double>("EEtimeNconst", 31.8);
0081   desc.add<double>("outOfTimeThresholdGain12pEB", 5);
0082   desc.add<double>("outOfTimeThresholdGain12mEB", 5);
0083   desc.add<double>("outOfTimeThresholdGain61pEB", 5);
0084   desc.add<double>("outOfTimeThresholdGain61mEB", 5);
0085   desc.add<double>("outOfTimeThresholdGain12pEE", 1000);
0086   desc.add<double>("outOfTimeThresholdGain12mEE", 1000);
0087   desc.add<double>("outOfTimeThresholdGain61pEE", 1000);
0088   desc.add<double>("outOfTimeThresholdGain61mEE", 1000);
0089   desc.add<double>("amplitudeThresholdEB", 10);
0090   desc.add<double>("amplitudeThresholdEE", 10);
0091   desc.add<uint32_t>("maxNumberHitsEB", 61200);
0092   desc.add<uint32_t>("maxNumberHitsEE", 14648);
0093   desc.addUntracked<std::vector<uint32_t>>("kernelMinimizeThreads", {32, 1, 1});
0094   desc.add<bool>("shouldRunTimingComputation", true);
0095   confDesc.addWithDefaultLabel(desc);
0096 }
0097
0098 EcalUncalibRecHitProducerGPU::EcalUncalibRecHitProducerGPU(const edm::ParameterSet& ps)
0099     : digisTokenEB_{consumes<InputProduct>(ps.getParameter<edm::InputTag>("digisLabelEB"))},
0100       digisTokenEE_{consumes<InputProduct>(ps.getParameter<edm::InputTag>("digisLabelEE"))},
0101       recHitsTokenEB_{produces<OutputProduct>(ps.getParameter<std::string>("recHitsLabelEB"))},
0102       recHitsTokenEE_{produces<OutputProduct>(ps.getParameter<std::string>("recHitsLabelEE"))},
0103       pedestalsToken_{esConsumes<EcalPedestalsGPU, EcalPedestalsRcd>()},
0104       gainRatiosToken_{esConsumes<EcalGainRatiosGPU, EcalGainRatiosRcd>()},
0105       pulseShapesToken_{esConsumes<EcalPulseShapesGPU, EcalPulseShapesRcd>()},
0106       pulseCovariancesToken_{esConsumes<EcalPulseCovariancesGPU, EcalPulseCovariancesRcd>()},
0107       samplesCorrelationToken_{esConsumes<EcalSamplesCorrelationGPU, EcalSamplesCorrelationRcd>()},
0108       timeBiasCorrectionsToken_{esConsumes<EcalTimeBiasCorrectionsGPU, EcalTimeBiasCorrectionsRcd>()},
0109       timeCalibConstantsToken_{esConsumes<EcalTimeCalibConstantsGPU, EcalTimeCalibConstantsRcd>()},
0110       sampleMaskToken_{esConsumes<EcalSampleMask, EcalSampleMaskRcd>()},
0111       timeOffsetConstantToken_{esConsumes<EcalTimeOffsetConstant, EcalTimeOffsetConstantRcd>()},
0112       multifitParametersToken_{esConsumes<EcalMultifitParametersGPU, JobConfigurationGPURecord>()} {
0113   std::pair<double, double> EBtimeFitLimits, EEtimeFitLimits;
0114   EBtimeFitLimits.first = ps.getParameter<double>("EBtimeFitLimits_Lower");
0115   EBtimeFitLimits.second = ps.getParameter<double>("EBtimeFitLimits_Upper");
0116   EEtimeFitLimits.first = ps.getParameter<double>("EEtimeFitLimits_Lower");
0117   EEtimeFitLimits.second = ps.getParameter<double>("EEtimeFitLimits_Upper");
0118
0119   auto EBtimeConstantTerm = ps.getParameter<double>("EBtimeConstantTerm");
0120   auto EEtimeConstantTerm = ps.getParameter<double>("EEtimeConstantTerm");
0121   auto EBtimeNconst = ps.getParameter<double>("EBtimeNconst");
0122   auto EEtimeNconst = ps.getParameter<double>("EEtimeNconst");
0123
0124   auto outOfTimeThreshG12pEB = ps.getParameter<double>("outOfTimeThresholdGain12pEB");
0125   auto outOfTimeThreshG12mEB = ps.getParameter<double>("outOfTimeThresholdGain12mEB");
0126   auto outOfTimeThreshG61pEB = ps.getParameter<double>("outOfTimeThresholdGain61pEB");
0127   auto outOfTimeThreshG61mEB = ps.getParameter<double>("outOfTimeThresholdGain61mEB");
0128   auto outOfTimeThreshG12pEE = ps.getParameter<double>("outOfTimeThresholdGain12pEE");
0129   auto outOfTimeThreshG12mEE = ps.getParameter<double>("outOfTimeThresholdGain12mEE");
0130   auto outOfTimeThreshG61pEE = ps.getParameter<double>("outOfTimeThresholdGain61pEE");
0131   auto outOfTimeThreshG61mEE = ps.getParameter<double>("outOfTimeThresholdGain61mEE");
0132   auto amplitudeThreshEB = ps.getParameter<double>("amplitudeThresholdEB");
0133   auto amplitudeThreshEE = ps.getParameter<double>("amplitudeThresholdEE");
0134
0135   // max number of digis to allocate for
0136   configParameters_.maxNumberHitsEB = ps.getParameter<uint32_t>("maxNumberHitsEB");
0137   configParameters_.maxNumberHitsEE = ps.getParameter<uint32_t>("maxNumberHitsEE");
0138
0139   // switch to run timing computation kernels
0140   configParameters_.shouldRunTimingComputation = ps.getParameter<bool>("shouldRunTimingComputation");
0141
0142   // minimize kernel launch conf
0143   auto threadsMinimize = ps.getUntrackedParameter<std::vector<uint32_t>>("kernelMinimizeThreads");
0144   configParameters_.kernelMinimizeThreads[0] = threadsMinimize[0];
0145   configParameters_.kernelMinimizeThreads[1] = threadsMinimize[1];
0146   configParameters_.kernelMinimizeThreads[2] = threadsMinimize[2];
0147
0148   //
0149   // configuration and physics parameters: done once
0150   // assume there is a single device
0151   // use sync copying
0152   //
0153
0154   // time fit parameters and limits
0155   configParameters_.timeFitLimitsFirstEB = EBtimeFitLimits.first;
0156   configParameters_.timeFitLimitsSecondEB = EBtimeFitLimits.second;
0157   configParameters_.timeFitLimitsFirstEE = EEtimeFitLimits.first;
0158   configParameters_.timeFitLimitsSecondEE = EEtimeFitLimits.second;
0159
0160   // time constant terms
0161   configParameters_.timeConstantTermEB = EBtimeConstantTerm;
0162   configParameters_.timeConstantTermEE = EEtimeConstantTerm;
0163
0164   // time N const
0165   configParameters_.timeNconstEB = EBtimeNconst;
0166   configParameters_.timeNconstEE = EEtimeNconst;
0167
0168   // amplitude threshold for time flags
0169   configParameters_.amplitudeThreshEB = amplitudeThreshEB;
0170   configParameters_.amplitudeThreshEE = amplitudeThreshEE;
0171
0172   // out of time thresholds gain-dependent
0173   configParameters_.outOfTimeThreshG12pEB = outOfTimeThreshG12pEB;
0174   configParameters_.outOfTimeThreshG12pEE = outOfTimeThreshG12pEE;
0175   configParameters_.outOfTimeThreshG61pEB = outOfTimeThreshG61pEB;
0176   configParameters_.outOfTimeThreshG61pEE = outOfTimeThreshG61pEE;
0177   configParameters_.outOfTimeThreshG12mEB = outOfTimeThreshG12mEB;
0178   configParameters_.outOfTimeThreshG12mEE = outOfTimeThreshG12mEE;
0179   configParameters_.outOfTimeThreshG61mEB = outOfTimeThreshG61mEB;
0180   configParameters_.outOfTimeThreshG61mEE = outOfTimeThreshG61mEE;
0181 }
0182
0183 EcalUncalibRecHitProducerGPU::~EcalUncalibRecHitProducerGPU() {}
0184
0185 void EcalUncalibRecHitProducerGPU::produce(edm::Event& event, edm::EventSetup const& setup) {
0186   //DurationMeasurer<std::chrono::milliseconds> timer{std::string{"produce duration"}};
0187
0188   // cuda products
0189   auto const& ebDigisProduct = event.get(digisTokenEB_);
0190   auto const& eeDigisProduct = event.get(digisTokenEE_);
0191   // event data
0192   ecal::multifit::EventOutputDataGPU eventOutputDataGPU;
0193
0194   // raii
0195   cms::cuda::ScopedContextProduce ctx{ebDigisProduct};
0196
0197   // get actual obj
0198   auto const& ebDigis = ctx.get(ebDigisProduct);
0199   auto const& eeDigis = ctx.get(eeDigisProduct);
0200   ecal::multifit::EventInputDataGPU inputDataGPU{ebDigis, eeDigis};
0201   const uint32_t neb = ebDigis.size;
0202   const uint32_t nee = eeDigis.size;
0203
0204   // stop here if there are no digis
0205   if (neb + nee > 0) {
0206     if ((neb > configParameters_.maxNumberHitsEB) || (nee > configParameters_.maxNumberHitsEE)) {
0207       edm::LogError("EcalUncalibRecHitProducerGPU")
0208           << "Max number of channels exceeded in barrel or endcap. Number of barrel channels: " << neb
0209           << " with maxNumberHitsEB=" << configParameters_.maxNumberHitsEB << ", number of endcap channels: " << nee
0210           << " with maxNumberHitsEE=" << configParameters_.maxNumberHitsEE;
0211     }
0212
0213     // conditions
0214     auto const& timeCalibConstantsData = setup.getData(timeCalibConstantsToken_);
0215     auto const& sampleMaskData = setup.getData(sampleMaskToken_);
0216     auto const& timeOffsetConstantData = setup.getData(timeOffsetConstantToken_);
0217     auto const& multifitParametersData = setup.getData(multifitParametersToken_);
0218
0219     auto const& pedestals = setup.getData(pedestalsToken_).getProduct(ctx.stream());
0220     auto const& gainRatios = setup.getData(gainRatiosToken_).getProduct(ctx.stream());
0221     auto const& pulseShapes = setup.getData(pulseShapesToken_).getProduct(ctx.stream());
0222     auto const& pulseCovariances = setup.getData(pulseCovariancesToken_).getProduct(ctx.stream());
0223     auto const& samplesCorrelation = setup.getData(samplesCorrelationToken_).getProduct(ctx.stream());
0224     auto const& timeBiasCorrections = setup.getData(timeBiasCorrectionsToken_).getProduct(ctx.stream());
0225     auto const& timeCalibConstants = timeCalibConstantsData.getProduct(ctx.stream());
0226     auto const& multifitParameters = multifitParametersData.getProduct(ctx.stream());
0227
0228     // assign ptrs/values: this is done not to change how things look downstream
0229     configParameters_.amplitudeFitParametersEB = multifitParameters.amplitudeFitParametersEB.get();
0230     configParameters_.amplitudeFitParametersEE = multifitParameters.amplitudeFitParametersEE.get();
0231     configParameters_.timeFitParametersEB = multifitParameters.timeFitParametersEB.get();
0232     configParameters_.timeFitParametersEE = multifitParameters.timeFitParametersEE.get();
0233     configParameters_.timeFitParametersSizeEB = multifitParametersData.getValues()[2].get().size();
0234     configParameters_.timeFitParametersSizeEE = multifitParametersData.getValues()[3].get().size();
0235
0236     // bundle up conditions
0237     ecal::multifit::ConditionsProducts conditions{pedestals,
0238                                                   gainRatios,
0239                                                   pulseShapes,
0240                                                   pulseCovariances,
0241                                                   samplesCorrelation,
0242                                                   timeBiasCorrections,
0243                                                   timeCalibConstants,
0244                                                   sampleMaskData,
0245                                                   timeOffsetConstantData,
0246                                                   timeCalibConstantsData.getOffset(),
0247                                                   multifitParameters};
0248
0249     // dev mem
0250     eventOutputDataGPU.allocate(configParameters_, ctx.stream());
0251
0252     // scratch mem
0253     ecal::multifit::EventDataForScratchGPU eventDataForScratchGPU;
0254     eventDataForScratchGPU.allocate(configParameters_, ctx.stream());
0255
0256     //
0257     // schedule algorithms
0258     //
0259     ecal::multifit::entryPoint(
0260         inputDataGPU, eventOutputDataGPU, eventDataForScratchGPU, conditions, configParameters_, ctx.stream());
0261   }
0262
0263   // set the size of eb and ee
0264   eventOutputDataGPU.recHitsEB.size = neb;
0265   eventOutputDataGPU.recHitsEE.size = nee;
0266
0267   // put into the event
0268   ctx.emplace(event, recHitsTokenEB_, std::move(eventOutputDataGPU.recHitsEB));
0269   ctx.emplace(event, recHitsTokenEE_, std::move(eventOutputDataGPU.recHitsEE));
0270 }
0271
0272 DEFINE_FWK_MODULE(EcalUncalibRecHitProducerGPU);