Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-04-06 12:25:51

0001 #include "FWCore/Framework/interface/Event.h"
0002 #include "FWCore/Framework/interface/EventSetup.h"
0003 #include "FWCore/Framework/interface/MakerMacros.h"
0004 #include "FWCore/Framework/interface/stream/EDProducer.h"
0005 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0006 #include "FWCore/ServiceRegistry/interface/Service.h"
0007 #include "HeterogeneousCore/CUDACore/interface/JobConfigurationGPURecord.h"
0008 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0009 #include "HeterogeneousCore/CUDAServices/interface/CUDAInterface.h"
0010 
0011 #include "SimpleAlgoGPU.h"
0012 
0013 #include "CondFormats/DataRecord/interface/HcalCombinedRecordsGPU.h"
0014 #include "CondFormats/DataRecord/interface/HcalGainWidthsRcd.h"
0015 #include "CondFormats/DataRecord/interface/HcalGainsRcd.h"
0016 #include "CondFormats/DataRecord/interface/HcalLUTCorrsRcd.h"
0017 #include "CondFormats/DataRecord/interface/HcalQIEDataRcd.h"
0018 #include "CondFormats/DataRecord/interface/HcalQIETypesRcd.h"
0019 #include "CondFormats/DataRecord/interface/HcalRecoParamsRcd.h"
0020 #include "CondFormats/DataRecord/interface/HcalRespCorrsRcd.h"
0021 #include "CondFormats/DataRecord/interface/HcalSiPMCharacteristicsRcd.h"
0022 #include "CondFormats/DataRecord/interface/HcalSiPMParametersRcd.h"
0023 #include "CondFormats/DataRecord/interface/HcalTimeCorrsRcd.h"
0024 #include "CondFormats/DataRecord/interface/HcalChannelQualityRcd.h"
0025 
0026 class HBHERecHitProducerGPU : public edm::stream::EDProducer<edm::ExternalWork> {
0027 public:
0028   explicit HBHERecHitProducerGPU(edm::ParameterSet const&);
0029   ~HBHERecHitProducerGPU() override;
0030   static void fillDescriptions(edm::ConfigurationDescriptions&);
0031 
0032 private:
0033   void acquire(edm::Event const&, edm::EventSetup const&, edm::WaitingTaskWithArenaHolder) override;
0034   void produce(edm::Event&, edm::EventSetup const&) override;
0035 
0036   using IProductTypef01 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor1, calo::common::DevStoragePolicy>>;
0037   edm::EDGetTokenT<IProductTypef01> digisTokenF01HE_;
0038 
0039   using IProductTypef5 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor5, calo::common::DevStoragePolicy>>;
0040   edm::EDGetTokenT<IProductTypef5> digisTokenF5HB_;
0041 
0042   using IProductTypef3 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor3, calo::common::DevStoragePolicy>>;
0043   edm::EDGetTokenT<IProductTypef3> digisTokenF3HB_;
0044 
0045   using RecHitType = hcal::RecHitCollection<calo::common::DevStoragePolicy>;
0046   using OProductType = cms::cuda::Product<RecHitType>;
0047   edm::EDPutTokenT<OProductType> rechitsM0Token_;
0048 
0049   const edm::ESGetToken<HcalRecoParamsWithPulseShapesGPU, HcalRecoParamsRcd> recoParamsToken_;
0050   const edm::ESGetToken<HcalGainWidthsGPU, HcalGainWidthsRcd> gainWidthsToken_;
0051   const edm::ESGetToken<HcalGainsGPU, HcalGainsRcd> gainsToken_;
0052   const edm::ESGetToken<HcalLUTCorrsGPU, HcalLUTCorrsRcd> lutCorrsToken_;
0053   const edm::ESGetToken<HcalConvertedPedestalWidthsGPU, HcalConvertedPedestalWidthsRcd> pedestalWidthsToken_;
0054   const edm::ESGetToken<HcalConvertedEffectivePedestalWidthsGPU, HcalConvertedPedestalWidthsRcd>
0055       effectivePedestalWidthsToken_;
0056   const edm::ESGetToken<HcalConvertedPedestalsGPU, HcalConvertedPedestalsRcd> pedestalsToken_;
0057   edm::ESGetToken<HcalConvertedEffectivePedestalsGPU, HcalConvertedPedestalsRcd> effectivePedestalsToken_;
0058   const edm::ESGetToken<HcalQIECodersGPU, HcalQIEDataRcd> qieCodersToken_;
0059   const edm::ESGetToken<HcalRespCorrsGPU, HcalRespCorrsRcd> respCorrsToken_;
0060   const edm::ESGetToken<HcalTimeCorrsGPU, HcalTimeCorrsRcd> timeCorrsToken_;
0061   const edm::ESGetToken<HcalQIETypesGPU, HcalQIETypesRcd> qieTypesToken_;
0062   const edm::ESGetToken<HcalTopology, HcalRecNumberingRecord> topologyToken_;
0063   const edm::ESGetToken<HcalDDDRecConstants, HcalRecNumberingRecord> recConstantsToken_;
0064   const edm::ESGetToken<HcalSiPMParametersGPU, HcalSiPMParametersRcd> sipmParametersToken_;
0065   const edm::ESGetToken<HcalSiPMCharacteristicsGPU, HcalSiPMCharacteristicsRcd> sipmCharacteristicsToken_;
0066   const edm::ESGetToken<HcalChannelQualityGPU, HcalChannelQualityRcd> chQualProductToken_;
0067   const edm::ESGetToken<HcalMahiPulseOffsetsGPU, JobConfigurationGPURecord> pulseOffsetsToken_;
0068 
0069   hcal::reconstruction::ConfigParameters configParameters_;
0070   hcal::reconstruction::OutputDataGPU outputGPU_;
0071   cms::cuda::ContextState cudaState_;
0072 };
0073 
0074 HBHERecHitProducerGPU::HBHERecHitProducerGPU(edm::ParameterSet const& ps)
0075     : digisTokenF01HE_{consumes<IProductTypef01>(ps.getParameter<edm::InputTag>("digisLabelF01HE"))},
0076       digisTokenF5HB_{consumes<IProductTypef5>(ps.getParameter<edm::InputTag>("digisLabelF5HB"))},
0077       digisTokenF3HB_{consumes<IProductTypef3>(ps.getParameter<edm::InputTag>("digisLabelF3HB"))},
0078       rechitsM0Token_{produces<OProductType>(ps.getParameter<std::string>("recHitsLabelM0HBHE"))},
0079       recoParamsToken_{esConsumes()},
0080       gainWidthsToken_{esConsumes()},
0081       gainsToken_{esConsumes()},
0082       lutCorrsToken_{esConsumes()},
0083       pedestalWidthsToken_{esConsumes()},
0084       effectivePedestalWidthsToken_{esConsumes()},
0085       pedestalsToken_{esConsumes()},
0086       qieCodersToken_{esConsumes()},
0087       respCorrsToken_{esConsumes()},
0088       timeCorrsToken_{esConsumes()},
0089       qieTypesToken_{esConsumes()},
0090       topologyToken_{esConsumes()},
0091       recConstantsToken_{esConsumes()},
0092       sipmParametersToken_{esConsumes()},
0093       sipmCharacteristicsToken_{esConsumes()},
0094       chQualProductToken_{esConsumes()},
0095       pulseOffsetsToken_{esConsumes()} {
0096   configParameters_.maxTimeSamples = ps.getParameter<uint32_t>("maxTimeSamples");
0097   configParameters_.kprep1dChannelsPerBlock = ps.getParameter<uint32_t>("kprep1dChannelsPerBlock");
0098   configParameters_.sipmQTSShift = ps.getParameter<int>("sipmQTSShift");
0099   configParameters_.sipmQNTStoSum = ps.getParameter<int>("sipmQNTStoSum");
0100   configParameters_.firstSampleShift = ps.getParameter<int>("firstSampleShift");
0101   configParameters_.useEffectivePedestals = ps.getParameter<bool>("useEffectivePedestals");
0102   if (configParameters_.useEffectivePedestals) {
0103     effectivePedestalsToken_ = esConsumes();
0104   }
0105 
0106   configParameters_.meanTime = ps.getParameter<double>("meanTime");
0107   configParameters_.timeSigmaSiPM = ps.getParameter<double>("timeSigmaSiPM");
0108   configParameters_.timeSigmaHPD = ps.getParameter<double>("timeSigmaHPD");
0109   configParameters_.ts4Thresh = ps.getParameter<double>("ts4Thresh");
0110 
0111   configParameters_.applyTimeSlew = ps.getParameter<bool>("applyTimeSlew");
0112   auto const tzeroValues = ps.getParameter<std::vector<double>>("tzeroTimeSlewParameters");
0113   auto const slopeValues = ps.getParameter<std::vector<double>>("slopeTimeSlewParameters");
0114   auto const tmaxValues = ps.getParameter<std::vector<double>>("tmaxTimeSlewParameters");
0115 
0116   configParameters_.tzeroTimeSlew = tzeroValues[HcalTimeSlew::Medium];
0117   configParameters_.slopeTimeSlew = slopeValues[HcalTimeSlew::Medium];
0118   configParameters_.tmaxTimeSlew = tmaxValues[HcalTimeSlew::Medium];
0119 
0120   auto threadsMinimize = ps.getParameter<std::vector<uint32_t>>("kernelMinimizeThreads");
0121   configParameters_.kernelMinimizeThreads[0] = threadsMinimize[0];
0122   configParameters_.kernelMinimizeThreads[1] = threadsMinimize[1];
0123   configParameters_.kernelMinimizeThreads[2] = threadsMinimize[2];
0124 }
0125 
0126 HBHERecHitProducerGPU::~HBHERecHitProducerGPU() {}
0127 
0128 void HBHERecHitProducerGPU::fillDescriptions(edm::ConfigurationDescriptions& cdesc) {
0129   edm::ParameterSetDescription desc;
0130   desc.add<uint32_t>("maxTimeSamples", 10);
0131   desc.add<uint32_t>("kprep1dChannelsPerBlock", 32);
0132   desc.add<edm::InputTag>("digisLabelF01HE", edm::InputTag{"hcalRawToDigiGPU", "f01HEDigisGPU"});
0133   desc.add<edm::InputTag>("digisLabelF5HB", edm::InputTag{"hcalRawToDigiGPU", "f5HBDigisGPU"});
0134   desc.add<edm::InputTag>("digisLabelF3HB", edm::InputTag{"hcalRawToDigiGPU", "f3HBDigisGPU"});
0135   desc.add<std::string>("recHitsLabelM0HBHE", "recHitsM0HBHE");
0136   desc.add<int>("sipmQTSShift", 0);
0137   desc.add<int>("sipmQNTStoSum", 3);
0138   desc.add<int>("firstSampleShift", 0);
0139   desc.add<bool>("useEffectivePedestals", true);
0140 
0141   desc.add<double>("meanTime", 0.f);
0142   desc.add<double>("timeSigmaSiPM", 2.5f);
0143   desc.add<double>("timeSigmaHPD", 5.0f);
0144   desc.add<double>("ts4Thresh", 0.0);
0145 
0146   desc.add<bool>("applyTimeSlew", true);
0147   desc.add<std::vector<double>>("tzeroTimeSlewParameters", {23.960177, 11.977461, 9.109694});
0148   desc.add<std::vector<double>>("slopeTimeSlewParameters", {-3.178648, -1.5610227, -1.075824});
0149   desc.add<std::vector<double>>("tmaxTimeSlewParameters", {16.00, 10.00, 6.25});
0150   desc.add<std::vector<uint32_t>>("kernelMinimizeThreads", {16, 1, 1});
0151 
0152   cdesc.addWithDefaultLabel(desc);
0153 }
0154 
0155 void HBHERecHitProducerGPU::acquire(edm::Event const& event,
0156                                     edm::EventSetup const& setup,
0157                                     edm::WaitingTaskWithArenaHolder holder) {
0158 #ifdef HCAL_MAHI_CPUDEBUG
0159   auto start = std::chrono::high_resolution_clock::now();
0160 #endif
0161 
0162   // input + raii
0163   auto const& f01HEProduct = event.get(digisTokenF01HE_);
0164   auto const& f5HBProduct = event.get(digisTokenF5HB_);
0165   auto const& f3HBProduct = event.get(digisTokenF3HB_);
0166   cms::cuda::ScopedContextAcquire ctx{f01HEProduct, std::move(holder), cudaState_};
0167   auto const& f01HEDigis = ctx.get(f01HEProduct);
0168   auto const& f5HBDigis = ctx.get(f5HBProduct);
0169   auto const& f3HBDigis = ctx.get(f3HBProduct);
0170   auto const totalChannels = f01HEDigis.size + f5HBDigis.size + f3HBDigis.size;
0171 
0172   hcal::reconstruction::InputDataGPU inputGPU{f01HEDigis, f5HBDigis, f3HBDigis};
0173 
0174   // conditions
0175   auto const& recoParamsProduct = setup.getData(recoParamsToken_).getProduct(ctx.stream());
0176 
0177   auto const& gainWidthsProduct = setup.getData(gainWidthsToken_).getProduct(ctx.stream());
0178 
0179   auto const& gainsProduct = setup.getData(gainsToken_).getProduct(ctx.stream());
0180 
0181   auto const& lutCorrsProduct = setup.getData(lutCorrsToken_).getProduct(ctx.stream());
0182 
0183   // use only 1 depending on useEffectivePedestals
0184   auto const& pedestalWidthsProduct = setup.getData(pedestalWidthsToken_).getProduct(ctx.stream());
0185   auto const& effectivePedestalWidthsProduct = setup.getData(effectivePedestalWidthsToken_).getProduct(ctx.stream());
0186 
0187   auto const& pedestals = setup.getData(pedestalsToken_);
0188   auto const& pedestalsProduct = pedestals.getProduct(ctx.stream());
0189 
0190   edm::ESHandle<HcalConvertedEffectivePedestalsGPU> effectivePedestalsHandle;
0191   if (configParameters_.useEffectivePedestals)
0192     effectivePedestalsHandle = setup.getHandle(effectivePedestalsToken_);
0193   auto const* effectivePedestalsProduct =
0194       configParameters_.useEffectivePedestals ? &effectivePedestalsHandle->getProduct(ctx.stream()) : nullptr;
0195 
0196   auto const& qieCodersProduct = setup.getData(qieCodersToken_).getProduct(ctx.stream());
0197 
0198   auto const& respCorrsProduct = setup.getData(respCorrsToken_).getProduct(ctx.stream());
0199 
0200   auto const& timeCorrsProduct = setup.getData(timeCorrsToken_).getProduct(ctx.stream());
0201 
0202   auto const& qieTypesProduct = setup.getData(qieTypesToken_).getProduct(ctx.stream());
0203 
0204   HcalTopology const& topology = setup.getData(topologyToken_);
0205   HcalDDDRecConstants const& recConstants = setup.getData(recConstantsToken_);
0206 
0207   auto const& sipmParametersProduct = setup.getData(sipmParametersToken_).getProduct(ctx.stream());
0208 
0209   auto const& sipmCharacteristicsProduct = setup.getData(sipmCharacteristicsToken_).getProduct(ctx.stream());
0210 
0211   auto const& chQualProduct = setup.getData(chQualProductToken_).getProduct(ctx.stream());
0212 
0213   auto const& pulseOffsets = setup.getData(pulseOffsetsToken_);
0214   auto const& pulseOffsetsProduct = pulseOffsets.getProduct(ctx.stream());
0215 
0216   // bundle up conditions
0217   hcal::reconstruction::ConditionsProducts conditions{gainWidthsProduct,
0218                                                       gainsProduct,
0219                                                       lutCorrsProduct,
0220                                                       pedestalWidthsProduct,
0221                                                       effectivePedestalWidthsProduct,
0222                                                       pedestalsProduct,
0223                                                       qieCodersProduct,
0224                                                       chQualProduct,
0225                                                       recoParamsProduct,
0226                                                       respCorrsProduct,
0227                                                       timeCorrsProduct,
0228                                                       qieTypesProduct,
0229                                                       sipmParametersProduct,
0230                                                       sipmCharacteristicsProduct,
0231                                                       effectivePedestalsProduct,
0232                                                       &topology,
0233                                                       &recConstants,
0234                                                       pedestals.offsetForHashes(),
0235                                                       pulseOffsetsProduct,
0236                                                       pulseOffsets.getValues()};
0237 
0238   // scratch mem on device
0239   hcal::reconstruction::ScratchDataGPU scratchGPU = {
0240       cms::cuda::make_device_unique<float[]>(totalChannels * configParameters_.maxTimeSamples, ctx.stream()),
0241       cms::cuda::make_device_unique<float[]>(totalChannels * configParameters_.maxTimeSamples, ctx.stream()),
0242       cms::cuda::make_device_unique<float[]>(totalChannels * configParameters_.maxTimeSamples, ctx.stream()),
0243       cms::cuda::make_device_unique<float[]>(
0244           totalChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples, ctx.stream()),
0245       cms::cuda::make_device_unique<float[]>(
0246           totalChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples, ctx.stream()),
0247       cms::cuda::make_device_unique<float[]>(
0248           totalChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples, ctx.stream()),
0249       cms::cuda::make_device_unique<int8_t[]>(totalChannels, ctx.stream()),
0250   };
0251 
0252   // output dev mem
0253   outputGPU_.allocate(configParameters_, totalChannels, ctx.stream());
0254 
0255   hcal::reconstruction::entryPoint(inputGPU, outputGPU_, conditions, scratchGPU, configParameters_, ctx.stream());
0256 
0257 #ifdef HCAL_MAHI_CPUDEBUG
0258   auto end = std::chrono::high_resolution_clock::now();
0259   auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
0260   std::cout << "acquire  duration = " << duration << std::endl;
0261 #endif
0262 }
0263 
0264 void HBHERecHitProducerGPU::produce(edm::Event& event, edm::EventSetup const& setup) {
0265   cms::cuda::ScopedContextProduce ctx{cudaState_};
0266   ctx.emplace(event, rechitsM0Token_, std::move(outputGPU_.recHits));
0267 }
0268 
0269 DEFINE_FWK_MODULE(HBHERecHitProducerGPU);