Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-09-28 23:48:50

0001 #include "FWCore/Framework/interface/Event.h"
0002 #include "FWCore/Framework/interface/EventSetup.h"
0003 #include "FWCore/Framework/interface/MakerMacros.h"
0004 #include "FWCore/Framework/interface/stream/EDProducer.h"
0005 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0006 #include "FWCore/ServiceRegistry/interface/Service.h"
0007 #include "HeterogeneousCore/CUDACore/interface/JobConfigurationGPURecord.h"
0008 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0009 #include "HeterogeneousCore/CUDAServices/interface/CUDAService.h"
0010 
0011 #include "SimpleAlgoGPU.h"
0012 
0013 class HBHERecHitProducerGPU : public edm::stream::EDProducer<edm::ExternalWork> {
0014 public:
0015   explicit HBHERecHitProducerGPU(edm::ParameterSet const&);
0016   ~HBHERecHitProducerGPU() override;
0017   static void fillDescriptions(edm::ConfigurationDescriptions&);
0018 
0019 private:
0020   void acquire(edm::Event const&, edm::EventSetup const&, edm::WaitingTaskWithArenaHolder) override;
0021   void produce(edm::Event&, edm::EventSetup const&) override;
0022 
0023   using IProductTypef01 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor1, calo::common::DevStoragePolicy>>;
0024   edm::EDGetTokenT<IProductTypef01> digisTokenF01HE_;
0025 
0026   using IProductTypef5 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor5, calo::common::DevStoragePolicy>>;
0027   edm::EDGetTokenT<IProductTypef5> digisTokenF5HB_;
0028 
0029   using IProductTypef3 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor3, calo::common::DevStoragePolicy>>;
0030   edm::EDGetTokenT<IProductTypef3> digisTokenF3HB_;
0031 
0032   using RecHitType = hcal::RecHitCollection<calo::common::DevStoragePolicy>;
0033   using OProductType = cms::cuda::Product<RecHitType>;
0034   edm::EDPutTokenT<OProductType> rechitsM0Token_;
0035 
0036   const edm::ESGetToken<HcalRecoParamsWithPulseShapesGPU, HcalRecoParamsRcd> recoParamsToken_;
0037   const edm::ESGetToken<HcalGainWidthsGPU, HcalGainWidthsRcd> gainWidthsToken_;
0038   const edm::ESGetToken<HcalGainsGPU, HcalGainsRcd> gainsToken_;
0039   const edm::ESGetToken<HcalLUTCorrsGPU, HcalLUTCorrsRcd> lutCorrsToken_;
0040   const edm::ESGetToken<HcalConvertedPedestalWidthsGPU, HcalConvertedPedestalWidthsRcd> pedestalWidthsToken_;
0041   const edm::ESGetToken<HcalConvertedEffectivePedestalWidthsGPU, HcalConvertedPedestalWidthsRcd>
0042       effectivePedestalWidthsToken_;
0043   const edm::ESGetToken<HcalConvertedPedestalsGPU, HcalConvertedPedestalsRcd> pedestalsToken_;
0044   edm::ESGetToken<HcalConvertedEffectivePedestalsGPU, HcalConvertedPedestalsRcd> effectivePedestalsToken_;
0045   const edm::ESGetToken<HcalQIECodersGPU, HcalQIEDataRcd> qieCodersToken_;
0046   const edm::ESGetToken<HcalRespCorrsGPU, HcalRespCorrsRcd> respCorrsToken_;
0047   const edm::ESGetToken<HcalTimeCorrsGPU, HcalTimeCorrsRcd> timeCorrsToken_;
0048   const edm::ESGetToken<HcalQIETypesGPU, HcalQIETypesRcd> qieTypesToken_;
0049   const edm::ESGetToken<HcalTopology, HcalRecNumberingRecord> topologyToken_;
0050   const edm::ESGetToken<HcalDDDRecConstants, HcalRecNumberingRecord> recConstantsToken_;
0051   const edm::ESGetToken<HcalSiPMParametersGPU, HcalSiPMParametersRcd> sipmParametersToken_;
0052   const edm::ESGetToken<HcalSiPMCharacteristicsGPU, HcalSiPMCharacteristicsRcd> sipmCharacteristicsToken_;
0053   const edm::ESGetToken<HcalChannelQualityGPU, HcalChannelQualityRcd> chQualProductToken_;
0054   const edm::ESGetToken<HcalMahiPulseOffsetsGPU, JobConfigurationGPURecord> pulseOffsetsToken_;
0055 
0056   hcal::reconstruction::ConfigParameters configParameters_;
0057   hcal::reconstruction::OutputDataGPU outputGPU_;
0058   cms::cuda::ContextState cudaState_;
0059 };
0060 
0061 HBHERecHitProducerGPU::HBHERecHitProducerGPU(edm::ParameterSet const& ps)
0062     : digisTokenF01HE_{consumes<IProductTypef01>(ps.getParameter<edm::InputTag>("digisLabelF01HE"))},
0063       digisTokenF5HB_{consumes<IProductTypef5>(ps.getParameter<edm::InputTag>("digisLabelF5HB"))},
0064       digisTokenF3HB_{consumes<IProductTypef3>(ps.getParameter<edm::InputTag>("digisLabelF3HB"))},
0065       rechitsM0Token_{produces<OProductType>(ps.getParameter<std::string>("recHitsLabelM0HBHE"))},
0066       recoParamsToken_{esConsumes()},
0067       gainWidthsToken_{esConsumes()},
0068       gainsToken_{esConsumes()},
0069       lutCorrsToken_{esConsumes()},
0070       pedestalWidthsToken_{esConsumes()},
0071       effectivePedestalWidthsToken_{esConsumes()},
0072       pedestalsToken_{esConsumes()},
0073       qieCodersToken_{esConsumes()},
0074       respCorrsToken_{esConsumes()},
0075       timeCorrsToken_{esConsumes()},
0076       qieTypesToken_{esConsumes()},
0077       topologyToken_{esConsumes()},
0078       recConstantsToken_{esConsumes()},
0079       sipmParametersToken_{esConsumes()},
0080       sipmCharacteristicsToken_{esConsumes()},
0081       chQualProductToken_{esConsumes()},
0082       pulseOffsetsToken_{esConsumes()} {
0083   configParameters_.maxChannels = ps.getParameter<uint32_t>("maxChannels");
0084   configParameters_.maxTimeSamples = ps.getParameter<uint32_t>("maxTimeSamples");
0085   configParameters_.kprep1dChannelsPerBlock = ps.getParameter<uint32_t>("kprep1dChannelsPerBlock");
0086   configParameters_.sipmQTSShift = ps.getParameter<int>("sipmQTSShift");
0087   configParameters_.sipmQNTStoSum = ps.getParameter<int>("sipmQNTStoSum");
0088   configParameters_.firstSampleShift = ps.getParameter<int>("firstSampleShift");
0089   configParameters_.useEffectivePedestals = ps.getParameter<bool>("useEffectivePedestals");
0090   if (configParameters_.useEffectivePedestals) {
0091     effectivePedestalsToken_ = esConsumes();
0092   }
0093 
0094   configParameters_.meanTime = ps.getParameter<double>("meanTime");
0095   configParameters_.timeSigmaSiPM = ps.getParameter<double>("timeSigmaSiPM");
0096   configParameters_.timeSigmaHPD = ps.getParameter<double>("timeSigmaHPD");
0097   configParameters_.ts4Thresh = ps.getParameter<double>("ts4Thresh");
0098 
0099   configParameters_.applyTimeSlew = ps.getParameter<bool>("applyTimeSlew");
0100   auto const tzeroValues = ps.getParameter<std::vector<double>>("tzeroTimeSlewParameters");
0101   auto const slopeValues = ps.getParameter<std::vector<double>>("slopeTimeSlewParameters");
0102   auto const tmaxValues = ps.getParameter<std::vector<double>>("tmaxTimeSlewParameters");
0103 
0104   configParameters_.tzeroTimeSlew = tzeroValues[HcalTimeSlew::Medium];
0105   configParameters_.slopeTimeSlew = slopeValues[HcalTimeSlew::Medium];
0106   configParameters_.tmaxTimeSlew = tmaxValues[HcalTimeSlew::Medium];
0107 
0108   auto threadsMinimize = ps.getParameter<std::vector<uint32_t>>("kernelMinimizeThreads");
0109   configParameters_.kernelMinimizeThreads[0] = threadsMinimize[0];
0110   configParameters_.kernelMinimizeThreads[1] = threadsMinimize[1];
0111   configParameters_.kernelMinimizeThreads[2] = threadsMinimize[2];
0112 }
0113 
0114 HBHERecHitProducerGPU::~HBHERecHitProducerGPU() {}
0115 
0116 void HBHERecHitProducerGPU::fillDescriptions(edm::ConfigurationDescriptions& cdesc) {
0117   edm::ParameterSetDescription desc;
0118   desc.add<uint32_t>("maxChannels", 10000u);
0119   desc.add<uint32_t>("maxTimeSamples", 10);
0120   desc.add<uint32_t>("kprep1dChannelsPerBlock", 32);
0121   desc.add<edm::InputTag>("digisLabelF01HE", edm::InputTag{"hcalRawToDigiGPU", "f01HEDigisGPU"});
0122   desc.add<edm::InputTag>("digisLabelF5HB", edm::InputTag{"hcalRawToDigiGPU", "f5HBDigisGPU"});
0123   desc.add<edm::InputTag>("digisLabelF3HB", edm::InputTag{"hcalRawToDigiGPU", "f3HBDigisGPU"});
0124   desc.add<std::string>("recHitsLabelM0HBHE", "recHitsM0HBHE");
0125   desc.add<int>("sipmQTSShift", 0);
0126   desc.add<int>("sipmQNTStoSum", 3);
0127   desc.add<int>("firstSampleShift", 0);
0128   desc.add<bool>("useEffectivePedestals", true);
0129 
0130   desc.add<double>("meanTime", 0.f);
0131   desc.add<double>("timeSigmaSiPM", 2.5f);
0132   desc.add<double>("timeSigmaHPD", 5.0f);
0133   desc.add<double>("ts4Thresh", 0.0);
0134 
0135   desc.add<bool>("applyTimeSlew", true);
0136   desc.add<std::vector<double>>("tzeroTimeSlewParameters", {23.960177, 11.977461, 9.109694});
0137   desc.add<std::vector<double>>("slopeTimeSlewParameters", {-3.178648, -1.5610227, -1.075824});
0138   desc.add<std::vector<double>>("tmaxTimeSlewParameters", {16.00, 10.00, 6.25});
0139   desc.add<std::vector<uint32_t>>("kernelMinimizeThreads", {16, 1, 1});
0140 
0141   cdesc.addWithDefaultLabel(desc);
0142 }
0143 
0144 void HBHERecHitProducerGPU::acquire(edm::Event const& event,
0145                                     edm::EventSetup const& setup,
0146                                     edm::WaitingTaskWithArenaHolder holder) {
0147 #ifdef HCAL_MAHI_CPUDEBUG
0148   auto start = std::chrono::high_resolution_clock::now();
0149 #endif
0150 
0151   // input + raii
0152   auto const& f01HEProduct = event.get(digisTokenF01HE_);
0153   auto const& f5HBProduct = event.get(digisTokenF5HB_);
0154   auto const& f3HBProduct = event.get(digisTokenF3HB_);
0155   cms::cuda::ScopedContextAcquire ctx{f01HEProduct, std::move(holder), cudaState_};
0156   auto const& f01HEDigis = ctx.get(f01HEProduct);
0157   auto const& f5HBDigis = ctx.get(f5HBProduct);
0158   auto const& f3HBDigis = ctx.get(f3HBProduct);
0159 
0160   hcal::reconstruction::InputDataGPU inputGPU{f01HEDigis, f5HBDigis, f3HBDigis};
0161 
0162   // conditions
0163   auto const& recoParamsProduct = setup.getData(recoParamsToken_).getProduct(ctx.stream());
0164 
0165   auto const& gainWidthsProduct = setup.getData(gainWidthsToken_).getProduct(ctx.stream());
0166 
0167   auto const& gainsProduct = setup.getData(gainsToken_).getProduct(ctx.stream());
0168 
0169   auto const& lutCorrsProduct = setup.getData(lutCorrsToken_).getProduct(ctx.stream());
0170 
0171   // use only 1 depending on useEffectivePedestals
0172   auto const& pedestalWidthsProduct = setup.getData(pedestalWidthsToken_).getProduct(ctx.stream());
0173   auto const& effectivePedestalWidthsProduct = setup.getData(effectivePedestalWidthsToken_).getProduct(ctx.stream());
0174 
0175   auto const& pedestals = setup.getData(pedestalsToken_);
0176   auto const& pedestalsProduct = pedestals.getProduct(ctx.stream());
0177 
0178   edm::ESHandle<HcalConvertedEffectivePedestalsGPU> effectivePedestalsHandle;
0179   if (configParameters_.useEffectivePedestals)
0180     effectivePedestalsHandle = setup.getHandle(effectivePedestalsToken_);
0181   auto const* effectivePedestalsProduct =
0182       configParameters_.useEffectivePedestals ? &effectivePedestalsHandle->getProduct(ctx.stream()) : nullptr;
0183 
0184   auto const& qieCodersProduct = setup.getData(qieCodersToken_).getProduct(ctx.stream());
0185 
0186   auto const& respCorrsProduct = setup.getData(respCorrsToken_).getProduct(ctx.stream());
0187 
0188   auto const& timeCorrsProduct = setup.getData(timeCorrsToken_).getProduct(ctx.stream());
0189 
0190   auto const& qieTypesProduct = setup.getData(qieTypesToken_).getProduct(ctx.stream());
0191 
0192   HcalTopology const& topology = setup.getData(topologyToken_);
0193   HcalDDDRecConstants const& recConstants = setup.getData(recConstantsToken_);
0194 
0195   auto const& sipmParametersProduct = setup.getData(sipmParametersToken_).getProduct(ctx.stream());
0196 
0197   auto const& sipmCharacteristicsProduct = setup.getData(sipmCharacteristicsToken_).getProduct(ctx.stream());
0198 
0199   auto const& chQualProduct = setup.getData(chQualProductToken_).getProduct(ctx.stream());
0200 
0201   auto const& pulseOffsets = setup.getData(pulseOffsetsToken_);
0202   auto const& pulseOffsetsProduct = pulseOffsets.getProduct(ctx.stream());
0203 
0204   // bundle up conditions
0205   hcal::reconstruction::ConditionsProducts conditions{gainWidthsProduct,
0206                                                       gainsProduct,
0207                                                       lutCorrsProduct,
0208                                                       pedestalWidthsProduct,
0209                                                       effectivePedestalWidthsProduct,
0210                                                       pedestalsProduct,
0211                                                       qieCodersProduct,
0212                                                       chQualProduct,
0213                                                       recoParamsProduct,
0214                                                       respCorrsProduct,
0215                                                       timeCorrsProduct,
0216                                                       qieTypesProduct,
0217                                                       sipmParametersProduct,
0218                                                       sipmCharacteristicsProduct,
0219                                                       effectivePedestalsProduct,
0220                                                       &topology,
0221                                                       &recConstants,
0222                                                       pedestals.offsetForHashes(),
0223                                                       pulseOffsetsProduct,
0224                                                       pulseOffsets.getValues()};
0225 
0226   // scratch mem on device
0227   hcal::reconstruction::ScratchDataGPU scratchGPU = {
0228       cms::cuda::make_device_unique<float[]>(configParameters_.maxChannels * configParameters_.maxTimeSamples,
0229                                              ctx.stream()),
0230       cms::cuda::make_device_unique<float[]>(configParameters_.maxChannels * configParameters_.maxTimeSamples,
0231                                              ctx.stream()),
0232       cms::cuda::make_device_unique<float[]>(configParameters_.maxChannels * configParameters_.maxTimeSamples,
0233                                              ctx.stream()),
0234       cms::cuda::make_device_unique<float[]>(
0235           configParameters_.maxChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples,
0236           ctx.stream()),
0237       cms::cuda::make_device_unique<float[]>(
0238           configParameters_.maxChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples,
0239           ctx.stream()),
0240       cms::cuda::make_device_unique<float[]>(
0241           configParameters_.maxChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples,
0242           ctx.stream()),
0243       cms::cuda::make_device_unique<int8_t[]>(configParameters_.maxChannels, ctx.stream()),
0244   };
0245 
0246   // output dev mem
0247   outputGPU_.allocate(configParameters_, ctx.stream());
0248 
0249   hcal::reconstruction::entryPoint(inputGPU, outputGPU_, conditions, scratchGPU, configParameters_, ctx.stream());
0250 
0251 #ifdef HCAL_MAHI_CPUDEBUG
0252   auto end = std::chrono::high_resolution_clock::now();
0253   auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
0254   std::cout << "acquire  duration = " << duration << std::endl;
0255 #endif
0256 }
0257 
0258 void HBHERecHitProducerGPU::produce(edm::Event& event, edm::EventSetup const& setup) {
0259   cms::cuda::ScopedContextProduce ctx{cudaState_};
0260   ctx.emplace(event, rechitsM0Token_, std::move(outputGPU_.recHits));
0261 }
0262 
0263 DEFINE_FWK_MODULE(HBHERecHitProducerGPU);