File indexing completed on 2022-10-05 02:13:41
0001 #include "FWCore/Framework/interface/Event.h"
0002 #include "FWCore/Framework/interface/EventSetup.h"
0003 #include "FWCore/Framework/interface/MakerMacros.h"
0004 #include "FWCore/Framework/interface/stream/EDProducer.h"
0005 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0006 #include "FWCore/ServiceRegistry/interface/Service.h"
0007 #include "HeterogeneousCore/CUDACore/interface/JobConfigurationGPURecord.h"
0008 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0009 #include "HeterogeneousCore/CUDAServices/interface/CUDAService.h"
0010
0011 #include "SimpleAlgoGPU.h"
0012
0013 class HBHERecHitProducerGPU : public edm::stream::EDProducer<edm::ExternalWork> {
0014 public:
0015 explicit HBHERecHitProducerGPU(edm::ParameterSet const&);
0016 ~HBHERecHitProducerGPU() override;
0017 static void fillDescriptions(edm::ConfigurationDescriptions&);
0018
0019 private:
0020 void acquire(edm::Event const&, edm::EventSetup const&, edm::WaitingTaskWithArenaHolder) override;
0021 void produce(edm::Event&, edm::EventSetup const&) override;
0022
0023 using IProductTypef01 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor1, calo::common::DevStoragePolicy>>;
0024 edm::EDGetTokenT<IProductTypef01> digisTokenF01HE_;
0025
0026 using IProductTypef5 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor5, calo::common::DevStoragePolicy>>;
0027 edm::EDGetTokenT<IProductTypef5> digisTokenF5HB_;
0028
0029 using IProductTypef3 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor3, calo::common::DevStoragePolicy>>;
0030 edm::EDGetTokenT<IProductTypef3> digisTokenF3HB_;
0031
0032 using RecHitType = hcal::RecHitCollection<calo::common::DevStoragePolicy>;
0033 using OProductType = cms::cuda::Product<RecHitType>;
0034 edm::EDPutTokenT<OProductType> rechitsM0Token_;
0035
0036 const edm::ESGetToken<HcalRecoParamsWithPulseShapesGPU, HcalRecoParamsRcd> recoParamsToken_;
0037 const edm::ESGetToken<HcalGainWidthsGPU, HcalGainWidthsRcd> gainWidthsToken_;
0038 const edm::ESGetToken<HcalGainsGPU, HcalGainsRcd> gainsToken_;
0039 const edm::ESGetToken<HcalLUTCorrsGPU, HcalLUTCorrsRcd> lutCorrsToken_;
0040 const edm::ESGetToken<HcalConvertedPedestalWidthsGPU, HcalConvertedPedestalWidthsRcd> pedestalWidthsToken_;
0041 const edm::ESGetToken<HcalConvertedEffectivePedestalWidthsGPU, HcalConvertedPedestalWidthsRcd>
0042 effectivePedestalWidthsToken_;
0043 const edm::ESGetToken<HcalConvertedPedestalsGPU, HcalConvertedPedestalsRcd> pedestalsToken_;
0044 edm::ESGetToken<HcalConvertedEffectivePedestalsGPU, HcalConvertedPedestalsRcd> effectivePedestalsToken_;
0045 const edm::ESGetToken<HcalQIECodersGPU, HcalQIEDataRcd> qieCodersToken_;
0046 const edm::ESGetToken<HcalRespCorrsGPU, HcalRespCorrsRcd> respCorrsToken_;
0047 const edm::ESGetToken<HcalTimeCorrsGPU, HcalTimeCorrsRcd> timeCorrsToken_;
0048 const edm::ESGetToken<HcalQIETypesGPU, HcalQIETypesRcd> qieTypesToken_;
0049 const edm::ESGetToken<HcalTopology, HcalRecNumberingRecord> topologyToken_;
0050 const edm::ESGetToken<HcalDDDRecConstants, HcalRecNumberingRecord> recConstantsToken_;
0051 const edm::ESGetToken<HcalSiPMParametersGPU, HcalSiPMParametersRcd> sipmParametersToken_;
0052 const edm::ESGetToken<HcalSiPMCharacteristicsGPU, HcalSiPMCharacteristicsRcd> sipmCharacteristicsToken_;
0053 const edm::ESGetToken<HcalChannelQualityGPU, HcalChannelQualityRcd> chQualProductToken_;
0054 const edm::ESGetToken<HcalMahiPulseOffsetsGPU, JobConfigurationGPURecord> pulseOffsetsToken_;
0055
0056 hcal::reconstruction::ConfigParameters configParameters_;
0057 hcal::reconstruction::OutputDataGPU outputGPU_;
0058 cms::cuda::ContextState cudaState_;
0059 };
0060
0061 HBHERecHitProducerGPU::HBHERecHitProducerGPU(edm::ParameterSet const& ps)
0062 : digisTokenF01HE_{consumes<IProductTypef01>(ps.getParameter<edm::InputTag>("digisLabelF01HE"))},
0063 digisTokenF5HB_{consumes<IProductTypef5>(ps.getParameter<edm::InputTag>("digisLabelF5HB"))},
0064 digisTokenF3HB_{consumes<IProductTypef3>(ps.getParameter<edm::InputTag>("digisLabelF3HB"))},
0065 rechitsM0Token_{produces<OProductType>(ps.getParameter<std::string>("recHitsLabelM0HBHE"))},
0066 recoParamsToken_{esConsumes()},
0067 gainWidthsToken_{esConsumes()},
0068 gainsToken_{esConsumes()},
0069 lutCorrsToken_{esConsumes()},
0070 pedestalWidthsToken_{esConsumes()},
0071 effectivePedestalWidthsToken_{esConsumes()},
0072 pedestalsToken_{esConsumes()},
0073 qieCodersToken_{esConsumes()},
0074 respCorrsToken_{esConsumes()},
0075 timeCorrsToken_{esConsumes()},
0076 qieTypesToken_{esConsumes()},
0077 topologyToken_{esConsumes()},
0078 recConstantsToken_{esConsumes()},
0079 sipmParametersToken_{esConsumes()},
0080 sipmCharacteristicsToken_{esConsumes()},
0081 chQualProductToken_{esConsumes()},
0082 pulseOffsetsToken_{esConsumes()} {
0083 configParameters_.maxTimeSamples = ps.getParameter<uint32_t>("maxTimeSamples");
0084 configParameters_.kprep1dChannelsPerBlock = ps.getParameter<uint32_t>("kprep1dChannelsPerBlock");
0085 configParameters_.sipmQTSShift = ps.getParameter<int>("sipmQTSShift");
0086 configParameters_.sipmQNTStoSum = ps.getParameter<int>("sipmQNTStoSum");
0087 configParameters_.firstSampleShift = ps.getParameter<int>("firstSampleShift");
0088 configParameters_.useEffectivePedestals = ps.getParameter<bool>("useEffectivePedestals");
0089 if (configParameters_.useEffectivePedestals) {
0090 effectivePedestalsToken_ = esConsumes();
0091 }
0092
0093 configParameters_.meanTime = ps.getParameter<double>("meanTime");
0094 configParameters_.timeSigmaSiPM = ps.getParameter<double>("timeSigmaSiPM");
0095 configParameters_.timeSigmaHPD = ps.getParameter<double>("timeSigmaHPD");
0096 configParameters_.ts4Thresh = ps.getParameter<double>("ts4Thresh");
0097
0098 configParameters_.applyTimeSlew = ps.getParameter<bool>("applyTimeSlew");
0099 auto const tzeroValues = ps.getParameter<std::vector<double>>("tzeroTimeSlewParameters");
0100 auto const slopeValues = ps.getParameter<std::vector<double>>("slopeTimeSlewParameters");
0101 auto const tmaxValues = ps.getParameter<std::vector<double>>("tmaxTimeSlewParameters");
0102
0103 configParameters_.tzeroTimeSlew = tzeroValues[HcalTimeSlew::Medium];
0104 configParameters_.slopeTimeSlew = slopeValues[HcalTimeSlew::Medium];
0105 configParameters_.tmaxTimeSlew = tmaxValues[HcalTimeSlew::Medium];
0106
0107 auto threadsMinimize = ps.getParameter<std::vector<uint32_t>>("kernelMinimizeThreads");
0108 configParameters_.kernelMinimizeThreads[0] = threadsMinimize[0];
0109 configParameters_.kernelMinimizeThreads[1] = threadsMinimize[1];
0110 configParameters_.kernelMinimizeThreads[2] = threadsMinimize[2];
0111 }
0112
0113 HBHERecHitProducerGPU::~HBHERecHitProducerGPU() {}
0114
0115 void HBHERecHitProducerGPU::fillDescriptions(edm::ConfigurationDescriptions& cdesc) {
0116 edm::ParameterSetDescription desc;
0117 desc.add<uint32_t>("maxTimeSamples", 10);
0118 desc.add<uint32_t>("kprep1dChannelsPerBlock", 32);
0119 desc.add<edm::InputTag>("digisLabelF01HE", edm::InputTag{"hcalRawToDigiGPU", "f01HEDigisGPU"});
0120 desc.add<edm::InputTag>("digisLabelF5HB", edm::InputTag{"hcalRawToDigiGPU", "f5HBDigisGPU"});
0121 desc.add<edm::InputTag>("digisLabelF3HB", edm::InputTag{"hcalRawToDigiGPU", "f3HBDigisGPU"});
0122 desc.add<std::string>("recHitsLabelM0HBHE", "recHitsM0HBHE");
0123 desc.add<int>("sipmQTSShift", 0);
0124 desc.add<int>("sipmQNTStoSum", 3);
0125 desc.add<int>("firstSampleShift", 0);
0126 desc.add<bool>("useEffectivePedestals", true);
0127
0128 desc.add<double>("meanTime", 0.f);
0129 desc.add<double>("timeSigmaSiPM", 2.5f);
0130 desc.add<double>("timeSigmaHPD", 5.0f);
0131 desc.add<double>("ts4Thresh", 0.0);
0132
0133 desc.add<bool>("applyTimeSlew", true);
0134 desc.add<std::vector<double>>("tzeroTimeSlewParameters", {23.960177, 11.977461, 9.109694});
0135 desc.add<std::vector<double>>("slopeTimeSlewParameters", {-3.178648, -1.5610227, -1.075824});
0136 desc.add<std::vector<double>>("tmaxTimeSlewParameters", {16.00, 10.00, 6.25});
0137 desc.add<std::vector<uint32_t>>("kernelMinimizeThreads", {16, 1, 1});
0138
0139 cdesc.addWithDefaultLabel(desc);
0140 }
0141
0142 void HBHERecHitProducerGPU::acquire(edm::Event const& event,
0143 edm::EventSetup const& setup,
0144 edm::WaitingTaskWithArenaHolder holder) {
0145 #ifdef HCAL_MAHI_CPUDEBUG
0146 auto start = std::chrono::high_resolution_clock::now();
0147 #endif
0148
0149
0150 auto const& f01HEProduct = event.get(digisTokenF01HE_);
0151 auto const& f5HBProduct = event.get(digisTokenF5HB_);
0152 auto const& f3HBProduct = event.get(digisTokenF3HB_);
0153 cms::cuda::ScopedContextAcquire ctx{f01HEProduct, std::move(holder), cudaState_};
0154 auto const& f01HEDigis = ctx.get(f01HEProduct);
0155 auto const& f5HBDigis = ctx.get(f5HBProduct);
0156 auto const& f3HBDigis = ctx.get(f3HBProduct);
0157 auto const totalChannels = f01HEDigis.size + f5HBDigis.size + f3HBDigis.size;
0158
0159 hcal::reconstruction::InputDataGPU inputGPU{f01HEDigis, f5HBDigis, f3HBDigis};
0160
0161
0162 auto const& recoParamsProduct = setup.getData(recoParamsToken_).getProduct(ctx.stream());
0163
0164 auto const& gainWidthsProduct = setup.getData(gainWidthsToken_).getProduct(ctx.stream());
0165
0166 auto const& gainsProduct = setup.getData(gainsToken_).getProduct(ctx.stream());
0167
0168 auto const& lutCorrsProduct = setup.getData(lutCorrsToken_).getProduct(ctx.stream());
0169
0170
0171 auto const& pedestalWidthsProduct = setup.getData(pedestalWidthsToken_).getProduct(ctx.stream());
0172 auto const& effectivePedestalWidthsProduct = setup.getData(effectivePedestalWidthsToken_).getProduct(ctx.stream());
0173
0174 auto const& pedestals = setup.getData(pedestalsToken_);
0175 auto const& pedestalsProduct = pedestals.getProduct(ctx.stream());
0176
0177 edm::ESHandle<HcalConvertedEffectivePedestalsGPU> effectivePedestalsHandle;
0178 if (configParameters_.useEffectivePedestals)
0179 effectivePedestalsHandle = setup.getHandle(effectivePedestalsToken_);
0180 auto const* effectivePedestalsProduct =
0181 configParameters_.useEffectivePedestals ? &effectivePedestalsHandle->getProduct(ctx.stream()) : nullptr;
0182
0183 auto const& qieCodersProduct = setup.getData(qieCodersToken_).getProduct(ctx.stream());
0184
0185 auto const& respCorrsProduct = setup.getData(respCorrsToken_).getProduct(ctx.stream());
0186
0187 auto const& timeCorrsProduct = setup.getData(timeCorrsToken_).getProduct(ctx.stream());
0188
0189 auto const& qieTypesProduct = setup.getData(qieTypesToken_).getProduct(ctx.stream());
0190
0191 HcalTopology const& topology = setup.getData(topologyToken_);
0192 HcalDDDRecConstants const& recConstants = setup.getData(recConstantsToken_);
0193
0194 auto const& sipmParametersProduct = setup.getData(sipmParametersToken_).getProduct(ctx.stream());
0195
0196 auto const& sipmCharacteristicsProduct = setup.getData(sipmCharacteristicsToken_).getProduct(ctx.stream());
0197
0198 auto const& chQualProduct = setup.getData(chQualProductToken_).getProduct(ctx.stream());
0199
0200 auto const& pulseOffsets = setup.getData(pulseOffsetsToken_);
0201 auto const& pulseOffsetsProduct = pulseOffsets.getProduct(ctx.stream());
0202
0203
0204 hcal::reconstruction::ConditionsProducts conditions{gainWidthsProduct,
0205 gainsProduct,
0206 lutCorrsProduct,
0207 pedestalWidthsProduct,
0208 effectivePedestalWidthsProduct,
0209 pedestalsProduct,
0210 qieCodersProduct,
0211 chQualProduct,
0212 recoParamsProduct,
0213 respCorrsProduct,
0214 timeCorrsProduct,
0215 qieTypesProduct,
0216 sipmParametersProduct,
0217 sipmCharacteristicsProduct,
0218 effectivePedestalsProduct,
0219 &topology,
0220 &recConstants,
0221 pedestals.offsetForHashes(),
0222 pulseOffsetsProduct,
0223 pulseOffsets.getValues()};
0224
0225
0226 hcal::reconstruction::ScratchDataGPU scratchGPU = {
0227 cms::cuda::make_device_unique<float[]>(totalChannels * configParameters_.maxTimeSamples, ctx.stream()),
0228 cms::cuda::make_device_unique<float[]>(totalChannels * configParameters_.maxTimeSamples, ctx.stream()),
0229 cms::cuda::make_device_unique<float[]>(totalChannels * configParameters_.maxTimeSamples, ctx.stream()),
0230 cms::cuda::make_device_unique<float[]>(
0231 totalChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples, ctx.stream()),
0232 cms::cuda::make_device_unique<float[]>(
0233 totalChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples, ctx.stream()),
0234 cms::cuda::make_device_unique<float[]>(
0235 totalChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples, ctx.stream()),
0236 cms::cuda::make_device_unique<int8_t[]>(totalChannels, ctx.stream()),
0237 };
0238
0239
0240 outputGPU_.allocate(configParameters_, totalChannels, ctx.stream());
0241
0242 hcal::reconstruction::entryPoint(inputGPU, outputGPU_, conditions, scratchGPU, configParameters_, ctx.stream());
0243
0244 #ifdef HCAL_MAHI_CPUDEBUG
0245 auto end = std::chrono::high_resolution_clock::now();
0246 auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
0247 std::cout << "acquire duration = " << duration << std::endl;
0248 #endif
0249 }
0250
0251 void HBHERecHitProducerGPU::produce(edm::Event& event, edm::EventSetup const& setup) {
0252 cms::cuda::ScopedContextProduce ctx{cudaState_};
0253 ctx.emplace(event, rechitsM0Token_, std::move(outputGPU_.recHits));
0254 }
0255
0256 DEFINE_FWK_MODULE(HBHERecHitProducerGPU);
