File indexing completed on 2021-09-28 23:48:50
0001 #include "FWCore/Framework/interface/Event.h"
0002 #include "FWCore/Framework/interface/EventSetup.h"
0003 #include "FWCore/Framework/interface/MakerMacros.h"
0004 #include "FWCore/Framework/interface/stream/EDProducer.h"
0005 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0006 #include "FWCore/ServiceRegistry/interface/Service.h"
0007 #include "HeterogeneousCore/CUDACore/interface/JobConfigurationGPURecord.h"
0008 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0009 #include "HeterogeneousCore/CUDAServices/interface/CUDAService.h"
0010
0011 #include "SimpleAlgoGPU.h"
0012
0013 class HBHERecHitProducerGPU : public edm::stream::EDProducer<edm::ExternalWork> {
0014 public:
0015 explicit HBHERecHitProducerGPU(edm::ParameterSet const&);
0016 ~HBHERecHitProducerGPU() override;
0017 static void fillDescriptions(edm::ConfigurationDescriptions&);
0018
0019 private:
0020 void acquire(edm::Event const&, edm::EventSetup const&, edm::WaitingTaskWithArenaHolder) override;
0021 void produce(edm::Event&, edm::EventSetup const&) override;
0022
0023 using IProductTypef01 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor1, calo::common::DevStoragePolicy>>;
0024 edm::EDGetTokenT<IProductTypef01> digisTokenF01HE_;
0025
0026 using IProductTypef5 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor5, calo::common::DevStoragePolicy>>;
0027 edm::EDGetTokenT<IProductTypef5> digisTokenF5HB_;
0028
0029 using IProductTypef3 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor3, calo::common::DevStoragePolicy>>;
0030 edm::EDGetTokenT<IProductTypef3> digisTokenF3HB_;
0031
0032 using RecHitType = hcal::RecHitCollection<calo::common::DevStoragePolicy>;
0033 using OProductType = cms::cuda::Product<RecHitType>;
0034 edm::EDPutTokenT<OProductType> rechitsM0Token_;
0035
0036 const edm::ESGetToken<HcalRecoParamsWithPulseShapesGPU, HcalRecoParamsRcd> recoParamsToken_;
0037 const edm::ESGetToken<HcalGainWidthsGPU, HcalGainWidthsRcd> gainWidthsToken_;
0038 const edm::ESGetToken<HcalGainsGPU, HcalGainsRcd> gainsToken_;
0039 const edm::ESGetToken<HcalLUTCorrsGPU, HcalLUTCorrsRcd> lutCorrsToken_;
0040 const edm::ESGetToken<HcalConvertedPedestalWidthsGPU, HcalConvertedPedestalWidthsRcd> pedestalWidthsToken_;
0041 const edm::ESGetToken<HcalConvertedEffectivePedestalWidthsGPU, HcalConvertedPedestalWidthsRcd>
0042 effectivePedestalWidthsToken_;
0043 const edm::ESGetToken<HcalConvertedPedestalsGPU, HcalConvertedPedestalsRcd> pedestalsToken_;
0044 edm::ESGetToken<HcalConvertedEffectivePedestalsGPU, HcalConvertedPedestalsRcd> effectivePedestalsToken_;
0045 const edm::ESGetToken<HcalQIECodersGPU, HcalQIEDataRcd> qieCodersToken_;
0046 const edm::ESGetToken<HcalRespCorrsGPU, HcalRespCorrsRcd> respCorrsToken_;
0047 const edm::ESGetToken<HcalTimeCorrsGPU, HcalTimeCorrsRcd> timeCorrsToken_;
0048 const edm::ESGetToken<HcalQIETypesGPU, HcalQIETypesRcd> qieTypesToken_;
0049 const edm::ESGetToken<HcalTopology, HcalRecNumberingRecord> topologyToken_;
0050 const edm::ESGetToken<HcalDDDRecConstants, HcalRecNumberingRecord> recConstantsToken_;
0051 const edm::ESGetToken<HcalSiPMParametersGPU, HcalSiPMParametersRcd> sipmParametersToken_;
0052 const edm::ESGetToken<HcalSiPMCharacteristicsGPU, HcalSiPMCharacteristicsRcd> sipmCharacteristicsToken_;
0053 const edm::ESGetToken<HcalChannelQualityGPU, HcalChannelQualityRcd> chQualProductToken_;
0054 const edm::ESGetToken<HcalMahiPulseOffsetsGPU, JobConfigurationGPURecord> pulseOffsetsToken_;
0055
0056 hcal::reconstruction::ConfigParameters configParameters_;
0057 hcal::reconstruction::OutputDataGPU outputGPU_;
0058 cms::cuda::ContextState cudaState_;
0059 };
0060
0061 HBHERecHitProducerGPU::HBHERecHitProducerGPU(edm::ParameterSet const& ps)
0062 : digisTokenF01HE_{consumes<IProductTypef01>(ps.getParameter<edm::InputTag>("digisLabelF01HE"))},
0063 digisTokenF5HB_{consumes<IProductTypef5>(ps.getParameter<edm::InputTag>("digisLabelF5HB"))},
0064 digisTokenF3HB_{consumes<IProductTypef3>(ps.getParameter<edm::InputTag>("digisLabelF3HB"))},
0065 rechitsM0Token_{produces<OProductType>(ps.getParameter<std::string>("recHitsLabelM0HBHE"))},
0066 recoParamsToken_{esConsumes()},
0067 gainWidthsToken_{esConsumes()},
0068 gainsToken_{esConsumes()},
0069 lutCorrsToken_{esConsumes()},
0070 pedestalWidthsToken_{esConsumes()},
0071 effectivePedestalWidthsToken_{esConsumes()},
0072 pedestalsToken_{esConsumes()},
0073 qieCodersToken_{esConsumes()},
0074 respCorrsToken_{esConsumes()},
0075 timeCorrsToken_{esConsumes()},
0076 qieTypesToken_{esConsumes()},
0077 topologyToken_{esConsumes()},
0078 recConstantsToken_{esConsumes()},
0079 sipmParametersToken_{esConsumes()},
0080 sipmCharacteristicsToken_{esConsumes()},
0081 chQualProductToken_{esConsumes()},
0082 pulseOffsetsToken_{esConsumes()} {
0083 configParameters_.maxChannels = ps.getParameter<uint32_t>("maxChannels");
0084 configParameters_.maxTimeSamples = ps.getParameter<uint32_t>("maxTimeSamples");
0085 configParameters_.kprep1dChannelsPerBlock = ps.getParameter<uint32_t>("kprep1dChannelsPerBlock");
0086 configParameters_.sipmQTSShift = ps.getParameter<int>("sipmQTSShift");
0087 configParameters_.sipmQNTStoSum = ps.getParameter<int>("sipmQNTStoSum");
0088 configParameters_.firstSampleShift = ps.getParameter<int>("firstSampleShift");
0089 configParameters_.useEffectivePedestals = ps.getParameter<bool>("useEffectivePedestals");
0090 if (configParameters_.useEffectivePedestals) {
0091 effectivePedestalsToken_ = esConsumes();
0092 }
0093
0094 configParameters_.meanTime = ps.getParameter<double>("meanTime");
0095 configParameters_.timeSigmaSiPM = ps.getParameter<double>("timeSigmaSiPM");
0096 configParameters_.timeSigmaHPD = ps.getParameter<double>("timeSigmaHPD");
0097 configParameters_.ts4Thresh = ps.getParameter<double>("ts4Thresh");
0098
0099 configParameters_.applyTimeSlew = ps.getParameter<bool>("applyTimeSlew");
0100 auto const tzeroValues = ps.getParameter<std::vector<double>>("tzeroTimeSlewParameters");
0101 auto const slopeValues = ps.getParameter<std::vector<double>>("slopeTimeSlewParameters");
0102 auto const tmaxValues = ps.getParameter<std::vector<double>>("tmaxTimeSlewParameters");
0103
0104 configParameters_.tzeroTimeSlew = tzeroValues[HcalTimeSlew::Medium];
0105 configParameters_.slopeTimeSlew = slopeValues[HcalTimeSlew::Medium];
0106 configParameters_.tmaxTimeSlew = tmaxValues[HcalTimeSlew::Medium];
0107
0108 auto threadsMinimize = ps.getParameter<std::vector<uint32_t>>("kernelMinimizeThreads");
0109 configParameters_.kernelMinimizeThreads[0] = threadsMinimize[0];
0110 configParameters_.kernelMinimizeThreads[1] = threadsMinimize[1];
0111 configParameters_.kernelMinimizeThreads[2] = threadsMinimize[2];
0112 }
0113
0114 HBHERecHitProducerGPU::~HBHERecHitProducerGPU() {}
0115
0116 void HBHERecHitProducerGPU::fillDescriptions(edm::ConfigurationDescriptions& cdesc) {
0117 edm::ParameterSetDescription desc;
0118 desc.add<uint32_t>("maxChannels", 10000u);
0119 desc.add<uint32_t>("maxTimeSamples", 10);
0120 desc.add<uint32_t>("kprep1dChannelsPerBlock", 32);
0121 desc.add<edm::InputTag>("digisLabelF01HE", edm::InputTag{"hcalRawToDigiGPU", "f01HEDigisGPU"});
0122 desc.add<edm::InputTag>("digisLabelF5HB", edm::InputTag{"hcalRawToDigiGPU", "f5HBDigisGPU"});
0123 desc.add<edm::InputTag>("digisLabelF3HB", edm::InputTag{"hcalRawToDigiGPU", "f3HBDigisGPU"});
0124 desc.add<std::string>("recHitsLabelM0HBHE", "recHitsM0HBHE");
0125 desc.add<int>("sipmQTSShift", 0);
0126 desc.add<int>("sipmQNTStoSum", 3);
0127 desc.add<int>("firstSampleShift", 0);
0128 desc.add<bool>("useEffectivePedestals", true);
0129
0130 desc.add<double>("meanTime", 0.f);
0131 desc.add<double>("timeSigmaSiPM", 2.5f);
0132 desc.add<double>("timeSigmaHPD", 5.0f);
0133 desc.add<double>("ts4Thresh", 0.0);
0134
0135 desc.add<bool>("applyTimeSlew", true);
0136 desc.add<std::vector<double>>("tzeroTimeSlewParameters", {23.960177, 11.977461, 9.109694});
0137 desc.add<std::vector<double>>("slopeTimeSlewParameters", {-3.178648, -1.5610227, -1.075824});
0138 desc.add<std::vector<double>>("tmaxTimeSlewParameters", {16.00, 10.00, 6.25});
0139 desc.add<std::vector<uint32_t>>("kernelMinimizeThreads", {16, 1, 1});
0140
0141 cdesc.addWithDefaultLabel(desc);
0142 }
0143
0144 void HBHERecHitProducerGPU::acquire(edm::Event const& event,
0145 edm::EventSetup const& setup,
0146 edm::WaitingTaskWithArenaHolder holder) {
0147 #ifdef HCAL_MAHI_CPUDEBUG
0148 auto start = std::chrono::high_resolution_clock::now();
0149 #endif
0150
0151
0152 auto const& f01HEProduct = event.get(digisTokenF01HE_);
0153 auto const& f5HBProduct = event.get(digisTokenF5HB_);
0154 auto const& f3HBProduct = event.get(digisTokenF3HB_);
0155 cms::cuda::ScopedContextAcquire ctx{f01HEProduct, std::move(holder), cudaState_};
0156 auto const& f01HEDigis = ctx.get(f01HEProduct);
0157 auto const& f5HBDigis = ctx.get(f5HBProduct);
0158 auto const& f3HBDigis = ctx.get(f3HBProduct);
0159
0160 hcal::reconstruction::InputDataGPU inputGPU{f01HEDigis, f5HBDigis, f3HBDigis};
0161
0162
0163 auto const& recoParamsProduct = setup.getData(recoParamsToken_).getProduct(ctx.stream());
0164
0165 auto const& gainWidthsProduct = setup.getData(gainWidthsToken_).getProduct(ctx.stream());
0166
0167 auto const& gainsProduct = setup.getData(gainsToken_).getProduct(ctx.stream());
0168
0169 auto const& lutCorrsProduct = setup.getData(lutCorrsToken_).getProduct(ctx.stream());
0170
0171
0172 auto const& pedestalWidthsProduct = setup.getData(pedestalWidthsToken_).getProduct(ctx.stream());
0173 auto const& effectivePedestalWidthsProduct = setup.getData(effectivePedestalWidthsToken_).getProduct(ctx.stream());
0174
0175 auto const& pedestals = setup.getData(pedestalsToken_);
0176 auto const& pedestalsProduct = pedestals.getProduct(ctx.stream());
0177
0178 edm::ESHandle<HcalConvertedEffectivePedestalsGPU> effectivePedestalsHandle;
0179 if (configParameters_.useEffectivePedestals)
0180 effectivePedestalsHandle = setup.getHandle(effectivePedestalsToken_);
0181 auto const* effectivePedestalsProduct =
0182 configParameters_.useEffectivePedestals ? &effectivePedestalsHandle->getProduct(ctx.stream()) : nullptr;
0183
0184 auto const& qieCodersProduct = setup.getData(qieCodersToken_).getProduct(ctx.stream());
0185
0186 auto const& respCorrsProduct = setup.getData(respCorrsToken_).getProduct(ctx.stream());
0187
0188 auto const& timeCorrsProduct = setup.getData(timeCorrsToken_).getProduct(ctx.stream());
0189
0190 auto const& qieTypesProduct = setup.getData(qieTypesToken_).getProduct(ctx.stream());
0191
0192 HcalTopology const& topology = setup.getData(topologyToken_);
0193 HcalDDDRecConstants const& recConstants = setup.getData(recConstantsToken_);
0194
0195 auto const& sipmParametersProduct = setup.getData(sipmParametersToken_).getProduct(ctx.stream());
0196
0197 auto const& sipmCharacteristicsProduct = setup.getData(sipmCharacteristicsToken_).getProduct(ctx.stream());
0198
0199 auto const& chQualProduct = setup.getData(chQualProductToken_).getProduct(ctx.stream());
0200
0201 auto const& pulseOffsets = setup.getData(pulseOffsetsToken_);
0202 auto const& pulseOffsetsProduct = pulseOffsets.getProduct(ctx.stream());
0203
0204
0205 hcal::reconstruction::ConditionsProducts conditions{gainWidthsProduct,
0206 gainsProduct,
0207 lutCorrsProduct,
0208 pedestalWidthsProduct,
0209 effectivePedestalWidthsProduct,
0210 pedestalsProduct,
0211 qieCodersProduct,
0212 chQualProduct,
0213 recoParamsProduct,
0214 respCorrsProduct,
0215 timeCorrsProduct,
0216 qieTypesProduct,
0217 sipmParametersProduct,
0218 sipmCharacteristicsProduct,
0219 effectivePedestalsProduct,
0220 &topology,
0221 &recConstants,
0222 pedestals.offsetForHashes(),
0223 pulseOffsetsProduct,
0224 pulseOffsets.getValues()};
0225
0226
0227 hcal::reconstruction::ScratchDataGPU scratchGPU = {
0228 cms::cuda::make_device_unique<float[]>(configParameters_.maxChannels * configParameters_.maxTimeSamples,
0229 ctx.stream()),
0230 cms::cuda::make_device_unique<float[]>(configParameters_.maxChannels * configParameters_.maxTimeSamples,
0231 ctx.stream()),
0232 cms::cuda::make_device_unique<float[]>(configParameters_.maxChannels * configParameters_.maxTimeSamples,
0233 ctx.stream()),
0234 cms::cuda::make_device_unique<float[]>(
0235 configParameters_.maxChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples,
0236 ctx.stream()),
0237 cms::cuda::make_device_unique<float[]>(
0238 configParameters_.maxChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples,
0239 ctx.stream()),
0240 cms::cuda::make_device_unique<float[]>(
0241 configParameters_.maxChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples,
0242 ctx.stream()),
0243 cms::cuda::make_device_unique<int8_t[]>(configParameters_.maxChannels, ctx.stream()),
0244 };
0245
0246
0247 outputGPU_.allocate(configParameters_, ctx.stream());
0248
0249 hcal::reconstruction::entryPoint(inputGPU, outputGPU_, conditions, scratchGPU, configParameters_, ctx.stream());
0250
0251 #ifdef HCAL_MAHI_CPUDEBUG
0252 auto end = std::chrono::high_resolution_clock::now();
0253 auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
0254 std::cout << "acquire duration = " << duration << std::endl;
0255 #endif
0256 }
0257
0258 void HBHERecHitProducerGPU::produce(edm::Event& event, edm::EventSetup const& setup) {
0259 cms::cuda::ScopedContextProduce ctx{cudaState_};
0260 ctx.emplace(event, rechitsM0Token_, std::move(outputGPU_.recHits));
0261 }
0262
0263 DEFINE_FWK_MODULE(HBHERecHitProducerGPU);
