File indexing completed on 2024-04-06 12:25:51
0001 #include "FWCore/Framework/interface/Event.h"
0002 #include "FWCore/Framework/interface/EventSetup.h"
0003 #include "FWCore/Framework/interface/MakerMacros.h"
0004 #include "FWCore/Framework/interface/stream/EDProducer.h"
0005 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0006 #include "FWCore/ServiceRegistry/interface/Service.h"
0007 #include "HeterogeneousCore/CUDACore/interface/JobConfigurationGPURecord.h"
0008 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0009 #include "HeterogeneousCore/CUDAServices/interface/CUDAInterface.h"
0010
0011 #include "SimpleAlgoGPU.h"
0012
0013 #include "CondFormats/DataRecord/interface/HcalCombinedRecordsGPU.h"
0014 #include "CondFormats/DataRecord/interface/HcalGainWidthsRcd.h"
0015 #include "CondFormats/DataRecord/interface/HcalGainsRcd.h"
0016 #include "CondFormats/DataRecord/interface/HcalLUTCorrsRcd.h"
0017 #include "CondFormats/DataRecord/interface/HcalQIEDataRcd.h"
0018 #include "CondFormats/DataRecord/interface/HcalQIETypesRcd.h"
0019 #include "CondFormats/DataRecord/interface/HcalRecoParamsRcd.h"
0020 #include "CondFormats/DataRecord/interface/HcalRespCorrsRcd.h"
0021 #include "CondFormats/DataRecord/interface/HcalSiPMCharacteristicsRcd.h"
0022 #include "CondFormats/DataRecord/interface/HcalSiPMParametersRcd.h"
0023 #include "CondFormats/DataRecord/interface/HcalTimeCorrsRcd.h"
0024 #include "CondFormats/DataRecord/interface/HcalChannelQualityRcd.h"
0025
0026 class HBHERecHitProducerGPU : public edm::stream::EDProducer<edm::ExternalWork> {
0027 public:
0028 explicit HBHERecHitProducerGPU(edm::ParameterSet const&);
0029 ~HBHERecHitProducerGPU() override;
0030 static void fillDescriptions(edm::ConfigurationDescriptions&);
0031
0032 private:
0033 void acquire(edm::Event const&, edm::EventSetup const&, edm::WaitingTaskWithArenaHolder) override;
0034 void produce(edm::Event&, edm::EventSetup const&) override;
0035
0036 using IProductTypef01 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor1, calo::common::DevStoragePolicy>>;
0037 edm::EDGetTokenT<IProductTypef01> digisTokenF01HE_;
0038
0039 using IProductTypef5 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor5, calo::common::DevStoragePolicy>>;
0040 edm::EDGetTokenT<IProductTypef5> digisTokenF5HB_;
0041
0042 using IProductTypef3 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor3, calo::common::DevStoragePolicy>>;
0043 edm::EDGetTokenT<IProductTypef3> digisTokenF3HB_;
0044
0045 using RecHitType = hcal::RecHitCollection<calo::common::DevStoragePolicy>;
0046 using OProductType = cms::cuda::Product<RecHitType>;
0047 edm::EDPutTokenT<OProductType> rechitsM0Token_;
0048
0049 const edm::ESGetToken<HcalRecoParamsWithPulseShapesGPU, HcalRecoParamsRcd> recoParamsToken_;
0050 const edm::ESGetToken<HcalGainWidthsGPU, HcalGainWidthsRcd> gainWidthsToken_;
0051 const edm::ESGetToken<HcalGainsGPU, HcalGainsRcd> gainsToken_;
0052 const edm::ESGetToken<HcalLUTCorrsGPU, HcalLUTCorrsRcd> lutCorrsToken_;
0053 const edm::ESGetToken<HcalConvertedPedestalWidthsGPU, HcalConvertedPedestalWidthsRcd> pedestalWidthsToken_;
0054 const edm::ESGetToken<HcalConvertedEffectivePedestalWidthsGPU, HcalConvertedPedestalWidthsRcd>
0055 effectivePedestalWidthsToken_;
0056 const edm::ESGetToken<HcalConvertedPedestalsGPU, HcalConvertedPedestalsRcd> pedestalsToken_;
0057 edm::ESGetToken<HcalConvertedEffectivePedestalsGPU, HcalConvertedPedestalsRcd> effectivePedestalsToken_;
0058 const edm::ESGetToken<HcalQIECodersGPU, HcalQIEDataRcd> qieCodersToken_;
0059 const edm::ESGetToken<HcalRespCorrsGPU, HcalRespCorrsRcd> respCorrsToken_;
0060 const edm::ESGetToken<HcalTimeCorrsGPU, HcalTimeCorrsRcd> timeCorrsToken_;
0061 const edm::ESGetToken<HcalQIETypesGPU, HcalQIETypesRcd> qieTypesToken_;
0062 const edm::ESGetToken<HcalTopology, HcalRecNumberingRecord> topologyToken_;
0063 const edm::ESGetToken<HcalDDDRecConstants, HcalRecNumberingRecord> recConstantsToken_;
0064 const edm::ESGetToken<HcalSiPMParametersGPU, HcalSiPMParametersRcd> sipmParametersToken_;
0065 const edm::ESGetToken<HcalSiPMCharacteristicsGPU, HcalSiPMCharacteristicsRcd> sipmCharacteristicsToken_;
0066 const edm::ESGetToken<HcalChannelQualityGPU, HcalChannelQualityRcd> chQualProductToken_;
0067 const edm::ESGetToken<HcalMahiPulseOffsetsGPU, JobConfigurationGPURecord> pulseOffsetsToken_;
0068
0069 hcal::reconstruction::ConfigParameters configParameters_;
0070 hcal::reconstruction::OutputDataGPU outputGPU_;
0071 cms::cuda::ContextState cudaState_;
0072 };
0073
0074 HBHERecHitProducerGPU::HBHERecHitProducerGPU(edm::ParameterSet const& ps)
0075 : digisTokenF01HE_{consumes<IProductTypef01>(ps.getParameter<edm::InputTag>("digisLabelF01HE"))},
0076 digisTokenF5HB_{consumes<IProductTypef5>(ps.getParameter<edm::InputTag>("digisLabelF5HB"))},
0077 digisTokenF3HB_{consumes<IProductTypef3>(ps.getParameter<edm::InputTag>("digisLabelF3HB"))},
0078 rechitsM0Token_{produces<OProductType>(ps.getParameter<std::string>("recHitsLabelM0HBHE"))},
0079 recoParamsToken_{esConsumes()},
0080 gainWidthsToken_{esConsumes()},
0081 gainsToken_{esConsumes()},
0082 lutCorrsToken_{esConsumes()},
0083 pedestalWidthsToken_{esConsumes()},
0084 effectivePedestalWidthsToken_{esConsumes()},
0085 pedestalsToken_{esConsumes()},
0086 qieCodersToken_{esConsumes()},
0087 respCorrsToken_{esConsumes()},
0088 timeCorrsToken_{esConsumes()},
0089 qieTypesToken_{esConsumes()},
0090 topologyToken_{esConsumes()},
0091 recConstantsToken_{esConsumes()},
0092 sipmParametersToken_{esConsumes()},
0093 sipmCharacteristicsToken_{esConsumes()},
0094 chQualProductToken_{esConsumes()},
0095 pulseOffsetsToken_{esConsumes()} {
0096 configParameters_.maxTimeSamples = ps.getParameter<uint32_t>("maxTimeSamples");
0097 configParameters_.kprep1dChannelsPerBlock = ps.getParameter<uint32_t>("kprep1dChannelsPerBlock");
0098 configParameters_.sipmQTSShift = ps.getParameter<int>("sipmQTSShift");
0099 configParameters_.sipmQNTStoSum = ps.getParameter<int>("sipmQNTStoSum");
0100 configParameters_.firstSampleShift = ps.getParameter<int>("firstSampleShift");
0101 configParameters_.useEffectivePedestals = ps.getParameter<bool>("useEffectivePedestals");
0102 if (configParameters_.useEffectivePedestals) {
0103 effectivePedestalsToken_ = esConsumes();
0104 }
0105
0106 configParameters_.meanTime = ps.getParameter<double>("meanTime");
0107 configParameters_.timeSigmaSiPM = ps.getParameter<double>("timeSigmaSiPM");
0108 configParameters_.timeSigmaHPD = ps.getParameter<double>("timeSigmaHPD");
0109 configParameters_.ts4Thresh = ps.getParameter<double>("ts4Thresh");
0110
0111 configParameters_.applyTimeSlew = ps.getParameter<bool>("applyTimeSlew");
0112 auto const tzeroValues = ps.getParameter<std::vector<double>>("tzeroTimeSlewParameters");
0113 auto const slopeValues = ps.getParameter<std::vector<double>>("slopeTimeSlewParameters");
0114 auto const tmaxValues = ps.getParameter<std::vector<double>>("tmaxTimeSlewParameters");
0115
0116 configParameters_.tzeroTimeSlew = tzeroValues[HcalTimeSlew::Medium];
0117 configParameters_.slopeTimeSlew = slopeValues[HcalTimeSlew::Medium];
0118 configParameters_.tmaxTimeSlew = tmaxValues[HcalTimeSlew::Medium];
0119
0120 auto threadsMinimize = ps.getParameter<std::vector<uint32_t>>("kernelMinimizeThreads");
0121 configParameters_.kernelMinimizeThreads[0] = threadsMinimize[0];
0122 configParameters_.kernelMinimizeThreads[1] = threadsMinimize[1];
0123 configParameters_.kernelMinimizeThreads[2] = threadsMinimize[2];
0124 }
0125
0126 HBHERecHitProducerGPU::~HBHERecHitProducerGPU() {}
0127
0128 void HBHERecHitProducerGPU::fillDescriptions(edm::ConfigurationDescriptions& cdesc) {
0129 edm::ParameterSetDescription desc;
0130 desc.add<uint32_t>("maxTimeSamples", 10);
0131 desc.add<uint32_t>("kprep1dChannelsPerBlock", 32);
0132 desc.add<edm::InputTag>("digisLabelF01HE", edm::InputTag{"hcalRawToDigiGPU", "f01HEDigisGPU"});
0133 desc.add<edm::InputTag>("digisLabelF5HB", edm::InputTag{"hcalRawToDigiGPU", "f5HBDigisGPU"});
0134 desc.add<edm::InputTag>("digisLabelF3HB", edm::InputTag{"hcalRawToDigiGPU", "f3HBDigisGPU"});
0135 desc.add<std::string>("recHitsLabelM0HBHE", "recHitsM0HBHE");
0136 desc.add<int>("sipmQTSShift", 0);
0137 desc.add<int>("sipmQNTStoSum", 3);
0138 desc.add<int>("firstSampleShift", 0);
0139 desc.add<bool>("useEffectivePedestals", true);
0140
0141 desc.add<double>("meanTime", 0.f);
0142 desc.add<double>("timeSigmaSiPM", 2.5f);
0143 desc.add<double>("timeSigmaHPD", 5.0f);
0144 desc.add<double>("ts4Thresh", 0.0);
0145
0146 desc.add<bool>("applyTimeSlew", true);
0147 desc.add<std::vector<double>>("tzeroTimeSlewParameters", {23.960177, 11.977461, 9.109694});
0148 desc.add<std::vector<double>>("slopeTimeSlewParameters", {-3.178648, -1.5610227, -1.075824});
0149 desc.add<std::vector<double>>("tmaxTimeSlewParameters", {16.00, 10.00, 6.25});
0150 desc.add<std::vector<uint32_t>>("kernelMinimizeThreads", {16, 1, 1});
0151
0152 cdesc.addWithDefaultLabel(desc);
0153 }
0154
0155 void HBHERecHitProducerGPU::acquire(edm::Event const& event,
0156 edm::EventSetup const& setup,
0157 edm::WaitingTaskWithArenaHolder holder) {
0158 #ifdef HCAL_MAHI_CPUDEBUG
0159 auto start = std::chrono::high_resolution_clock::now();
0160 #endif
0161
0162
0163 auto const& f01HEProduct = event.get(digisTokenF01HE_);
0164 auto const& f5HBProduct = event.get(digisTokenF5HB_);
0165 auto const& f3HBProduct = event.get(digisTokenF3HB_);
0166 cms::cuda::ScopedContextAcquire ctx{f01HEProduct, std::move(holder), cudaState_};
0167 auto const& f01HEDigis = ctx.get(f01HEProduct);
0168 auto const& f5HBDigis = ctx.get(f5HBProduct);
0169 auto const& f3HBDigis = ctx.get(f3HBProduct);
0170 auto const totalChannels = f01HEDigis.size + f5HBDigis.size + f3HBDigis.size;
0171
0172 hcal::reconstruction::InputDataGPU inputGPU{f01HEDigis, f5HBDigis, f3HBDigis};
0173
0174
0175 auto const& recoParamsProduct = setup.getData(recoParamsToken_).getProduct(ctx.stream());
0176
0177 auto const& gainWidthsProduct = setup.getData(gainWidthsToken_).getProduct(ctx.stream());
0178
0179 auto const& gainsProduct = setup.getData(gainsToken_).getProduct(ctx.stream());
0180
0181 auto const& lutCorrsProduct = setup.getData(lutCorrsToken_).getProduct(ctx.stream());
0182
0183
0184 auto const& pedestalWidthsProduct = setup.getData(pedestalWidthsToken_).getProduct(ctx.stream());
0185 auto const& effectivePedestalWidthsProduct = setup.getData(effectivePedestalWidthsToken_).getProduct(ctx.stream());
0186
0187 auto const& pedestals = setup.getData(pedestalsToken_);
0188 auto const& pedestalsProduct = pedestals.getProduct(ctx.stream());
0189
0190 edm::ESHandle<HcalConvertedEffectivePedestalsGPU> effectivePedestalsHandle;
0191 if (configParameters_.useEffectivePedestals)
0192 effectivePedestalsHandle = setup.getHandle(effectivePedestalsToken_);
0193 auto const* effectivePedestalsProduct =
0194 configParameters_.useEffectivePedestals ? &effectivePedestalsHandle->getProduct(ctx.stream()) : nullptr;
0195
0196 auto const& qieCodersProduct = setup.getData(qieCodersToken_).getProduct(ctx.stream());
0197
0198 auto const& respCorrsProduct = setup.getData(respCorrsToken_).getProduct(ctx.stream());
0199
0200 auto const& timeCorrsProduct = setup.getData(timeCorrsToken_).getProduct(ctx.stream());
0201
0202 auto const& qieTypesProduct = setup.getData(qieTypesToken_).getProduct(ctx.stream());
0203
0204 HcalTopology const& topology = setup.getData(topologyToken_);
0205 HcalDDDRecConstants const& recConstants = setup.getData(recConstantsToken_);
0206
0207 auto const& sipmParametersProduct = setup.getData(sipmParametersToken_).getProduct(ctx.stream());
0208
0209 auto const& sipmCharacteristicsProduct = setup.getData(sipmCharacteristicsToken_).getProduct(ctx.stream());
0210
0211 auto const& chQualProduct = setup.getData(chQualProductToken_).getProduct(ctx.stream());
0212
0213 auto const& pulseOffsets = setup.getData(pulseOffsetsToken_);
0214 auto const& pulseOffsetsProduct = pulseOffsets.getProduct(ctx.stream());
0215
0216
0217 hcal::reconstruction::ConditionsProducts conditions{gainWidthsProduct,
0218 gainsProduct,
0219 lutCorrsProduct,
0220 pedestalWidthsProduct,
0221 effectivePedestalWidthsProduct,
0222 pedestalsProduct,
0223 qieCodersProduct,
0224 chQualProduct,
0225 recoParamsProduct,
0226 respCorrsProduct,
0227 timeCorrsProduct,
0228 qieTypesProduct,
0229 sipmParametersProduct,
0230 sipmCharacteristicsProduct,
0231 effectivePedestalsProduct,
0232 &topology,
0233 &recConstants,
0234 pedestals.offsetForHashes(),
0235 pulseOffsetsProduct,
0236 pulseOffsets.getValues()};
0237
0238
0239 hcal::reconstruction::ScratchDataGPU scratchGPU = {
0240 cms::cuda::make_device_unique<float[]>(totalChannels * configParameters_.maxTimeSamples, ctx.stream()),
0241 cms::cuda::make_device_unique<float[]>(totalChannels * configParameters_.maxTimeSamples, ctx.stream()),
0242 cms::cuda::make_device_unique<float[]>(totalChannels * configParameters_.maxTimeSamples, ctx.stream()),
0243 cms::cuda::make_device_unique<float[]>(
0244 totalChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples, ctx.stream()),
0245 cms::cuda::make_device_unique<float[]>(
0246 totalChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples, ctx.stream()),
0247 cms::cuda::make_device_unique<float[]>(
0248 totalChannels * configParameters_.maxTimeSamples * configParameters_.maxTimeSamples, ctx.stream()),
0249 cms::cuda::make_device_unique<int8_t[]>(totalChannels, ctx.stream()),
0250 };
0251
0252
0253 outputGPU_.allocate(configParameters_, totalChannels, ctx.stream());
0254
0255 hcal::reconstruction::entryPoint(inputGPU, outputGPU_, conditions, scratchGPU, configParameters_, ctx.stream());
0256
0257 #ifdef HCAL_MAHI_CPUDEBUG
0258 auto end = std::chrono::high_resolution_clock::now();
0259 auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
0260 std::cout << "acquire duration = " << duration << std::endl;
0261 #endif
0262 }
0263
0264 void HBHERecHitProducerGPU::produce(edm::Event& event, edm::EventSetup const& setup) {
0265 cms::cuda::ScopedContextProduce ctx{cudaState_};
0266 ctx.emplace(event, rechitsM0Token_, std::move(outputGPU_.recHits));
0267 }
0268
0269 DEFINE_FWK_MODULE(HBHERecHitProducerGPU);
