HcalRawToDigi/plugins/HcalRawToDigiGPU.cc

0001 #include <iostream>
0002
0003 #include "CondFormats/DataRecord/interface/HcalElectronicsMapRcd.h"
0004 #include "DataFormats/FEDRawData/interface/FEDNumbering.h"
0005 #include "DataFormats/FEDRawData/interface/FEDRawDataCollection.h"
0006 #include "FWCore/Framework/interface/Event.h"
0007 #include "FWCore/Framework/interface/EventSetup.h"
0008 #include "FWCore/Framework/interface/MakerMacros.h"
0009 #include "FWCore/Framework/interface/stream/EDProducer.h"
0010 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0011 #include "FWCore/ServiceRegistry/interface/Service.h"
0012 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0013 #include "HeterogeneousCore/CUDAServices/interface/CUDAService.h"
0014 #include "HeterogeneousCore/CUDAUtilities/interface/cudaCheck.h"
0015 #include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
0016
0017 #include "DeclsForKernels.h"
0018 #include "DecodeGPU.h"
0019 #include "ElectronicsMappingGPU.h"
0020
0021 class HcalRawToDigiGPU : public edm::stream::EDProducer<edm::ExternalWork> {
0022 public:
0023   explicit HcalRawToDigiGPU(edm::ParameterSet const& ps);
0024   ~HcalRawToDigiGPU() override;
0025   static void fillDescriptions(edm::ConfigurationDescriptions&);
0026
0027 private:
0028   void acquire(edm::Event const&, edm::EventSetup const&, edm::WaitingTaskWithArenaHolder) override;
0029   void produce(edm::Event&, edm::EventSetup const&) override;
0030
0031 private:
0032   edm::ESGetToken<hcal::raw::ElectronicsMappingGPU, HcalElectronicsMapRcd> eMappingToken_;
0033   edm::EDGetTokenT<FEDRawDataCollection> rawDataToken_;
0034   using ProductTypef01 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor1, calo::common::DevStoragePolicy>>;
0035   edm::EDPutTokenT<ProductTypef01> digisF01HEToken_;
0036   using ProductTypef5 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor5, calo::common::DevStoragePolicy>>;
0037   edm::EDPutTokenT<ProductTypef5> digisF5HBToken_;
0038   using ProductTypef3 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor3, calo::common::DevStoragePolicy>>;
0039   edm::EDPutTokenT<ProductTypef3> digisF3HBToken_;
0040
0041   cms::cuda::ContextState cudaState_;
0042
0043   std::vector<int> fedsToUnpack_;
0044
0045   hcal::raw::ConfigurationParameters config_;
0046   hcal::raw::OutputDataGPU outputGPU_;
0047   hcal::raw::OutputDataCPU outputCPU_;
0048 };
0049
0050 void HcalRawToDigiGPU::fillDescriptions(edm::ConfigurationDescriptions& confDesc) {
0051   edm::ParameterSetDescription desc;
0052
0053   desc.add<edm::InputTag>("InputLabel", edm::InputTag("rawDataCollector"));
0054   auto nFeds = FEDNumbering::MAXHCALuTCAFEDID - FEDNumbering::MINHCALuTCAFEDID + 1;
0055   std::vector<int> feds(nFeds);
0056   for (int i = 0; i < nFeds; ++i)
0057     feds[i] = i + FEDNumbering::MINHCALuTCAFEDID;
0058   desc.add<std::vector<int>>("FEDs", feds);
0059   desc.add<uint32_t>("maxChannelsF01HE", 10000u);
0060   desc.add<uint32_t>("maxChannelsF5HB", 10000u);
0061   desc.add<uint32_t>("maxChannelsF3HB", 10000u);
0062   desc.add<uint32_t>("nsamplesF01HE", 8);
0063   desc.add<uint32_t>("nsamplesF5HB", 8);
0064   desc.add<uint32_t>("nsamplesF3HB", 8);
0065   desc.add<std::string>("digisLabelF5HB", "f5HBDigisGPU");
0066   desc.add<std::string>("digisLabelF01HE", "f01HEDigisGPU");
0067   desc.add<std::string>("digisLabelF3HB", "f3HBDigisGPU");
0068
0069   std::string label = "hcalRawToDigiGPU";
0070   confDesc.add(label, desc);
0071 }
0072
0073 HcalRawToDigiGPU::HcalRawToDigiGPU(const edm::ParameterSet& ps)
0074     : eMappingToken_{esConsumes()},
0075       rawDataToken_{consumes<FEDRawDataCollection>(ps.getParameter<edm::InputTag>("InputLabel"))},
0076       digisF01HEToken_{produces<ProductTypef01>(ps.getParameter<std::string>("digisLabelF01HE"))},
0077       digisF5HBToken_{produces<ProductTypef5>(ps.getParameter<std::string>("digisLabelF5HB"))},
0078       digisF3HBToken_{produces<ProductTypef3>(ps.getParameter<std::string>("digisLabelF3HB"))},
0079       fedsToUnpack_{ps.getParameter<std::vector<int>>("FEDs")} {
0080   config_.maxChannelsF01HE = ps.getParameter<uint32_t>("maxChannelsF01HE");
0081   config_.maxChannelsF5HB = ps.getParameter<uint32_t>("maxChannelsF5HB");
0082   config_.maxChannelsF3HB = ps.getParameter<uint32_t>("maxChannelsF3HB");
0083   config_.nsamplesF01HE = ps.getParameter<uint32_t>("nsamplesF01HE");
0084   config_.nsamplesF5HB = ps.getParameter<uint32_t>("nsamplesF5HB");
0085   config_.nsamplesF3HB = ps.getParameter<uint32_t>("nsamplesF3HB");
0086 }
0087
0088 HcalRawToDigiGPU::~HcalRawToDigiGPU() {}
0089
0090 void HcalRawToDigiGPU::acquire(edm::Event const& event,
0091                                edm::EventSetup const& setup,
0092                                edm::WaitingTaskWithArenaHolder holder) {
0093   // raii
0094   cms::cuda::ScopedContextAcquire ctx{event.streamID(), std::move(holder), cudaState_};
0095
0096   // conditions
0097   auto const& eMappingProduct = setup.getData(eMappingToken_).getProduct(ctx.stream());
0098
0099   // bundle up conditions
0100   hcal::raw::ConditionsProducts conditions{eMappingProduct};
0101
0102   // event data
0103   edm::Handle<FEDRawDataCollection> rawDataHandle;
0104   event.getByToken(rawDataToken_, rawDataHandle);
0105
0106   // scratch
0107   hcal::raw::ScratchDataGPU scratchGPU = {
0108       cms::cuda::make_device_unique<uint32_t[]>(hcal::raw::numOutputCollections, ctx.stream())};
0109
0110   // input cpu data
0111   hcal::raw::InputDataCPU inputCPU = {cms::cuda::make_host_unique<unsigned char[]>(
0112                                           hcal::raw::utca_nfeds_max * hcal::raw::nbytes_per_fed_max, ctx.stream()),
0113                                       cms::cuda::make_host_unique<uint32_t[]>(hcal::raw::utca_nfeds_max, ctx.stream()),
0114                                       cms::cuda::make_host_unique<int[]>(hcal::raw::utca_nfeds_max, ctx.stream())};
0115
0116   // input data gpu
0117   hcal::raw::InputDataGPU inputGPU = {
0118       cms::cuda::make_device_unique<unsigned char[]>(hcal::raw::utca_nfeds_max * hcal::raw::nbytes_per_fed_max,
0119                                                      ctx.stream()),
0120       cms::cuda::make_device_unique<uint32_t[]>(hcal::raw::utca_nfeds_max, ctx.stream()),
0121       cms::cuda::make_device_unique<int[]>(hcal::raw::utca_nfeds_max, ctx.stream())};
0122
0123   // output cpu
0124   outputCPU_ = {cms::cuda::make_host_unique<uint32_t[]>(hcal::raw::numOutputCollections, ctx.stream())};
0125
0126   // output gpu
0127   outputGPU_.allocate(config_, ctx.stream());
0128
0129   // iterate over feds
0130   // TODO: another idea
0131   //   - loop over all feds to unpack and enqueue cuda memcpy
0132   //   - accumulate the sizes
0133   //   - after the loop launch cuda memcpy for sizes
0134   //   - enqueue the kernel
0135   uint32_t currentCummOffset = 0;
0136   uint32_t counter = 0;
0137   for (auto const& fed : fedsToUnpack_) {
0138     auto const& data = rawDataHandle->FEDData(fed);
0139     auto const nbytes = data.size();
0140
0141     // skip empty feds
0142     if (nbytes < hcal::raw::empty_event_size)
0143       continue;
0144
0145 #ifdef HCAL_RAWDECODE_CPUDEBUG
0146     printf("fed = %d nbytes = %lu\n", fed, nbytes);
0147 #endif
0148
0149     // copy raw data into plain buffer
0150     std::memcpy(inputCPU.data.get() + currentCummOffset, data.data(), nbytes);
0151     // set the offset in bytes from the start
0152     inputCPU.offsets[counter] = currentCummOffset;
0153     inputCPU.feds[counter] = fed;
0154
0155     // this is the current offset into the vector
0156     currentCummOffset += nbytes;
0157     ++counter;
0158   }
0159
0160   hcal::raw::entryPoint(inputCPU,
0161                         inputGPU,
0162                         outputGPU_,
0163                         scratchGPU,
0164                         outputCPU_,
0165                         conditions,
0166                         config_,
0167                         ctx.stream(),
0168                         counter,
0169                         currentCummOffset);
0170 }
0171
0172 void HcalRawToDigiGPU::produce(edm::Event& event, edm::EventSetup const& setup) {
0173   cms::cuda::ScopedContextProduce ctx{cudaState_};
0174
0175 #ifdef HCAL_RAWDECODE_CPUDEBUG
0176   printf("f01he channels = %u f5hb channesl = %u\n",
0177          outputCPU_.nchannels[hcal::raw::OutputF01HE],
0178          outputCPU_.nchannels[hcal::raw::OutputF5HB]);
0179 #endif
0180
0181   // FIXME: use sizes of views directly for cuda mem cpy?
0182   auto const nchannelsF01HE = outputCPU_.nchannels[hcal::raw::OutputF01HE];
0183   auto const nchannelsF5HB = outputCPU_.nchannels[hcal::raw::OutputF5HB];
0184   auto const nchannelsF3HB = outputCPU_.nchannels[hcal::raw::OutputF3HB];
0185   outputGPU_.digisF01HE.size = nchannelsF01HE;
0186   outputGPU_.digisF5HB.size = nchannelsF5HB;
0187   outputGPU_.digisF3HB.size = nchannelsF3HB;
0188   outputGPU_.digisF01HE.stride = hcal::compute_stride<hcal::Flavor1>(config_.nsamplesF01HE);
0189   outputGPU_.digisF5HB.stride = hcal::compute_stride<hcal::Flavor5>(config_.nsamplesF5HB);
0190   outputGPU_.digisF3HB.stride = hcal::compute_stride<hcal::Flavor3>(config_.nsamplesF3HB);
0191
0192   ctx.emplace(event, digisF01HEToken_, std::move(outputGPU_.digisF01HE));
0193   ctx.emplace(event, digisF5HBToken_, std::move(outputGPU_.digisF5HB));
0194   ctx.emplace(event, digisF3HBToken_, std::move(outputGPU_.digisF3HB));
0195
0196   // reset ptrs that are carried as members
0197   outputCPU_.nchannels.reset();
0198 }
0199
0200 DEFINE_FWK_MODULE(HcalRawToDigiGPU);