HcalRawToDigi/plugins/HcalRawToDigiGPU.cc

0001 #include "CUDADataFormats/Common/interface/Product.h"
0002 #include "CondFormats/DataRecord/interface/HcalElectronicsMapRcd.h"
0003 #include "DataFormats/FEDRawData/interface/FEDNumbering.h"
0004 #include "DataFormats/FEDRawData/interface/FEDRawDataCollection.h"
0005 #include "FWCore/Framework/interface/Event.h"
0006 #include "FWCore/Framework/interface/EventSetup.h"
0007 #include "FWCore/Framework/interface/MakerMacros.h"
0008 #include "FWCore/Framework/interface/stream/EDProducer.h"
0009 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0010 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0011
0012 #include "DeclsForKernels.h"
0013 #include "DecodeGPU.h"
0014 #include "ElectronicsMappingGPU.h"
0015
0016 class HcalRawToDigiGPU : public edm::stream::EDProducer<edm::ExternalWork> {
0017 public:
0018   explicit HcalRawToDigiGPU(edm::ParameterSet const& ps);
0019   ~HcalRawToDigiGPU() override;
0020   static void fillDescriptions(edm::ConfigurationDescriptions&);
0021
0022 private:
0023   void acquire(edm::Event const&, edm::EventSetup const&, edm::WaitingTaskWithArenaHolder) override;
0024   void produce(edm::Event&, edm::EventSetup const&) override;
0025
0026 private:
0027   edm::ESGetToken<hcal::raw::ElectronicsMappingGPU, HcalElectronicsMapRcd> eMappingToken_;
0028   edm::EDGetTokenT<FEDRawDataCollection> rawDataToken_;
0029   using ProductTypef01 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor1, calo::common::DevStoragePolicy>>;
0030   edm::EDPutTokenT<ProductTypef01> digisF01HEToken_;
0031   using ProductTypef5 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor5, calo::common::DevStoragePolicy>>;
0032   edm::EDPutTokenT<ProductTypef5> digisF5HBToken_;
0033   using ProductTypef3 = cms::cuda::Product<hcal::DigiCollection<hcal::Flavor3, calo::common::DevStoragePolicy>>;
0034   edm::EDPutTokenT<ProductTypef3> digisF3HBToken_;
0035
0036   cms::cuda::ContextState cudaState_;
0037
0038   std::vector<int> fedsToUnpack_;
0039
0040   hcal::raw::ConfigurationParameters config_;
0041   hcal::raw::OutputDataGPU outputGPU_;
0042   hcal::raw::OutputDataCPU outputCPU_;
0043 };
0044
0045 void HcalRawToDigiGPU::fillDescriptions(edm::ConfigurationDescriptions& confDesc) {
0046   edm::ParameterSetDescription desc;
0047
0048   desc.add<edm::InputTag>("InputLabel", edm::InputTag("rawDataCollector"));
0049   auto nFeds = FEDNumbering::MAXHCALuTCAFEDID - FEDNumbering::MINHCALuTCAFEDID + 1;
0050   std::vector<int> feds(nFeds);
0051   for (int i = 0; i < nFeds; ++i)
0052     feds[i] = i + FEDNumbering::MINHCALuTCAFEDID;
0053   desc.add<std::vector<int>>("FEDs", feds);
0054   desc.add<uint32_t>("maxChannelsF01HE", 10000u);
0055   desc.add<uint32_t>("maxChannelsF5HB", 10000u);
0056   desc.add<uint32_t>("maxChannelsF3HB", 10000u);
0057   desc.add<uint32_t>("nsamplesF01HE", 8);
0058   desc.add<uint32_t>("nsamplesF5HB", 8);
0059   desc.add<uint32_t>("nsamplesF3HB", 8);
0060   desc.add<std::string>("digisLabelF5HB", "f5HBDigisGPU");
0061   desc.add<std::string>("digisLabelF01HE", "f01HEDigisGPU");
0062   desc.add<std::string>("digisLabelF3HB", "f3HBDigisGPU");
0063
0064   std::string label = "hcalRawToDigiGPU";
0065   confDesc.add(label, desc);
0066 }
0067
0068 HcalRawToDigiGPU::HcalRawToDigiGPU(const edm::ParameterSet& ps)
0069     : eMappingToken_{esConsumes()},
0070       rawDataToken_{consumes<FEDRawDataCollection>(ps.getParameter<edm::InputTag>("InputLabel"))},
0071       digisF01HEToken_{produces<ProductTypef01>(ps.getParameter<std::string>("digisLabelF01HE"))},
0072       digisF5HBToken_{produces<ProductTypef5>(ps.getParameter<std::string>("digisLabelF5HB"))},
0073       digisF3HBToken_{produces<ProductTypef3>(ps.getParameter<std::string>("digisLabelF3HB"))},
0074       fedsToUnpack_{ps.getParameter<std::vector<int>>("FEDs")} {
0075   config_.maxChannelsF01HE = ps.getParameter<uint32_t>("maxChannelsF01HE");
0076   config_.maxChannelsF5HB = ps.getParameter<uint32_t>("maxChannelsF5HB");
0077   config_.maxChannelsF3HB = ps.getParameter<uint32_t>("maxChannelsF3HB");
0078   config_.nsamplesF01HE = ps.getParameter<uint32_t>("nsamplesF01HE");
0079   config_.nsamplesF5HB = ps.getParameter<uint32_t>("nsamplesF5HB");
0080   config_.nsamplesF3HB = ps.getParameter<uint32_t>("nsamplesF3HB");
0081 }
0082
0083 HcalRawToDigiGPU::~HcalRawToDigiGPU() {}
0084
0085 void HcalRawToDigiGPU::acquire(edm::Event const& event,
0086                                edm::EventSetup const& setup,
0087                                edm::WaitingTaskWithArenaHolder holder) {
0088   // raii
0089   cms::cuda::ScopedContextAcquire ctx{event.streamID(), std::move(holder), cudaState_};
0090
0091   // conditions
0092   auto const& eMappingProduct = setup.getData(eMappingToken_).getProduct(ctx.stream());
0093
0094   // bundle up conditions
0095   hcal::raw::ConditionsProducts conditions{eMappingProduct};
0096
0097   // event data
0098   edm::Handle<FEDRawDataCollection> rawDataHandle;
0099   event.getByToken(rawDataToken_, rawDataHandle);
0100
0101   // scratch
0102   hcal::raw::ScratchDataGPU scratchGPU = {
0103       cms::cuda::make_device_unique<uint32_t[]>(hcal::raw::numOutputCollections, ctx.stream())};
0104
0105   // input cpu data
0106   hcal::raw::InputDataCPU inputCPU = {cms::cuda::make_host_unique<unsigned char[]>(
0107                                           hcal::raw::utca_nfeds_max * hcal::raw::nbytes_per_fed_max, ctx.stream()),
0108                                       cms::cuda::make_host_unique<uint32_t[]>(hcal::raw::utca_nfeds_max, ctx.stream()),
0109                                       cms::cuda::make_host_unique<int[]>(hcal::raw::utca_nfeds_max, ctx.stream())};
0110
0111   // input data gpu
0112   hcal::raw::InputDataGPU inputGPU = {
0113       cms::cuda::make_device_unique<unsigned char[]>(hcal::raw::utca_nfeds_max * hcal::raw::nbytes_per_fed_max,
0114                                                      ctx.stream()),
0115       cms::cuda::make_device_unique<uint32_t[]>(hcal::raw::utca_nfeds_max, ctx.stream()),
0116       cms::cuda::make_device_unique<int[]>(hcal::raw::utca_nfeds_max, ctx.stream())};
0117
0118   // output cpu
0119   outputCPU_ = {cms::cuda::make_host_unique<uint32_t[]>(hcal::raw::numOutputCollections, ctx.stream())};
0120
0121   // output gpu
0122   outputGPU_.allocate(config_, ctx.stream());
0123
0124   // iterate over feds
0125   // TODO: another idea
0126   //   - loop over all feds to unpack and enqueue cuda memcpy
0127   //   - accumulate the sizes
0128   //   - after the loop launch cuda memcpy for sizes
0129   //   - enqueue the kernel
0130   uint32_t currentCummOffset = 0;
0131   uint32_t counter = 0;
0132   for (auto const& fed : fedsToUnpack_) {
0133     auto const& data = rawDataHandle->FEDData(fed);
0134     auto const nbytes = data.size();
0135
0136     // skip empty feds
0137     if (nbytes < hcal::raw::empty_event_size)
0138       continue;
0139
0140 #ifdef HCAL_RAWDECODE_CPUDEBUG
0141     printf("fed = %d nbytes = %lu\n", fed, nbytes);
0142 #endif
0143
0144     // copy raw data into plain buffer
0145     std::memcpy(inputCPU.data.get() + currentCummOffset, data.data(), nbytes);
0146     // set the offset in bytes from the start
0147     inputCPU.offsets[counter] = currentCummOffset;
0148     inputCPU.feds[counter] = fed;
0149
0150     // this is the current offset into the vector
0151     currentCummOffset += nbytes;
0152     ++counter;
0153   }
0154
0155   hcal::raw::entryPoint(inputCPU,
0156                         inputGPU,
0157                         outputGPU_,
0158                         scratchGPU,
0159                         outputCPU_,
0160                         conditions,
0161                         config_,
0162                         ctx.stream(),
0163                         counter,
0164                         currentCummOffset);
0165 }
0166
0167 void HcalRawToDigiGPU::produce(edm::Event& event, edm::EventSetup const& setup) {
0168   cms::cuda::ScopedContextProduce ctx{cudaState_};
0169
0170 #ifdef HCAL_RAWDECODE_CPUDEBUG
0171   printf("f01he channels = %u f5hb channesl = %u\n",
0172          outputCPU_.nchannels[hcal::raw::OutputF01HE],
0173          outputCPU_.nchannels[hcal::raw::OutputF5HB]);
0174 #endif
0175
0176   // FIXME: use sizes of views directly for cuda mem cpy?
0177   auto const nchannelsF01HE = outputCPU_.nchannels[hcal::raw::OutputF01HE];
0178   auto const nchannelsF5HB = outputCPU_.nchannels[hcal::raw::OutputF5HB];
0179   auto const nchannelsF3HB = outputCPU_.nchannels[hcal::raw::OutputF3HB];
0180   outputGPU_.digisF01HE.size = nchannelsF01HE;
0181   outputGPU_.digisF5HB.size = nchannelsF5HB;
0182   outputGPU_.digisF3HB.size = nchannelsF3HB;
0183   outputGPU_.digisF01HE.stride = hcal::compute_stride<hcal::Flavor1>(config_.nsamplesF01HE);
0184   outputGPU_.digisF5HB.stride = hcal::compute_stride<hcal::Flavor5>(config_.nsamplesF5HB);
0185   outputGPU_.digisF3HB.stride = hcal::compute_stride<hcal::Flavor3>(config_.nsamplesF3HB);
0186
0187   ctx.emplace(event, digisF01HEToken_, std::move(outputGPU_.digisF01HE));
0188   ctx.emplace(event, digisF5HBToken_, std::move(outputGPU_.digisF5HB));
0189   ctx.emplace(event, digisF3HBToken_, std::move(outputGPU_.digisF3HB));
0190
0191   // reset ptrs that are carried as members
0192   outputCPU_.nchannels.reset();
0193 }
0194
0195 DEFINE_FWK_MODULE(HcalRawToDigiGPU);