EcalRawToDigi/plugins/EcalRawToDigiGPU.cc

0001 #include "CUDADataFormats/EcalDigi/interface/DigisCollection.h"
0002 #include "CondFormats/DataRecord/interface/EcalMappingElectronicsRcd.h"
0003 #include "CondFormats/EcalObjects/interface/ElectronicsMappingGPU.h"
0004 #include "DataFormats/EcalDigi/interface/EcalDigiCollections.h"
0005 #include "DataFormats/FEDRawData/interface/FEDRawDataCollection.h"
0006 #include "FWCore/Framework/interface/Event.h"
0007 #include "FWCore/Framework/interface/EventSetup.h"
0008 #include "FWCore/Framework/interface/MakerMacros.h"
0009 #include "FWCore/Framework/interface/stream/EDProducer.h"
0010 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0011 #include "FWCore/Utilities/interface/ESGetToken.h"
0012 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0013 #include "HeterogeneousCore/CUDAUtilities/interface/cudaCheck.h"
0014
0015 #include "DeclsForKernels.h"
0016 #include "UnpackGPU.h"
0017
0018 class EcalRawToDigiGPU : public edm::stream::EDProducer<edm::ExternalWork> {
0019 public:
0020   explicit EcalRawToDigiGPU(edm::ParameterSet const& ps);
0021   ~EcalRawToDigiGPU() override = default;
0022   static void fillDescriptions(edm::ConfigurationDescriptions&);
0023
0024 private:
0025   void acquire(edm::Event const&, edm::EventSetup const&, edm::WaitingTaskWithArenaHolder) override;
0026   void produce(edm::Event&, edm::EventSetup const&) override;
0027
0028 private:
0029   edm::EDGetTokenT<FEDRawDataCollection> rawDataToken_;
0030   using OutputProduct = cms::cuda::Product<ecal::DigisCollection<calo::common::DevStoragePolicy>>;
0031   edm::EDPutTokenT<OutputProduct> digisEBToken_, digisEEToken_;
0032   edm::ESGetToken<ecal::raw::ElectronicsMappingGPU, EcalMappingElectronicsRcd> eMappingToken_;
0033
0034   cms::cuda::ContextState cudaState_;
0035
0036   std::vector<int> fedsToUnpack_;
0037
0038   ecal::raw::ConfigurationParameters config_;
0039   ecal::raw::OutputDataGPU outputGPU_;
0040   ecal::raw::OutputDataCPU outputCPU_;
0041 };
0042
0043 void EcalRawToDigiGPU::fillDescriptions(edm::ConfigurationDescriptions& confDesc) {
0044   edm::ParameterSetDescription desc;
0045
0046   desc.add<edm::InputTag>("InputLabel", edm::InputTag("rawDataCollector"));
0047   std::vector<int> feds(54);
0048   for (uint32_t i = 0; i < 54; ++i)
0049     feds[i] = i + 601;
0050   desc.add<std::vector<int>>("FEDs", feds);
0051   desc.add<uint32_t>("maxChannelsEB", 61200);
0052   desc.add<uint32_t>("maxChannelsEE", 14648);
0053   desc.add<std::string>("digisLabelEB", "ebDigis");
0054   desc.add<std::string>("digisLabelEE", "eeDigis");
0055
0056   std::string label = "ecalRawToDigiGPU";
0057   confDesc.add(label, desc);
0058 }
0059
0060 EcalRawToDigiGPU::EcalRawToDigiGPU(const edm::ParameterSet& ps)
0061     : rawDataToken_{consumes<FEDRawDataCollection>(ps.getParameter<edm::InputTag>("InputLabel"))},
0062       digisEBToken_{produces<OutputProduct>(ps.getParameter<std::string>("digisLabelEB"))},
0063       digisEEToken_{produces<OutputProduct>(ps.getParameter<std::string>("digisLabelEE"))},
0064       eMappingToken_{esConsumes<ecal::raw::ElectronicsMappingGPU, EcalMappingElectronicsRcd>()},
0065       fedsToUnpack_{ps.getParameter<std::vector<int>>("FEDs")} {
0066   config_.maxChannelsEB = ps.getParameter<uint32_t>("maxChannelsEB");
0067   config_.maxChannelsEE = ps.getParameter<uint32_t>("maxChannelsEE");
0068 }
0069
0070 void EcalRawToDigiGPU::acquire(edm::Event const& event,
0071                                edm::EventSetup const& setup,
0072                                edm::WaitingTaskWithArenaHolder holder) {
0073   // raii
0074   cms::cuda::ScopedContextAcquire ctx{event.streamID(), std::move(holder), cudaState_};
0075
0076   // conditions
0077   edm::ESHandle<ecal::raw::ElectronicsMappingGPU> eMappingHandle = setup.getHandle(eMappingToken_);
0078   auto const& eMappingProduct = eMappingHandle->getProduct(ctx.stream());
0079
0080   // bundle up conditions
0081   ecal::raw::ConditionsProducts conditions{eMappingProduct};
0082
0083   // event data
0084   edm::Handle<FEDRawDataCollection> rawDataHandle;
0085   event.getByToken(rawDataToken_, rawDataHandle);
0086
0087   // scratch
0088   ecal::raw::ScratchDataGPU scratchGPU = {cms::cuda::make_device_unique<uint32_t[]>(2, ctx.stream())};
0089
0090   // make a first iteration over the FEDs to compute the total buffer size
0091   uint32_t size = 0;
0092   uint32_t feds = 0;
0093   for (auto const& fed : fedsToUnpack_) {
0094     auto const& data = rawDataHandle->FEDData(fed);
0095     auto const nbytes = data.size();
0096
0097     // skip empty FEDs
0098     if (nbytes < globalFieds::EMPTYEVENTSIZE)
0099       continue;
0100
0101     size += nbytes;
0102     ++feds;
0103   }
0104
0105   // input cpu data
0106   ecal::raw::InputDataCPU inputCPU = {cms::cuda::make_host_unique<unsigned char[]>(size, ctx.stream()),
0107                                       cms::cuda::make_host_unique<uint32_t[]>(feds, ctx.stream()),
0108                                       cms::cuda::make_host_unique<int[]>(feds, ctx.stream())};
0109
0110   // input data gpu
0111   ecal::raw::InputDataGPU inputGPU = {cms::cuda::make_device_unique<unsigned char[]>(size, ctx.stream()),
0112                                       cms::cuda::make_device_unique<uint32_t[]>(feds, ctx.stream()),
0113                                       cms::cuda::make_device_unique<int[]>(feds, ctx.stream())};
0114
0115   // output cpu
0116   outputCPU_ = {cms::cuda::make_host_unique<uint32_t[]>(2, ctx.stream())};
0117   // initialize the number of channels
0118   outputCPU_.nchannels[0] = 0;
0119   outputCPU_.nchannels[1] = 0;
0120
0121   // output gpu
0122   outputGPU_.allocate(config_, ctx.stream());
0123
0124   // iterate over FEDs to fill the cpu buffer
0125   uint32_t currentCummOffset = 0;
0126   uint32_t counter = 0;
0127   for (auto const& fed : fedsToUnpack_) {
0128     auto const& data = rawDataHandle->FEDData(fed);
0129     auto const nbytes = data.size();
0130
0131     // skip empty FEDs
0132     if (nbytes < globalFieds::EMPTYEVENTSIZE)
0133       continue;
0134
0135     // copy raw data into plain buffer
0136     std::memcpy(inputCPU.data.get() + currentCummOffset, data.data(), nbytes);
0137     // set the offset in bytes from the start
0138     inputCPU.offsets[counter] = currentCummOffset;
0139     inputCPU.feds[counter] = fed;
0140
0141     // this is the current offset into the vector
0142     currentCummOffset += nbytes;
0143     ++counter;
0144   }
0145   assert(currentCummOffset == size);
0146   assert(counter == feds);
0147
0148   // unpack if at least one FED has data
0149   if (counter > 0) {
0150     ecal::raw::entryPoint(
0151         inputCPU, inputGPU, outputGPU_, scratchGPU, outputCPU_, conditions, ctx.stream(), counter, currentCummOffset);
0152   }
0153 }
0154
0155 void EcalRawToDigiGPU::produce(edm::Event& event, edm::EventSetup const& setup) {
0156   cms::cuda::ScopedContextProduce ctx{cudaState_};
0157
0158   // get the number of channels
0159   outputGPU_.digisEB.size = outputCPU_.nchannels[0];
0160   outputGPU_.digisEE.size = outputCPU_.nchannels[1];
0161
0162   ctx.emplace(event, digisEBToken_, std::move(outputGPU_.digisEB));
0163   ctx.emplace(event, digisEEToken_, std::move(outputGPU_.digisEE));
0164
0165   // reset ptrs that are carried as members
0166   outputCPU_.nchannels.reset();
0167 }
0168
0169 DEFINE_FWK_MODULE(EcalRawToDigiGPU);