SiStripClusterizer/plugins/ClustersFromRawProducerGPU.cc

0001 /*
0002  */
0003 #include "RecoLocalTracker/Records/interface/SiStripClusterizerConditionsRcd.h"
0004
0005 #include "RecoLocalTracker/SiStripClusterizer/interface/StripClusterizerAlgorithmFactory.h"
0006
0007 #include "DataFormats/FEDRawData/interface/FEDRawDataCollection.h"
0008 #include "EventFilter/SiStripRawToDigi/interface/SiStripFEDBuffer.h"
0009
0010 #include "CalibFormats/SiStripObjects/interface/SiStripClusterizerConditions.h"
0011 #include "CalibFormats/SiStripObjects/interface/SiStripClusterizerConditionsGPU.h"
0012
0013 #include "FWCore/Framework/interface/stream/EDProducer.h"
0014 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0015 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0016 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0017 #include "FWCore/Utilities/interface/InputTag.h"
0018 #include "FWCore/Framework/interface/Event.h"
0019 #include "FWCore/Framework/interface/EventSetup.h"
0020 #include "FWCore/Framework/interface/ESHandle.h"
0021 #include "FWCore/Utilities/interface/Likely.h"
0022
0023 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0024
0025 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0026
0027 #include "SiStripRawToClusterGPUKernel.h"
0028 #include "ChannelLocsGPU.h"
0029
0030 //#include <sstream>
0031 #include <memory>
0032 #include <mutex>
0033
0034 namespace {
0035   std::unique_ptr<sistrip::FEDBuffer> fillBuffer(int fedId, const FEDRawData& rawData) {
0036     std::unique_ptr<sistrip::FEDBuffer> buffer;
0037
0038     // Check on FEDRawData pointer
0039     const auto st_buffer = sistrip::preconstructCheckFEDBuffer(rawData);
0040     if UNLIKELY (sistrip::FEDBufferStatusCode::SUCCESS != st_buffer) {
0041       LogDebug(sistrip::mlRawToCluster_) << "[ClustersFromRawProducer::" << __func__ << "]" << st_buffer
0042                                          << " for FED ID " << fedId;
0043       return buffer;
0044     }
0045     buffer = std::make_unique<sistrip::FEDBuffer>(rawData);
0046     const auto st_chan = buffer->findChannels();
0047     if UNLIKELY (sistrip::FEDBufferStatusCode::SUCCESS != st_chan) {
0048       LogDebug(sistrip::mlRawToCluster_) << "Exception caught when creating FEDBuffer object for FED " << fedId << ": "
0049                                          << st_chan;
0050       buffer.reset();
0051       return buffer;
0052     }
0053     if UNLIKELY (!buffer->doChecks(false)) {
0054       LogDebug(sistrip::mlRawToCluster_) << "Exception caught when creating FEDBuffer object for FED " << fedId
0055                                          << ": FED Buffer check fails";
0056       buffer.reset();
0057       return buffer;
0058     }
0059
0060     return buffer;
0061   }
0062 }  // namespace
0063
0064 class SiStripClusterizerFromRawGPU final : public edm::stream::EDProducer<edm::ExternalWork> {
0065 public:
0066   explicit SiStripClusterizerFromRawGPU(const edm::ParameterSet& conf)
0067       : buffers_(sistrip::FED_ID_MAX),
0068         raw_(sistrip::FED_ID_MAX),
0069         gpuAlgo_(conf.getParameter<edm::ParameterSet>("Clusterizer")),
0070         inputToken_(consumes(conf.getParameter<edm::InputTag>("ProductLabel"))),
0071         outputToken_(produces<cms::cuda::Product<SiStripClustersCUDADevice>>()),
0072         conditionsToken_(esConsumes(edm::ESInputTag{"", conf.getParameter<std::string>("ConditionsLabel")})),
0073         cpuConditionsToken_(esConsumes(edm::ESInputTag{"", conf.getParameter<std::string>("ConditionsLabel")})) {}
0074
0075   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0076
0077 private:
0078   void acquire(edm::Event const& ev,
0079                edm::EventSetup const& es,
0080                edm::WaitingTaskWithArenaHolder waitingTaskHolder) override {
0081     const auto& conditions = es.getData(conditionsToken_);        //these need to be GPU conditions
0082     const auto& cpuConditions = es.getData(cpuConditionsToken_);  //CPU conditions
0083
0084     // Sets the current device and creates a CUDA stream
0085     cms::cuda::ScopedContextAcquire ctx{ev.streamID(), std::move(waitingTaskHolder), ctxState_};
0086
0087     // get raw data
0088     auto const& rawData = ev.get(inputToken_);
0089     run(rawData, cpuConditions);
0090
0091     // Queues asynchronous data transfers and kernels to the CUDA stream
0092     // returned by cms::cuda::ScopedContextAcquire::stream()
0093     gpuAlgo_.makeAsync(raw_, buffers_, conditions, ctx.stream());
0094
0095     // Destructor of ctx queues a callback to the CUDA stream notifying
0096     // waitingTaskHolder when the queued asynchronous work has finished
0097   }
0098
0099   void produce(edm::Event& ev, const edm::EventSetup& es) override {
0100     cms::cuda::ScopedContextProduce ctx{ctxState_};
0101
0102     // Now getResult() returns data in GPU memory that is passed to the
0103     // constructor of OutputData. cms::cuda::ScopedContextProduce::emplace() wraps the
0104     // OutputData to cms::cuda::Product<OutputData>. cms::cuda::Product<T> stores also
0105     // the current device and the CUDA stream since those will be needed
0106     // in the consumer side.
0107     ctx.emplace(ev, outputToken_, gpuAlgo_.getResults(ctx.stream()));
0108
0109     for (auto& buf : buffers_)
0110       buf.reset(nullptr);
0111   }
0112
0113 private:
0114   void run(const FEDRawDataCollection& rawColl, const SiStripClusterizerConditions& conditions);
0115   void fill(uint32_t idet, const FEDRawDataCollection& rawColl, const SiStripClusterizerConditions& conditions);
0116
0117 private:
0118   std::vector<std::unique_ptr<sistrip::FEDBuffer>> buffers_;
0119   std::vector<const FEDRawData*> raw_;
0120   cms::cuda::ContextState ctxState_;
0121
0122   stripgpu::SiStripRawToClusterGPUKernel gpuAlgo_;
0123
0124   edm::EDGetTokenT<FEDRawDataCollection> inputToken_;
0125   edm::EDPutTokenT<cms::cuda::Product<SiStripClustersCUDADevice>> outputToken_;
0126   edm::ESGetToken<stripgpu::SiStripClusterizerConditionsGPU, SiStripClusterizerConditionsRcd> conditionsToken_;
0127   edm::ESGetToken<SiStripClusterizerConditions, SiStripClusterizerConditionsRcd> cpuConditionsToken_;
0128 };
0129
0130 void SiStripClusterizerFromRawGPU::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0131   edm::ParameterSetDescription desc;
0132
0133   desc.add("ProductLabel", edm::InputTag("rawDataCollector"));
0134   desc.add<std::string>("ConditionsLabel", "");
0135
0136   edm::ParameterSetDescription clusterizer;
0137   StripClusterizerAlgorithmFactory::fillDescriptions(clusterizer);
0138   desc.add("Clusterizer", clusterizer);
0139
0140   descriptions.addWithDefaultLabel(desc);
0141 }
0142
0143 void SiStripClusterizerFromRawGPU::run(const FEDRawDataCollection& rawColl,
0144                                        const SiStripClusterizerConditions& conditions) {
0145   // loop over good det in cabling
0146   for (auto idet : conditions.allDetIds()) {
0147     fill(idet, rawColl, conditions);
0148   }  // end loop over dets
0149 }
0150
0151 void SiStripClusterizerFromRawGPU::fill(uint32_t idet,
0152                                         const FEDRawDataCollection& rawColl,
0153                                         const SiStripClusterizerConditions& conditions) {
0154   auto const& det = conditions.findDetId(idet);
0155   if (!det.valid())
0156     return;
0157
0158   // Loop over apv-pairs of det
0159   for (auto const conn : conditions.currentConnection(det)) {
0160     if UNLIKELY (!conn)
0161       continue;
0162
0163     const uint16_t fedId = conn->fedId();
0164
0165     // If fed id is null or connection is invalid continue
0166     if UNLIKELY (!fedId || !conn->isConnected()) {
0167       continue;
0168     }
0169
0170     // If Fed hasnt already been initialised, extract data and initialise
0171     sistrip::FEDBuffer* buffer = buffers_[fedId].get();
0172     if (!buffer) {
0173       const FEDRawData& rawData = rawColl.FEDData(fedId);
0174       raw_[fedId] = &rawData;
0175       buffers_[fedId] = fillBuffer(fedId, rawData);
0176     }
0177   }  // end loop over conn
0178 }
0179
0180 #include "FWCore/Framework/interface/MakerMacros.h"
0181 DEFINE_FWK_MODULE(SiStripClusterizerFromRawGPU);