SiPixelRecHits/plugins/SiPixelRecHitFromCUDA.cc

0001 #include <cuda_runtime.h>
0002
0003 #include <fmt/printf.h>
0004
0005 #include "CUDADataFormats/Common/interface/HostProduct.h"
0006 #include "CUDADataFormats/Common/interface/Product.h"
0007 #include "CUDADataFormats/TrackingRecHit/interface/TrackingRecHitSoADevice.h"
0008 #include "DataFormats/Common/interface/DetSetVectorNew.h"
0009 #include "DataFormats/Common/interface/Handle.h"
0010 #include "DataFormats/SiPixelCluster/interface/SiPixelCluster.h"
0011 #include "DataFormats/TrackerRecHit2D/interface/SiPixelRecHitCollection.h"
0012 #include "FWCore/Framework/interface/Event.h"
0013 #include "FWCore/Framework/interface/EventSetup.h"
0014 #include "FWCore/Framework/interface/MakerMacros.h"
0015 #include "FWCore/Framework/interface/stream/EDProducer.h"
0016 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0017 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0018 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0019 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0020 #include "FWCore/Utilities/interface/InputTag.h"
0021 #include "Geometry/CommonDetUnit/interface/PixelGeomDetUnit.h"
0022 #include "Geometry/Records/interface/TrackerDigiGeometryRecord.h"
0023 #include "Geometry/TrackerGeometryBuilder/interface/TrackerGeometry.h"
0024 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0025 #include "RecoLocalTracker/SiPixelRecHits/interface/pixelCPEforGPU.h"
0026
0027 template <typename TrackerTraits>
0028 class SiPixelRecHitFromCUDAT : public edm::stream::EDProducer<edm::ExternalWork> {
0029 public:
0030   explicit SiPixelRecHitFromCUDAT(const edm::ParameterSet& iConfig);
0031   ~SiPixelRecHitFromCUDAT() override = default;
0032
0033   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0034
0035   using HMSstorage = HostProduct<uint32_t[]>;
0036   using HitsOnDevice = TrackingRecHitSoADevice<TrackerTraits>;
0037
0038 private:
0039   void acquire(edm::Event const& iEvent,
0040                edm::EventSetup const& iSetup,
0041                edm::WaitingTaskWithArenaHolder waitingTaskHolder) override;
0042   void produce(edm::Event& iEvent, edm::EventSetup const& iSetup) override;
0043
0044   const edm::ESGetToken<TrackerGeometry, TrackerDigiGeometryRecord> geomToken_;
0045   const edm::EDGetTokenT<cms::cuda::Product<HitsOnDevice>> hitsToken_;  // CUDA hits
0046   const edm::EDGetTokenT<SiPixelClusterCollectionNew> clusterToken_;    // legacy clusters
0047   const edm::EDPutTokenT<SiPixelRecHitCollection> rechitsPutToken_;     // legacy rechits
0048   const edm::EDPutTokenT<HMSstorage> hostPutToken_;
0049
0050   uint32_t nHits_;
0051   cms::cuda::host::unique_ptr<float[]> store32_;
0052   cms::cuda::host::unique_ptr<uint32_t[]> hitsModuleStart_;
0053 };
0054
0055 template <typename TrackerTraits>
0056 SiPixelRecHitFromCUDAT<TrackerTraits>::SiPixelRecHitFromCUDAT(const edm::ParameterSet& iConfig)
0057     : geomToken_(esConsumes()),
0058       hitsToken_(consumes<cms::cuda::Product<HitsOnDevice>>(iConfig.getParameter<edm::InputTag>("pixelRecHitSrc"))),
0059       clusterToken_(consumes<SiPixelClusterCollectionNew>(iConfig.getParameter<edm::InputTag>("src"))),
0060       rechitsPutToken_(produces<SiPixelRecHitCollection>()),
0061       hostPutToken_(produces<HMSstorage>()) {}
0062
0063 template <typename TrackerTraits>
0064 void SiPixelRecHitFromCUDAT<TrackerTraits>::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0065   edm::ParameterSetDescription desc;
0066   desc.add<edm::InputTag>("pixelRecHitSrc", edm::InputTag("siPixelRecHitsPreSplittingCUDA"));
0067   desc.add<edm::InputTag>("src", edm::InputTag("siPixelClustersPreSplitting"));
0068
0069   descriptions.addWithDefaultLabel(desc);
0070 }
0071
0072 template <typename TrackerTraits>
0073 void SiPixelRecHitFromCUDAT<TrackerTraits>::acquire(edm::Event const& iEvent,
0074                                                     edm::EventSetup const& iSetup,
0075                                                     edm::WaitingTaskWithArenaHolder waitingTaskHolder) {
0076   cms::cuda::Product<HitsOnDevice> const& inputDataWrapped = iEvent.get(hitsToken_);
0077
0078   cms::cuda::ScopedContextAcquire ctx{inputDataWrapped, std::move(waitingTaskHolder)};
0079
0080   auto const& inputData = ctx.get(inputDataWrapped);
0081
0082   nHits_ = inputData.nHits();
0083   LogDebug("SiPixelRecHitFromCUDA") << "converting " << nHits_ << " Hits";
0084
0085   if (0 == nHits_)
0086     return;
0087   store32_ = inputData.localCoordToHostAsync(ctx.stream());
0088
0089   hitsModuleStart_ = inputData.hitsModuleStartToHostAsync(ctx.stream());
0090 }
0091
0092 template <typename TrackerTraits>
0093 void SiPixelRecHitFromCUDAT<TrackerTraits>::produce(edm::Event& iEvent, edm::EventSetup const& es) {
0094   // allocate a buffer for the indices of the clusters
0095   constexpr auto nMaxModules = TrackerTraits::numberOfModules;
0096   auto hmsp = std::make_unique<uint32_t[]>(nMaxModules + 1);
0097
0098   SiPixelRecHitCollection output;
0099   output.reserve(nMaxModules, nHits_);
0100
0101   if (0 == nHits_) {
0102     iEvent.emplace(rechitsPutToken_, std::move(output));
0103     iEvent.emplace(hostPutToken_, std::move(hmsp));
0104     return;
0105   }
0106   output.reserve(nMaxModules, nHits_);
0107
0108   std::copy(hitsModuleStart_.get(), hitsModuleStart_.get() + nMaxModules + 1, hmsp.get());
0109   // wrap the buffer in a HostProduct, and move it to the Event, without reallocating the buffer or affecting hitsModuleStart
0110   iEvent.emplace(hostPutToken_, std::move(hmsp));
0111
0112   auto xl = store32_.get();
0113   auto yl = xl + nHits_;
0114   auto xe = yl + nHits_;
0115   auto ye = xe + nHits_;
0116
0117   const TrackerGeometry* geom = &es.getData(geomToken_);
0118
0119   edm::Handle<SiPixelClusterCollectionNew> hclusters = iEvent.getHandle(clusterToken_);
0120   auto const& input = *hclusters;
0121
0122   constexpr uint32_t maxHitsInModule = TrackerTraits::maxHitsInModule;
0123
0124   int numberOfDetUnits = 0;
0125   int numberOfClusters = 0;
0126   for (auto const& dsv : input) {
0127     numberOfDetUnits++;
0128     unsigned int detid = dsv.detId();
0129     DetId detIdObject(detid);
0130     const GeomDetUnit* genericDet = geom->idToDetUnit(detIdObject);
0131     auto gind = genericDet->index();
0132     const PixelGeomDetUnit* pixDet = dynamic_cast<const PixelGeomDetUnit*>(genericDet);
0133     assert(pixDet);
0134     SiPixelRecHitCollection::FastFiller recHitsOnDetUnit(output, detid);
0135     auto fc = hitsModuleStart_[gind];
0136     auto lc = hitsModuleStart_[gind + 1];
0137     auto nhits = lc - fc;
0138
0139     assert(lc > fc);
0140     LogDebug("SiPixelRecHitFromCUDA") << "in det " << gind << ": conv " << nhits << " hits from " << dsv.size()
0141                                       << " legacy clusters" << ' ' << fc << ',' << lc << "\n";
0142     if (nhits > maxHitsInModule)
0143       edm::LogWarning("SiPixelRecHitFromCUDA") << fmt::sprintf(
0144           "Too many clusters %d in module %d. Only the first %d hits will be converted", nhits, gind, maxHitsInModule);
0145     nhits = std::min(nhits, maxHitsInModule);
0146
0147     LogDebug("SiPixelRecHitFromCUDA") << "in det " << gind << "conv " << nhits << " hits from " << dsv.size()
0148                                       << " legacy clusters" << ' ' << lc << ',' << fc;
0149
0150     if (0 == nhits)
0151       continue;
0152     auto jnd = [&](int k) { return fc + k; };
0153     assert(nhits <= dsv.size());
0154     if (nhits != dsv.size()) {
0155       edm::LogWarning("GPUHits2CPU") << "nhits!= nclus " << nhits << ' ' << dsv.size();
0156     }
0157     for (auto const& clust : dsv) {
0158       assert(clust.originalId() >= 0);
0159       assert(clust.originalId() < dsv.size());
0160       if (clust.originalId() >= nhits)
0161         continue;
0162       auto ij = jnd(clust.originalId());
0163       LocalPoint lp(xl[ij], yl[ij]);
0164       LocalError le(xe[ij], 0, ye[ij]);
0165       SiPixelRecHitQuality::QualWordType rqw = 0;
0166
0167       numberOfClusters++;
0168
0169       /* cpu version....  (for reference)
0170       std::tuple<LocalPoint, LocalError, SiPixelRecHitQuality::QualWordType> tuple = cpe_->getParameters( clust, *genericDet );
0171       LocalPoint lp( std::get<0>(tuple) );
0172       LocalError le( std::get<1>(tuple) );
0173       SiPixelRecHitQuality::QualWordType rqw( std::get<2>(tuple) );
0174       */
0175
0176       // Create a persistent edm::Ref to the cluster
0177       edm::Ref<edmNew::DetSetVector<SiPixelCluster>, SiPixelCluster> cluster = edmNew::makeRefTo(hclusters, &clust);
0178       // Make a RecHit and add it to the DetSet
0179       recHitsOnDetUnit.emplace_back(lp, le, rqw, *genericDet, cluster);
0180       // =============================
0181
0182       LogDebug("SiPixelRecHitFromCUDA") << "cluster " << numberOfClusters << " at " << lp << ' ' << le;
0183
0184     }  //  <-- End loop on Clusters
0185
0186     //  LogDebug("SiPixelRecHitGPU")
0187     LogDebug("SiPixelRecHitFromCUDA") << "found " << recHitsOnDetUnit.size() << " RecHits on " << detid;
0188
0189   }  //    <-- End loop on DetUnits
0190
0191   LogDebug("SiPixelRecHitFromCUDA") << "found " << numberOfDetUnits << " dets, " << numberOfClusters << " clusters";
0192
0193   iEvent.emplace(rechitsPutToken_, std::move(output));
0194 }
0195
0196 using SiPixelRecHitFromCUDAPhase1 = SiPixelRecHitFromCUDAT<pixelTopology::Phase1>;
0197 DEFINE_FWK_MODULE(SiPixelRecHitFromCUDAPhase1);
0198
0199 using SiPixelRecHitFromCUDAPhase2 = SiPixelRecHitFromCUDAT<pixelTopology::Phase2>;
0200 DEFINE_FWK_MODULE(SiPixelRecHitFromCUDAPhase2);
0201
0202 using SiPixelRecHitFromCUDAHIonPhase1 = SiPixelRecHitFromCUDAT<pixelTopology::HIonPhase1>;
0203 DEFINE_FWK_MODULE(SiPixelRecHitFromCUDAHIonPhase1);