PixelVertexFinding/plugins/PixelVertexProducerCUDA.cc

0001 #include <cuda_runtime.h>
0002
0003 #include "CUDADataFormats/Common/interface/Product.h"
0004 #include "Geometry/CommonTopologies/interface/SimplePixelTopology.h"
0005 #include "DataFormats/Common/interface/Handle.h"
0006 #include "FWCore/Framework/interface/ESHandle.h"
0007 #include "FWCore/Framework/interface/Event.h"
0008 #include "FWCore/Framework/interface/EventSetup.h"
0009 #include "FWCore/Framework/interface/MakerMacros.h"
0010 #include "FWCore/Framework/interface/global/EDProducer.h"
0011 #include "FWCore/Framework/interface/ConsumesCollector.h"
0012 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0013 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0014 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0015 #include "FWCore/Utilities/interface/InputTag.h"
0016 #include "FWCore/PluginManager/interface/ModuleDef.h"
0017 #include "FWCore/Utilities/interface/EDGetToken.h"
0018 #include "FWCore/Utilities/interface/RunningAverage.h"
0019 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0020
0021 #include "CUDADataFormats/Track/interface/TrackSoAHeterogeneousDevice.h"
0022 #include "CUDADataFormats/Track/interface/TrackSoAHeterogeneousHost.h"
0023 #include "CUDADataFormats/Vertex/interface/ZVertexSoAHeterogeneousDevice.h"
0024 #include "CUDADataFormats/Vertex/interface/ZVertexSoAHeterogeneousHost.h"
0025
0026 #include "gpuVertexFinder.h"
0027
0028 #undef PIXVERTEX_DEBUG_PRODUCE
0029
0030 template <typename TrackerTraits>
0031 class PixelVertexProducerCUDAT : public edm::global::EDProducer<> {
0032   using TracksSoADevice = TrackSoAHeterogeneousDevice<TrackerTraits>;
0033   using TracksSoAHost = TrackSoAHeterogeneousHost<TrackerTraits>;
0034   using GPUAlgo = gpuVertexFinder::Producer<TrackerTraits>;
0035
0036 public:
0037   explicit PixelVertexProducerCUDAT(const edm::ParameterSet& iConfig);
0038   ~PixelVertexProducerCUDAT() override = default;
0039
0040   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0041
0042 private:
0043   void produceOnGPU(edm::StreamID streamID, edm::Event& iEvent, const edm::EventSetup& iSetup) const;
0044   void produceOnCPU(edm::StreamID streamID, edm::Event& iEvent, const edm::EventSetup& iSetup) const;
0045   void produce(edm::StreamID streamID, edm::Event& iEvent, const edm::EventSetup& iSetup) const override;
0046
0047   bool onGPU_;
0048
0049   edm::EDGetTokenT<cms::cuda::Product<TracksSoADevice>> tokenGPUTrack_;
0050   edm::EDPutTokenT<cms::cuda::Product<ZVertexSoADevice>> tokenGPUVertex_;
0051   edm::EDGetTokenT<TracksSoAHost> tokenCPUTrack_;
0052   edm::EDPutTokenT<ZVertexSoAHost> tokenCPUVertex_;
0053
0054   const GPUAlgo gpuAlgo_;
0055
0056   // Tracking cuts before sending tracks to vertex algo
0057   const float ptMin_;
0058   const float ptMax_;
0059 };
0060
0061 template <typename TrackerTraits>
0062 PixelVertexProducerCUDAT<TrackerTraits>::PixelVertexProducerCUDAT(const edm::ParameterSet& conf)
0063     : onGPU_(conf.getParameter<bool>("onGPU")),
0064       gpuAlgo_(conf.getParameter<bool>("oneKernel"),
0065                conf.getParameter<bool>("useDensity"),
0066                conf.getParameter<bool>("useDBSCAN"),
0067                conf.getParameter<bool>("useIterative"),
0068                conf.getParameter<bool>("doSplitting"),
0069                conf.getParameter<int>("minT"),
0070                conf.getParameter<double>("eps"),
0071                conf.getParameter<double>("errmax"),
0072                conf.getParameter<double>("chi2max")),
0073       ptMin_(conf.getParameter<double>("PtMin")),  // 0.5 GeV
0074       ptMax_(conf.getParameter<double>("PtMax"))   // 75. GeV
0075 {
0076   if (onGPU_) {
0077     tokenGPUTrack_ = consumes(conf.getParameter<edm::InputTag>("pixelTrackSrc"));
0078     tokenGPUVertex_ = produces();
0079   } else {
0080     tokenCPUTrack_ = consumes(conf.getParameter<edm::InputTag>("pixelTrackSrc"));
0081     tokenCPUVertex_ = produces();
0082   }
0083 }
0084
0085 template <typename TrackerTraits>
0086 void PixelVertexProducerCUDAT<TrackerTraits>::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0087   edm::ParameterSetDescription desc;
0088
0089   // Only one of these three algos can be used at once.
0090   // Maybe this should become a Plugin Factory
0091   desc.add<bool>("onGPU", true);
0092   desc.add<bool>("oneKernel", true);
0093   desc.add<bool>("useDensity", true);
0094   desc.add<bool>("useDBSCAN", false);
0095   desc.add<bool>("useIterative", false);
0096   desc.add<bool>("doSplitting", true);
0097
0098   desc.add<int>("minT", 2);          // min number of neighbours to be "core"
0099   desc.add<double>("eps", 0.07);     // max absolute distance to cluster
0100   desc.add<double>("errmax", 0.01);  // max error to be "seed"
0101   desc.add<double>("chi2max", 9.);   // max normalized distance to cluster
0102
0103   desc.add<double>("PtMin", 0.5);
0104   desc.add<double>("PtMax", 75.);
0105   desc.add<edm::InputTag>("pixelTrackSrc", edm::InputTag("pixelTracksCUDA"));
0106
0107   descriptions.addWithDefaultLabel(desc);
0108 }
0109
0110 template <typename TrackerTraits>
0111 void PixelVertexProducerCUDAT<TrackerTraits>::produceOnGPU(edm::StreamID streamID,
0112                                                            edm::Event& iEvent,
0113                                                            const edm::EventSetup& iSetup) const {
0114   using TracksSoA = TrackSoAHeterogeneousDevice<TrackerTraits>;
0115   auto hTracks = iEvent.getHandle(tokenGPUTrack_);
0116
0117   cms::cuda::ScopedContextProduce ctx{*hTracks};
0118   auto& tracks = ctx.get(*hTracks);
0119
0120   ctx.emplace(iEvent, tokenGPUVertex_, gpuAlgo_.makeAsync(ctx.stream(), tracks.view(), ptMin_, ptMax_));
0121 }
0122
0123 template <typename TrackerTraits>
0124 void PixelVertexProducerCUDAT<TrackerTraits>::produceOnCPU(edm::StreamID streamID,
0125                                                            edm::Event& iEvent,
0126                                                            const edm::EventSetup& iSetup) const {
0127   auto& tracks = iEvent.get(tokenCPUTrack_);
0128
0129 #ifdef PIXVERTEX_DEBUG_PRODUCE
0130   auto const& tsoa = *tracks;
0131   auto maxTracks = tsoa.stride();
0132   std::cout << "size of SoA " << sizeof(tsoa) << " stride " << maxTracks << std::endl;
0133
0134   int32_t nt = 0;
0135   for (int32_t it = 0; it < maxTracks; ++it) {
0136     auto nHits = TracksUtilities<TrackerTraits>::nHits(tracks.view(), it);
0137     assert(nHits == int(tracks.view().hitIndices().size(it)));
0138     if (nHits == 0)
0139       break;  // this is a guard: maybe we need to move to nTracks...
0140     nt++;
0141   }
0142   std::cout << "found " << nt << " tracks in cpu SoA for Vertexing at " << tracks << std::endl;
0143 #endif  // PIXVERTEX_DEBUG_PRODUCE
0144
0145   iEvent.emplace(tokenCPUVertex_, gpuAlgo_.make(tracks.view(), ptMin_, ptMax_));
0146 }
0147
0148 template <typename TrackerTraits>
0149 void PixelVertexProducerCUDAT<TrackerTraits>::produce(edm::StreamID streamID,
0150                                                       edm::Event& iEvent,
0151                                                       const edm::EventSetup& iSetup) const {
0152   if (onGPU_) {
0153     produceOnGPU(streamID, iEvent, iSetup);
0154   } else {
0155     produceOnCPU(streamID, iEvent, iSetup);
0156   }
0157 }
0158
0159 using PixelVertexProducerCUDAPhase1 = PixelVertexProducerCUDAT<pixelTopology::Phase1>;
0160 DEFINE_FWK_MODULE(PixelVertexProducerCUDAPhase1);
0161
0162 using PixelVertexProducerCUDAPhase2 = PixelVertexProducerCUDAT<pixelTopology::Phase2>;
0163 DEFINE_FWK_MODULE(PixelVertexProducerCUDAPhase2);
0164
0165 using PixelVertexProducerCUDAHIonPhase1 = PixelVertexProducerCUDAT<pixelTopology::HIonPhase1>;
0166 DEFINE_FWK_MODULE(PixelVertexProducerCUDAHIonPhase1);