File indexing completed on 2021-12-24 02:22:15
0001
0002 #include <memory>
0003 #include <string>
0004 #include <vector>
0005
0006
0007 #include "CUDADataFormats/Common/interface/Product.h"
0008 #include "CUDADataFormats/SiPixelCluster/interface/SiPixelClustersCUDA.h"
0009 #include "CUDADataFormats/SiPixelDigi/interface/SiPixelDigiErrorsCUDA.h"
0010 #include "CUDADataFormats/SiPixelDigi/interface/SiPixelDigisCUDA.h"
0011 #include "CalibTracker/Records/interface/SiPixelGainCalibrationForHLTGPURcd.h"
0012 #include "CalibTracker/SiPixelESProducers/interface/SiPixelROCsStatusAndMappingWrapper.h"
0013 #include "CalibTracker/SiPixelESProducers/interface/SiPixelGainCalibrationForHLTGPU.h"
0014 #include "CondFormats/DataRecord/interface/SiPixelFedCablingMapRcd.h"
0015 #include "CondFormats/SiPixelObjects/interface/SiPixelFedCablingMap.h"
0016 #include "CondFormats/SiPixelObjects/interface/SiPixelFedCablingTree.h"
0017 #include "DataFormats/FEDRawData/interface/FEDNumbering.h"
0018 #include "DataFormats/FEDRawData/interface/FEDRawData.h"
0019 #include "DataFormats/FEDRawData/interface/FEDRawDataCollection.h"
0020 #include "EventFilter/SiPixelRawToDigi/interface/PixelDataFormatter.h"
0021 #include "EventFilter/SiPixelRawToDigi/interface/PixelUnpackingRegions.h"
0022 #include "FWCore/Framework/interface/ConsumesCollector.h"
0023 #include "FWCore/Framework/interface/ESHandle.h"
0024 #include "FWCore/Framework/interface/ESTransientHandle.h"
0025 #include "FWCore/Framework/interface/ESWatcher.h"
0026 #include "FWCore/Framework/interface/Event.h"
0027 #include "FWCore/Framework/interface/EventSetup.h"
0028 #include "FWCore/Framework/interface/MakerMacros.h"
0029 #include "FWCore/Framework/interface/stream/EDProducer.h"
0030 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0031 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0032 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0033 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0034 #include "FWCore/ServiceRegistry/interface/Service.h"
0035 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0036 #include "HeterogeneousCore/CUDAServices/interface/CUDAService.h"
0037 #include "RecoTracker/Record/interface/CkfComponentsRecord.h"
0038
0039
0040 #include "SiPixelClusterThresholds.h"
0041 #include "SiPixelRawToClusterGPUKernel.h"
0042
0043 class SiPixelRawToClusterCUDA : public edm::stream::EDProducer<edm::ExternalWork> {
0044 public:
0045 explicit SiPixelRawToClusterCUDA(const edm::ParameterSet& iConfig);
0046 ~SiPixelRawToClusterCUDA() override = default;
0047
0048 static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0049
0050 private:
0051 void acquire(const edm::Event& iEvent,
0052 const edm::EventSetup& iSetup,
0053 edm::WaitingTaskWithArenaHolder waitingTaskHolder) override;
0054 void produce(edm::Event& iEvent, const edm::EventSetup& iSetup) override;
0055
0056 edm::EDGetTokenT<FEDRawDataCollection> rawGetToken_;
0057
0058 edm::EDPutTokenT<cms::cuda::Product<SiPixelDigisCUDA>> digiPutToken_;
0059 edm::EDPutTokenT<cms::cuda::Product<SiPixelDigiErrorsCUDA>> digiErrorPutToken_;
0060 edm::EDPutTokenT<cms::cuda::Product<SiPixelClustersCUDA>> clusterPutToken_;
0061
0062 cms::cuda::ContextState ctxState_;
0063
0064 edm::ESWatcher<SiPixelFedCablingMapRcd> recordWatcher_;
0065 edm::ESGetToken<SiPixelROCsStatusAndMappingWrapper, CkfComponentsRecord> gpuMapToken_;
0066 edm::ESGetToken<SiPixelGainCalibrationForHLTGPU, SiPixelGainCalibrationForHLTGPURcd> gainsToken_;
0067 edm::ESGetToken<SiPixelFedCablingMap, SiPixelFedCablingMapRcd> cablingMapToken_;
0068
0069 std::unique_ptr<SiPixelFedCablingTree> cabling_;
0070 std::vector<unsigned int> fedIds_;
0071 const SiPixelFedCablingMap* cablingMap_ = nullptr;
0072 std::unique_ptr<PixelUnpackingRegions> regions_;
0073
0074 pixelgpudetails::SiPixelRawToClusterGPUKernel gpuAlgo_;
0075 std::unique_ptr<pixelgpudetails::SiPixelRawToClusterGPUKernel::WordFedAppender> wordFedAppender_;
0076 PixelDataFormatter::Errors errors_;
0077
0078 const bool isRun2_;
0079 const bool includeErrors_;
0080 const bool useQuality_;
0081 const uint32_t maxFedWords_;
0082 uint32_t nDigis_;
0083 const SiPixelClusterThresholds clusterThresholds_;
0084 };
0085
0086 SiPixelRawToClusterCUDA::SiPixelRawToClusterCUDA(const edm::ParameterSet& iConfig)
0087 : rawGetToken_(consumes<FEDRawDataCollection>(iConfig.getParameter<edm::InputTag>("InputLabel"))),
0088 digiPutToken_(produces<cms::cuda::Product<SiPixelDigisCUDA>>()),
0089 clusterPutToken_(produces<cms::cuda::Product<SiPixelClustersCUDA>>()),
0090 gpuMapToken_(esConsumes<SiPixelROCsStatusAndMappingWrapper, CkfComponentsRecord>()),
0091 gainsToken_(esConsumes<SiPixelGainCalibrationForHLTGPU, SiPixelGainCalibrationForHLTGPURcd>()),
0092 cablingMapToken_(esConsumes<SiPixelFedCablingMap, SiPixelFedCablingMapRcd>(
0093 edm::ESInputTag("", iConfig.getParameter<std::string>("CablingMapLabel")))),
0094 isRun2_(iConfig.getParameter<bool>("isRun2")),
0095 includeErrors_(iConfig.getParameter<bool>("IncludeErrors")),
0096 useQuality_(iConfig.getParameter<bool>("UseQualityInfo")),
0097 maxFedWords_(iConfig.getParameter<uint32_t>("MaxFEDWords")),
0098 clusterThresholds_{iConfig.getParameter<int32_t>("clusterThreshold_layer1"),
0099 iConfig.getParameter<int32_t>("clusterThreshold_otherLayers")} {
0100 if (includeErrors_) {
0101 digiErrorPutToken_ = produces<cms::cuda::Product<SiPixelDigiErrorsCUDA>>();
0102 }
0103
0104
0105 if (!iConfig.getParameter<edm::ParameterSet>("Regions").getParameterNames().empty()) {
0106 regions_ = std::make_unique<PixelUnpackingRegions>(iConfig, consumesCollector());
0107 }
0108
0109 edm::Service<CUDAService> cs;
0110 if (cs->enabled()) {
0111 wordFedAppender_ = std::make_unique<pixelgpudetails::SiPixelRawToClusterGPUKernel::WordFedAppender>(maxFedWords_);
0112 }
0113 }
0114
0115 void SiPixelRawToClusterCUDA::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0116 edm::ParameterSetDescription desc;
0117 desc.add<bool>("isRun2", true);
0118 desc.add<bool>("IncludeErrors", true);
0119 desc.add<bool>("UseQualityInfo", false);
0120 desc.add<uint32_t>("MaxFEDWords", pixelgpudetails::MAX_FED * pixelgpudetails::MAX_WORD);
0121 desc.add<int32_t>("clusterThreshold_layer1", kSiPixelClusterThresholdsDefaultPhase1.layer1);
0122 desc.add<int32_t>("clusterThreshold_otherLayers", kSiPixelClusterThresholdsDefaultPhase1.otherLayers);
0123 desc.add<edm::InputTag>("InputLabel", edm::InputTag("rawDataCollector"));
0124 {
0125 edm::ParameterSetDescription psd0;
0126 psd0.addOptional<std::vector<edm::InputTag>>("inputs");
0127 psd0.addOptional<std::vector<double>>("deltaPhi");
0128 psd0.addOptional<std::vector<double>>("maxZ");
0129 psd0.addOptional<edm::InputTag>("beamSpot");
0130 desc.add<edm::ParameterSetDescription>("Regions", psd0)
0131 ->setComment("## Empty Regions PSet means complete unpacking");
0132 }
0133 desc.add<std::string>("CablingMapLabel", "")->setComment("CablingMap label");
0134 descriptions.addWithDefaultLabel(desc);
0135 }
0136
0137 void SiPixelRawToClusterCUDA::acquire(const edm::Event& iEvent,
0138 const edm::EventSetup& iSetup,
0139 edm::WaitingTaskWithArenaHolder waitingTaskHolder) {
0140 cms::cuda::ScopedContextAcquire ctx{iEvent.streamID(), std::move(waitingTaskHolder), ctxState_};
0141
0142 auto hgpuMap = iSetup.getHandle(gpuMapToken_);
0143 if (hgpuMap->hasQuality() != useQuality_) {
0144 throw cms::Exception("LogicError")
0145 << "UseQuality of the module (" << useQuality_
0146 << ") differs the one from SiPixelROCsStatusAndMappingWrapper. Please fix your configuration.";
0147 }
0148
0149 const auto* gpuMap = hgpuMap->getGPUProductAsync(ctx.stream());
0150
0151 auto hgains = iSetup.getHandle(gainsToken_);
0152
0153 const auto* gpuGains = hgains->getGPUProductAsync(ctx.stream());
0154
0155 cms::cuda::device::unique_ptr<unsigned char[]> modulesToUnpackRegional;
0156 const unsigned char* gpuModulesToUnpack;
0157
0158 if (regions_) {
0159 regions_->run(iEvent, iSetup);
0160 LogDebug("SiPixelRawToCluster") << "region2unpack #feds: " << regions_->nFEDs();
0161 LogDebug("SiPixelRawToCluster") << "region2unpack #modules (BPIX,EPIX,total): " << regions_->nBarrelModules() << " "
0162 << regions_->nForwardModules() << " " << regions_->nModules();
0163 modulesToUnpackRegional = hgpuMap->getModToUnpRegionalAsync(*(regions_->modulesToUnpack()), ctx.stream());
0164 gpuModulesToUnpack = modulesToUnpackRegional.get();
0165 } else {
0166 gpuModulesToUnpack = hgpuMap->getModToUnpAllAsync(ctx.stream());
0167 }
0168
0169
0170 if (recordWatcher_.check(iSetup)) {
0171
0172 auto cablingMap = iSetup.getTransientHandle(cablingMapToken_);
0173 cablingMap_ = cablingMap.product();
0174 fedIds_ = cablingMap->fedIds();
0175 cabling_ = cablingMap->cablingTree();
0176 LogDebug("map version:") << cabling_->version();
0177 }
0178
0179 const auto& buffers = iEvent.get(rawGetToken_);
0180
0181 errors_.clear();
0182
0183
0184 unsigned int wordCounterGPU = 0;
0185 unsigned int fedCounter = 0;
0186 bool errorsInEvent = false;
0187
0188
0189 ErrorChecker errorcheck;
0190 for (int fedId : fedIds_) {
0191 if (regions_ && !regions_->mayUnpackFED(fedId))
0192 continue;
0193
0194
0195
0196
0197 assert(fedId >= FEDNumbering::MINSiPixeluTCAFEDID);
0198 fedCounter++;
0199
0200
0201 const FEDRawData& rawData = buffers.FEDData(fedId);
0202
0203
0204 int nWords = rawData.size() / sizeof(cms_uint64_t);
0205 if (nWords == 0) {
0206 continue;
0207 }
0208
0209
0210 const cms_uint64_t* trailer = reinterpret_cast<const cms_uint64_t*>(rawData.data()) + (nWords - 1);
0211 if (not errorcheck.checkCRC(errorsInEvent, fedId, trailer, errors_)) {
0212 continue;
0213 }
0214
0215
0216 const cms_uint64_t* header = reinterpret_cast<const cms_uint64_t*>(rawData.data());
0217 header--;
0218 bool moreHeaders = true;
0219 while (moreHeaders) {
0220 header++;
0221 bool headerStatus = errorcheck.checkHeader(errorsInEvent, fedId, header, errors_);
0222 moreHeaders = headerStatus;
0223 }
0224
0225
0226 bool moreTrailers = true;
0227 trailer++;
0228 while (moreTrailers) {
0229 trailer--;
0230 bool trailerStatus = errorcheck.checkTrailer(errorsInEvent, fedId, nWords, trailer, errors_);
0231 moreTrailers = trailerStatus;
0232 }
0233
0234 const cms_uint32_t* bw = (const cms_uint32_t*)(header + 1);
0235 const cms_uint32_t* ew = (const cms_uint32_t*)(trailer);
0236
0237 assert(0 == (ew - bw) % 2);
0238 wordFedAppender_->initializeWordFed(fedId, wordCounterGPU, bw, (ew - bw));
0239 wordCounterGPU += (ew - bw);
0240
0241 }
0242
0243 nDigis_ = wordCounterGPU;
0244
0245 if (nDigis_ == 0)
0246 return;
0247
0248 gpuAlgo_.makeClustersAsync(isRun2_,
0249 clusterThresholds_,
0250 gpuMap,
0251 gpuModulesToUnpack,
0252 gpuGains,
0253 *wordFedAppender_,
0254 std::move(errors_),
0255 wordCounterGPU,
0256 fedCounter,
0257 maxFedWords_,
0258 useQuality_,
0259 includeErrors_,
0260 edm::MessageDrop::instance()->debugEnabled,
0261 ctx.stream());
0262 }
0263
0264 void SiPixelRawToClusterCUDA::produce(edm::Event& iEvent, const edm::EventSetup& iSetup) {
0265 cms::cuda::ScopedContextProduce ctx{ctxState_};
0266
0267 if (nDigis_ == 0) {
0268
0269 auto tmp = std::make_pair(SiPixelDigisCUDA{}, SiPixelClustersCUDA{});
0270 ctx.emplace(iEvent, digiPutToken_, std::move(tmp.first));
0271 ctx.emplace(iEvent, clusterPutToken_, std::move(tmp.second));
0272 if (includeErrors_) {
0273 ctx.emplace(iEvent, digiErrorPutToken_, SiPixelDigiErrorsCUDA{});
0274 }
0275 return;
0276 }
0277
0278 auto tmp = gpuAlgo_.getResults();
0279 ctx.emplace(iEvent, digiPutToken_, std::move(tmp.first));
0280 ctx.emplace(iEvent, clusterPutToken_, std::move(tmp.second));
0281 if (includeErrors_) {
0282 ctx.emplace(iEvent, digiErrorPutToken_, gpuAlgo_.getErrors());
0283 }
0284 }
0285
0286
0287 DEFINE_FWK_MODULE(SiPixelRawToClusterCUDA);