File indexing completed on 2023-01-19 02:53:17
0001
0002 #include <memory>
0003 #include <string>
0004 #include <vector>
0005
0006
0007 #include "CUDADataFormats/Common/interface/Product.h"
0008 #include "CUDADataFormats/SiPixelCluster/interface/SiPixelClustersCUDA.h"
0009 #include "CUDADataFormats/SiPixelDigi/interface/SiPixelDigiErrorsCUDA.h"
0010 #include "CUDADataFormats/SiPixelDigi/interface/SiPixelDigisCUDA.h"
0011 #include "CalibTracker/Records/interface/SiPixelGainCalibrationForHLTGPURcd.h"
0012 #include "CalibTracker/SiPixelESProducers/interface/SiPixelROCsStatusAndMappingWrapper.h"
0013 #include "CalibTracker/SiPixelESProducers/interface/SiPixelGainCalibrationForHLTGPU.h"
0014 #include "CondFormats/DataRecord/interface/SiPixelFedCablingMapRcd.h"
0015 #include "CondFormats/SiPixelObjects/interface/SiPixelFedCablingMap.h"
0016 #include "CondFormats/SiPixelObjects/interface/SiPixelFedCablingTree.h"
0017 #include "DataFormats/FEDRawData/interface/FEDNumbering.h"
0018 #include "DataFormats/FEDRawData/interface/FEDRawData.h"
0019 #include "DataFormats/FEDRawData/interface/FEDRawDataCollection.h"
0020 #include "EventFilter/SiPixelRawToDigi/interface/PixelDataFormatter.h"
0021 #include "EventFilter/SiPixelRawToDigi/interface/PixelUnpackingRegions.h"
0022 #include "FWCore/Framework/interface/ConsumesCollector.h"
0023 #include "FWCore/Framework/interface/ESHandle.h"
0024 #include "FWCore/Framework/interface/ESTransientHandle.h"
0025 #include "FWCore/Framework/interface/ESWatcher.h"
0026 #include "FWCore/Framework/interface/Event.h"
0027 #include "FWCore/Framework/interface/EventSetup.h"
0028 #include "FWCore/Framework/interface/MakerMacros.h"
0029 #include "FWCore/Framework/interface/stream/EDProducer.h"
0030 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0031 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0032 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0033 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0034 #include "FWCore/ServiceRegistry/interface/Service.h"
0035 #include "Geometry/CommonTopologies/interface/SimplePixelTopology.h"
0036 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
0037 #include "HeterogeneousCore/CUDAServices/interface/CUDAService.h"
0038 #include "RecoTracker/Record/interface/CkfComponentsRecord.h"
0039
0040
0041 #include "SiPixelClusterThresholds.h"
0042 #include "SiPixelRawToClusterGPUKernel.h"
0043
0044 class SiPixelRawToClusterCUDA : public edm::stream::EDProducer<edm::ExternalWork> {
0045 public:
0046 explicit SiPixelRawToClusterCUDA(const edm::ParameterSet& iConfig);
0047 ~SiPixelRawToClusterCUDA() override = default;
0048
0049 static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0050
0051 private:
0052 void acquire(const edm::Event& iEvent,
0053 const edm::EventSetup& iSetup,
0054 edm::WaitingTaskWithArenaHolder waitingTaskHolder) override;
0055 void produce(edm::Event& iEvent, const edm::EventSetup& iSetup) override;
0056
0057 edm::EDGetTokenT<FEDRawDataCollection> rawGetToken_;
0058
0059 edm::EDPutTokenT<cms::cuda::Product<SiPixelDigisCUDA>> digiPutToken_;
0060 edm::EDPutTokenT<cms::cuda::Product<SiPixelDigiErrorsCUDA>> digiErrorPutToken_;
0061 edm::EDPutTokenT<cms::cuda::Product<SiPixelClustersCUDA>> clusterPutToken_;
0062
0063 cms::cuda::ContextState ctxState_;
0064
0065 edm::ESWatcher<SiPixelFedCablingMapRcd> recordWatcher_;
0066 edm::ESGetToken<SiPixelROCsStatusAndMappingWrapper, CkfComponentsRecord> gpuMapToken_;
0067 edm::ESGetToken<SiPixelGainCalibrationForHLTGPU, SiPixelGainCalibrationForHLTGPURcd> gainsToken_;
0068 edm::ESGetToken<SiPixelFedCablingMap, SiPixelFedCablingMapRcd> cablingMapToken_;
0069
0070 std::unique_ptr<SiPixelFedCablingTree> cabling_;
0071 std::vector<unsigned int> fedIds_;
0072 const SiPixelFedCablingMap* cablingMap_ = nullptr;
0073 std::unique_ptr<PixelUnpackingRegions> regions_;
0074
0075 pixelgpudetails::SiPixelRawToClusterGPUKernel gpuAlgo_;
0076 PixelDataFormatter::Errors errors_;
0077
0078 const bool isRun2_;
0079 const bool includeErrors_;
0080 const bool useQuality_;
0081 uint32_t nDigis_;
0082 const SiPixelClusterThresholds clusterThresholds_;
0083 };
0084
0085 SiPixelRawToClusterCUDA::SiPixelRawToClusterCUDA(const edm::ParameterSet& iConfig)
0086 : rawGetToken_(consumes<FEDRawDataCollection>(iConfig.getParameter<edm::InputTag>("InputLabel"))),
0087 digiPutToken_(produces<cms::cuda::Product<SiPixelDigisCUDA>>()),
0088 clusterPutToken_(produces<cms::cuda::Product<SiPixelClustersCUDA>>()),
0089 gpuMapToken_(esConsumes<SiPixelROCsStatusAndMappingWrapper, CkfComponentsRecord>()),
0090 gainsToken_(esConsumes<SiPixelGainCalibrationForHLTGPU, SiPixelGainCalibrationForHLTGPURcd>()),
0091 cablingMapToken_(esConsumes<SiPixelFedCablingMap, SiPixelFedCablingMapRcd>(
0092 edm::ESInputTag("", iConfig.getParameter<std::string>("CablingMapLabel")))),
0093 isRun2_(iConfig.getParameter<bool>("isRun2")),
0094 includeErrors_(iConfig.getParameter<bool>("IncludeErrors")),
0095 useQuality_(iConfig.getParameter<bool>("UseQualityInfo")),
0096 clusterThresholds_{iConfig.getParameter<int32_t>("clusterThreshold_layer1"),
0097 iConfig.getParameter<int32_t>("clusterThreshold_otherLayers")} {
0098 if (includeErrors_) {
0099 digiErrorPutToken_ = produces<cms::cuda::Product<SiPixelDigiErrorsCUDA>>();
0100 }
0101
0102
0103 if (!iConfig.getParameter<edm::ParameterSet>("Regions").getParameterNames().empty()) {
0104 regions_ = std::make_unique<PixelUnpackingRegions>(iConfig, consumesCollector());
0105 }
0106 }
0107
0108 void SiPixelRawToClusterCUDA::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0109 edm::ParameterSetDescription desc;
0110 desc.add<bool>("isRun2", true);
0111 desc.add<bool>("IncludeErrors", true);
0112 desc.add<bool>("UseQualityInfo", false);
0113
0114
0115 desc.addOptionalNode(edm::ParameterDescription<uint32_t>("MaxFEDWords", 0, true), false)
0116 ->setComment("This parameter is obsolete and will be ignored.");
0117 desc.add<int32_t>("clusterThreshold_layer1", kSiPixelClusterThresholdsDefaultPhase1.layer1);
0118 desc.add<int32_t>("clusterThreshold_otherLayers", kSiPixelClusterThresholdsDefaultPhase1.otherLayers);
0119 desc.add<edm::InputTag>("InputLabel", edm::InputTag("rawDataCollector"));
0120 {
0121 edm::ParameterSetDescription psd0;
0122 psd0.addOptional<std::vector<edm::InputTag>>("inputs");
0123 psd0.addOptional<std::vector<double>>("deltaPhi");
0124 psd0.addOptional<std::vector<double>>("maxZ");
0125 psd0.addOptional<edm::InputTag>("beamSpot");
0126 desc.add<edm::ParameterSetDescription>("Regions", psd0)
0127 ->setComment("## Empty Regions PSet means complete unpacking");
0128 }
0129 desc.add<std::string>("CablingMapLabel", "")->setComment("CablingMap label");
0130 descriptions.addWithDefaultLabel(desc);
0131 }
0132
0133 void SiPixelRawToClusterCUDA::acquire(const edm::Event& iEvent,
0134 const edm::EventSetup& iSetup,
0135 edm::WaitingTaskWithArenaHolder waitingTaskHolder) {
0136 cms::cuda::ScopedContextAcquire ctx{iEvent.streamID(), std::move(waitingTaskHolder), ctxState_};
0137
0138 auto hgpuMap = iSetup.getHandle(gpuMapToken_);
0139 if (hgpuMap->hasQuality() != useQuality_) {
0140 throw cms::Exception("LogicError")
0141 << "UseQuality of the module (" << useQuality_
0142 << ") differs the one from SiPixelROCsStatusAndMappingWrapper. Please fix your configuration.";
0143 }
0144
0145 const auto* gpuMap = hgpuMap->getGPUProductAsync(ctx.stream());
0146
0147 auto hgains = iSetup.getHandle(gainsToken_);
0148
0149 const auto* gpuGains = hgains->getGPUProductAsync(ctx.stream());
0150
0151 cms::cuda::device::unique_ptr<unsigned char[]> modulesToUnpackRegional;
0152 const unsigned char* gpuModulesToUnpack;
0153
0154 if (regions_) {
0155 regions_->run(iEvent, iSetup);
0156 LogDebug("SiPixelRawToCluster") << "region2unpack #feds: " << regions_->nFEDs();
0157 LogDebug("SiPixelRawToCluster") << "region2unpack #modules (BPIX,EPIX,total): " << regions_->nBarrelModules() << " "
0158 << regions_->nForwardModules() << " " << regions_->nModules();
0159 modulesToUnpackRegional = hgpuMap->getModToUnpRegionalAsync(*(regions_->modulesToUnpack()), ctx.stream());
0160 gpuModulesToUnpack = modulesToUnpackRegional.get();
0161 } else {
0162 gpuModulesToUnpack = hgpuMap->getModToUnpAllAsync(ctx.stream());
0163 }
0164
0165
0166 if (recordWatcher_.check(iSetup)) {
0167
0168 auto cablingMap = iSetup.getTransientHandle(cablingMapToken_);
0169 cablingMap_ = cablingMap.product();
0170 fedIds_ = cablingMap->fedIds();
0171 cabling_ = cablingMap->cablingTree();
0172 LogDebug("map version:") << cabling_->version();
0173 }
0174
0175 const auto& buffers = iEvent.get(rawGetToken_);
0176
0177 errors_.clear();
0178
0179
0180 unsigned int wordCounter = 0;
0181 unsigned int fedCounter = 0;
0182 bool errorsInEvent = false;
0183
0184 std::vector<unsigned int> index(fedIds_.size(), 0);
0185 std::vector<cms_uint32_t const*> start(fedIds_.size(), nullptr);
0186 std::vector<ptrdiff_t> words(fedIds_.size(), 0);
0187
0188
0189 ErrorChecker errorcheck;
0190 for (uint32_t i = 0; i < fedIds_.size(); ++i) {
0191 const int fedId = fedIds_[i];
0192 if (regions_ && !regions_->mayUnpackFED(fedId))
0193 continue;
0194
0195
0196
0197
0198 assert(fedId >= FEDNumbering::MINSiPixeluTCAFEDID);
0199 fedCounter++;
0200
0201
0202 const FEDRawData& rawData = buffers.FEDData(fedId);
0203
0204
0205 int nWords = rawData.size() / sizeof(cms_uint64_t);
0206 if (nWords == 0) {
0207 continue;
0208 }
0209
0210
0211 const cms_uint64_t* trailer = reinterpret_cast<const cms_uint64_t*>(rawData.data()) + (nWords - 1);
0212 if (not errorcheck.checkCRC(errorsInEvent, fedId, trailer, errors_)) {
0213 continue;
0214 }
0215
0216
0217 const cms_uint64_t* header = reinterpret_cast<const cms_uint64_t*>(rawData.data());
0218 header--;
0219 bool moreHeaders = true;
0220 while (moreHeaders) {
0221 header++;
0222 bool headerStatus = errorcheck.checkHeader(errorsInEvent, fedId, header, errors_);
0223 moreHeaders = headerStatus;
0224 }
0225
0226
0227 bool moreTrailers = true;
0228 trailer++;
0229 while (moreTrailers) {
0230 trailer--;
0231 bool trailerStatus = errorcheck.checkTrailer(errorsInEvent, fedId, nWords, trailer, errors_);
0232 moreTrailers = trailerStatus;
0233 }
0234
0235 const cms_uint32_t* bw = (const cms_uint32_t*)(header + 1);
0236 const cms_uint32_t* ew = (const cms_uint32_t*)(trailer);
0237
0238 assert(0 == (ew - bw) % 2);
0239 index[i] = wordCounter;
0240 start[i] = bw;
0241 words[i] = (ew - bw);
0242 wordCounter += (ew - bw);
0243
0244 }
0245
0246 nDigis_ = wordCounter;
0247
0248 if (nDigis_ == 0)
0249 return;
0250
0251
0252 pixelgpudetails::SiPixelRawToClusterGPUKernel::WordFedAppender wordFedAppender(nDigis_, ctx.stream());
0253 for (uint32_t i = 0; i < fedIds_.size(); ++i) {
0254 wordFedAppender.initializeWordFed(fedIds_[i], index[i], start[i], words[i]);
0255 }
0256
0257 gpuAlgo_.makeClustersAsync(isRun2_,
0258 clusterThresholds_,
0259 gpuMap,
0260 gpuModulesToUnpack,
0261 gpuGains,
0262 wordFedAppender,
0263 std::move(errors_),
0264 wordCounter,
0265 fedCounter,
0266 useQuality_,
0267 includeErrors_,
0268 edm::MessageDrop::instance()->debugEnabled,
0269 ctx.stream());
0270 }
0271
0272 void SiPixelRawToClusterCUDA::produce(edm::Event& iEvent, const edm::EventSetup& iSetup) {
0273 cms::cuda::ScopedContextProduce ctx{ctxState_};
0274
0275 if (nDigis_ == 0) {
0276
0277
0278
0279
0280
0281 SiPixelDigisCUDA digis_d = SiPixelDigisCUDA(nDigis_, ctx.stream());
0282 SiPixelClustersCUDA clusters_d = SiPixelClustersCUDA(pixelTopology::Phase1::numberOfModules, ctx.stream());
0283 ctx.emplace(iEvent, digiPutToken_, std::move(digis_d));
0284 ctx.emplace(iEvent, clusterPutToken_, std::move(clusters_d));
0285 if (includeErrors_) {
0286 ctx.emplace(iEvent, digiErrorPutToken_, SiPixelDigiErrorsCUDA{});
0287 }
0288 return;
0289 }
0290
0291 auto tmp = gpuAlgo_.getResults();
0292 ctx.emplace(iEvent, digiPutToken_, std::move(tmp.first));
0293 ctx.emplace(iEvent, clusterPutToken_, std::move(tmp.second));
0294 if (includeErrors_) {
0295 ctx.emplace(iEvent, digiErrorPutToken_, gpuAlgo_.getErrors());
0296 }
0297 }
0298
0299
0300 DEFINE_FWK_MODULE(SiPixelRawToClusterCUDA);