SiPixelESProducers/src/SiPixelROCsStatusAndMappingWrapper.cc

0001 // C++ includes
0002 #include <algorithm>
0003 #include <iomanip>
0004 #include <iostream>
0005 #include <vector>
0006
0007 // CUDA includes
0008 #include <cuda_runtime.h>
0009
0010 // CMSSW includes
0011 #include "CUDADataFormats/SiPixelCluster/interface/gpuClusteringConstants.h"
0012 #include "CalibTracker/SiPixelESProducers/interface/SiPixelROCsStatusAndMappingWrapper.h"
0013 #include "CondFormats/SiPixelObjects/interface/SiPixelFedCablingMap.h"
0014 #include "CondFormats/SiPixelObjects/interface/SiPixelFedCablingTree.h"
0015 #include "CondFormats/SiPixelObjects/interface/SiPixelQuality.h"
0016 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0017 #include "Geometry/CommonDetUnit/interface/GeomDetType.h"
0018 #include "Geometry/TrackerGeometryBuilder/interface/TrackerGeometry.h"
0019 #include "HeterogeneousCore/CUDAUtilities/interface/cudaCheck.h"
0020 #include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
0021 #include "HeterogeneousCore/CUDAUtilities/interface/host_unique_ptr.h"
0022
0023 SiPixelROCsStatusAndMappingWrapper::SiPixelROCsStatusAndMappingWrapper(SiPixelFedCablingMap const& cablingMap,
0024                                                                        TrackerGeometry const& trackerGeom,
0025                                                                        SiPixelQuality const* badPixelInfo)
0026     : cablingMap_(&cablingMap), modToUnpDefault(pixelgpudetails::MAX_SIZE), hasQuality_(badPixelInfo != nullptr) {
0027   cudaCheck(cudaMallocHost(&cablingMapHost, sizeof(SiPixelROCsStatusAndMapping)));
0028
0029   std::vector<unsigned int> const& fedIds = cablingMap.fedIds();
0030   std::unique_ptr<SiPixelFedCablingTree> const& cabling = cablingMap.cablingTree();
0031
0032   unsigned int startFed = *(fedIds.begin());
0033   unsigned int endFed = *(fedIds.end() - 1);
0034
0035   sipixelobjects::CablingPathToDetUnit path;
0036   int index = 1;
0037
0038   for (unsigned int fed = startFed; fed <= endFed; fed++) {
0039     for (unsigned int link = 1; link <= pixelgpudetails::MAX_LINK; link++) {
0040       for (unsigned int roc = 1; roc <= pixelgpudetails::MAX_ROC; roc++) {
0041         path = {fed, link, roc};
0042         const sipixelobjects::PixelROC* pixelRoc = cabling->findItem(path);
0043         cablingMapHost->fed[index] = fed;
0044         cablingMapHost->link[index] = link;
0045         cablingMapHost->roc[index] = roc;
0046         if (pixelRoc != nullptr) {
0047           cablingMapHost->rawId[index] = pixelRoc->rawId();
0048           cablingMapHost->rocInDet[index] = pixelRoc->idInDetUnit();
0049           modToUnpDefault[index] = false;
0050           if (badPixelInfo != nullptr)
0051             cablingMapHost->badRocs[index] = badPixelInfo->IsRocBad(pixelRoc->rawId(), pixelRoc->idInDetUnit());
0052           else
0053             cablingMapHost->badRocs[index] = false;
0054         } else {  // store some dummy number
0055           cablingMapHost->rawId[index] = gpuClustering::invalidModuleId;
0056           cablingMapHost->rocInDet[index] = gpuClustering::invalidModuleId;
0057           cablingMapHost->badRocs[index] = true;
0058           modToUnpDefault[index] = true;
0059         }
0060         index++;
0061       }
0062     }
0063   }  // end of FED loop
0064
0065   // Given FedId, Link and idinLnk; use the following formula
0066   // to get the rawId and idinDU
0067   // index = (FedID-1200) * MAX_LINK* MAX_ROC + (Link-1)* MAX_ROC + idinLnk;
0068   // where, MAX_LINK = 48, MAX_ROC = 8 for Phase1 as mentioned Danek's email
0069   // FedID varies between 1200 to 1338 (In total 108 FED's)
0070   // Link varies between 1 to 48
0071   // idinLnk varies between 1 to 8
0072
0073   for (int i = 1; i < index; i++) {
0074     if (cablingMapHost->rawId[i] == gpuClustering::invalidModuleId) {
0075       cablingMapHost->moduleId[i] = gpuClustering::invalidModuleId;
0076     } else {
0077       /*
0078       std::cout << cablingMapHost->rawId[i] << std::endl;
0079       */
0080       auto gdet = trackerGeom.idToDetUnit(cablingMapHost->rawId[i]);
0081       if (!gdet) {
0082         LogDebug("SiPixelROCsStatusAndMapping") << " Not found: " << cablingMapHost->rawId[i] << std::endl;
0083         continue;
0084       }
0085       cablingMapHost->moduleId[i] = gdet->index();
0086     }
0087     LogDebug("SiPixelROCsStatusAndMapping")
0088         << "----------------------------------------------------------------------------" << std::endl;
0089     LogDebug("SiPixelROCsStatusAndMapping")
0090         << i << std::setw(20) << cablingMapHost->fed[i] << std::setw(20) << cablingMapHost->link[i] << std::setw(20)
0091         << cablingMapHost->roc[i] << std::endl;
0092     LogDebug("SiPixelROCsStatusAndMapping")
0093         << i << std::setw(20) << cablingMapHost->rawId[i] << std::setw(20) << cablingMapHost->rocInDet[i]
0094         << std::setw(20) << cablingMapHost->moduleId[i] << std::endl;
0095     LogDebug("SiPixelROCsStatusAndMapping")
0096         << i << std::setw(20) << (bool)cablingMapHost->badRocs[i] << std::setw(20) << std::endl;
0097     LogDebug("SiPixelROCsStatusAndMapping")
0098         << "----------------------------------------------------------------------------" << std::endl;
0099   }
0100
0101   cablingMapHost->size = index - 1;
0102 }
0103
0104 SiPixelROCsStatusAndMappingWrapper::~SiPixelROCsStatusAndMappingWrapper() { cudaCheck(cudaFreeHost(cablingMapHost)); }
0105
0106 const SiPixelROCsStatusAndMapping* SiPixelROCsStatusAndMappingWrapper::getGPUProductAsync(
0107     cudaStream_t cudaStream) const {
0108   const auto& data = gpuData_.dataForCurrentDeviceAsync(cudaStream, [this](GPUData& data, cudaStream_t stream) {
0109     // allocate
0110     cudaCheck(cudaMalloc(&data.cablingMapDevice, sizeof(SiPixelROCsStatusAndMapping)));
0111
0112     // transfer
0113     cudaCheck(cudaMemcpyAsync(
0114         data.cablingMapDevice, this->cablingMapHost, sizeof(SiPixelROCsStatusAndMapping), cudaMemcpyDefault, stream));
0115   });
0116   return data.cablingMapDevice;
0117 }
0118
0119 const unsigned char* SiPixelROCsStatusAndMappingWrapper::getModToUnpAllAsync(cudaStream_t cudaStream) const {
0120   const auto& data =
0121       modToUnp_.dataForCurrentDeviceAsync(cudaStream, [this](ModulesToUnpack& data, cudaStream_t stream) {
0122         cudaCheck(cudaMalloc((void**)&data.modToUnpDefault, pixelgpudetails::MAX_SIZE_BYTE_BOOL));
0123         cudaCheck(cudaMemcpyAsync(data.modToUnpDefault,
0124                                   this->modToUnpDefault.data(),
0125                                   this->modToUnpDefault.size() * sizeof(unsigned char),
0126                                   cudaMemcpyDefault,
0127                                   stream));
0128       });
0129   return data.modToUnpDefault;
0130 }
0131
0132 cms::cuda::device::unique_ptr<unsigned char[]> SiPixelROCsStatusAndMappingWrapper::getModToUnpRegionalAsync(
0133     std::set<unsigned int> const& modules, cudaStream_t cudaStream) const {
0134   auto modToUnpDevice = cms::cuda::make_device_unique<unsigned char[]>(pixelgpudetails::MAX_SIZE, cudaStream);
0135   auto modToUnpHost = cms::cuda::make_host_unique<unsigned char[]>(pixelgpudetails::MAX_SIZE, cudaStream);
0136
0137   std::vector<unsigned int> const& fedIds = cablingMap_->fedIds();
0138   std::unique_ptr<SiPixelFedCablingTree> const& cabling = cablingMap_->cablingTree();
0139
0140   unsigned int startFed = *(fedIds.begin());
0141   unsigned int endFed = *(fedIds.end() - 1);
0142
0143   sipixelobjects::CablingPathToDetUnit path;
0144   int index = 1;
0145
0146   for (unsigned int fed = startFed; fed <= endFed; fed++) {
0147     for (unsigned int link = 1; link <= pixelgpudetails::MAX_LINK; link++) {
0148       for (unsigned int roc = 1; roc <= pixelgpudetails::MAX_ROC; roc++) {
0149         path = {fed, link, roc};
0150         const sipixelobjects::PixelROC* pixelRoc = cabling->findItem(path);
0151         if (pixelRoc != nullptr) {
0152           modToUnpHost[index] = (not modules.empty()) and (modules.find(pixelRoc->rawId()) == modules.end());
0153         } else {  // store some dummy number
0154           modToUnpHost[index] = true;
0155         }
0156         index++;
0157       }
0158     }
0159   }
0160
0161   cudaCheck(cudaMemcpyAsync(modToUnpDevice.get(),
0162                             modToUnpHost.get(),
0163                             pixelgpudetails::MAX_SIZE * sizeof(unsigned char),
0164                             cudaMemcpyHostToDevice,
0165                             cudaStream));
0166   return modToUnpDevice;
0167 }
0168
0169 SiPixelROCsStatusAndMappingWrapper::GPUData::~GPUData() { cudaCheck(cudaFree(cablingMapDevice)); }
0170
0171 SiPixelROCsStatusAndMappingWrapper::ModulesToUnpack::~ModulesToUnpack() { cudaCheck(cudaFree(modToUnpDefault)); }