HGCalRecProducers/plugins/HeterogeneousHGCalHEFConditions.cc

0001 #include "RecoLocalCalo/HGCalRecProducers/plugins/HeterogeneousHGCalHEFConditions.h"
0002
0003 HeterogeneousHGCalHEFConditionsWrapper::HeterogeneousHGCalHEFConditionsWrapper(
0004     const HGCalParameters* cpuHGCalParameters) {
0005   //HGCalParameters as defined in CMSSW
0006   this->sizes_params_ = calculate_memory_bytes_params_(cpuHGCalParameters);
0007   this->chunk_params_ = allocate_memory_params_(this->sizes_params_);
0008   transfer_data_to_heterogeneous_pointers_params_(this->sizes_params_, cpuHGCalParameters);
0009 }
0010
0011 size_t HeterogeneousHGCalHEFConditionsWrapper::allocate_memory_params_(const std::vector<size_t>& sz) {
0012   size_t chunk_ = std::accumulate(sz.begin(), sz.end(), 0);  //total memory required in bytes
0013   cudaCheck(cudaMallocHost(&this->params_.cellFineX_, chunk_));
0014   return chunk_;
0015 }
0016
0017 void HeterogeneousHGCalHEFConditionsWrapper::transfer_data_to_heterogeneous_pointers_params_(
0018     const std::vector<size_t>& sz, const HGCalParameters* cpuParams) {
0019   //store cumulative sum in bytes and convert it to sizes in units of C++ typesHEF, i.e., number if items to be transferred to GPU
0020   std::vector<size_t> cumsum_sizes(sz.size() + 1, 0);  //starting with zero
0021   std::partial_sum(sz.begin(), sz.end(), cumsum_sizes.begin() + 1);
0022   for (unsigned int i = 1; i < cumsum_sizes.size(); ++i)  //start at second element (the first is zero)
0023   {
0024     size_t typesHEFsize = 0;
0025     if (cpar::typesHEF[i - 1] == cpar::HeterogeneousHGCalHEFParametersType::Double)
0026       typesHEFsize = sizeof(double);
0027     else if (cpar::typesHEF[i - 1] == cpar::HeterogeneousHGCalHEFParametersType::Int32_t)
0028       typesHEFsize = sizeof(int32_t);
0029     else
0030       throw cms::Exception("HeterogeneousHGCalHEFConditionsWrapper") << "Wrong HeterogeneousHGCalParameters type";
0031     cumsum_sizes[i] /= typesHEFsize;
0032   }
0033
0034   for (unsigned int j = 0; j < sz.size(); ++j) {
0035     //setting the pointers
0036     if (j != 0) {
0037       const unsigned int jm1 = j - 1;
0038       const size_t shift = cumsum_sizes[j] - cumsum_sizes[jm1];
0039       if (cpar::typesHEF[jm1] == cpar::HeterogeneousHGCalHEFParametersType::Double and
0040           cpar::typesHEF[j] == cpar::HeterogeneousHGCalHEFParametersType::Double)
0041         select_pointer_d_(&this->params_, j) = select_pointer_d_(&this->params_, jm1) + shift;
0042       else if (cpar::typesHEF[jm1] == cpar::HeterogeneousHGCalHEFParametersType::Double and
0043                cpar::typesHEF[j] == cpar::HeterogeneousHGCalHEFParametersType::Int32_t)
0044         select_pointer_i_(&this->params_, j) =
0045             reinterpret_cast<int32_t*>(select_pointer_d_(&this->params_, jm1) + shift);
0046     }
0047
0048     //copying the pointers' content
0049     for (unsigned int i = cumsum_sizes[j]; i < cumsum_sizes[j + 1]; ++i) {
0050       unsigned int index = i - cumsum_sizes[j];
0051       if (cpar::typesHEF[j] == cpar::HeterogeneousHGCalHEFParametersType::Double) {
0052         select_pointer_d_(&this->params_, j)[index] = select_pointer_d_(cpuParams, j)[index];
0053       } else if (cpar::typesHEF[j] == cpar::HeterogeneousHGCalHEFParametersType::Int32_t) {
0054         select_pointer_i_(&this->params_, j)[index] = select_pointer_i_(cpuParams, j)[index];
0055       } else
0056         throw cms::Exception("HeterogeneousHGCalHEFConditionsWrapper") << "Wrong HeterogeneousHGCalParameters type";
0057     }
0058   }
0059 }
0060
0061 std::vector<size_t> HeterogeneousHGCalHEFConditionsWrapper::calculate_memory_bytes_params_(
0062     const HGCalParameters* cpuParams) {
0063   size_t npointers = hgcal_conditions::parameters::typesHEF.size();
0064   std::vector<size_t> sizes(npointers);
0065   for (unsigned int i = 0; i < npointers; ++i) {
0066     if (cpar::typesHEF[i] == cpar::HeterogeneousHGCalHEFParametersType::Double)
0067       sizes[i] = select_pointer_d_(cpuParams, i).size();
0068     else
0069       sizes[i] = select_pointer_i_(cpuParams, i).size();
0070   }
0071
0072   std::vector<size_t> sizes_units(npointers);
0073   for (unsigned int i = 0; i < npointers; ++i) {
0074     if (cpar::typesHEF[i] == cpar::HeterogeneousHGCalHEFParametersType::Double)
0075       sizes_units[i] = sizeof(double);
0076     else if (cpar::typesHEF[i] == cpar::HeterogeneousHGCalHEFParametersType::Int32_t)
0077       sizes_units[i] = sizeof(int32_t);
0078   }
0079
0080   //element by element multiplication
0081   this->sizes_params_.resize(npointers);
0082   std::transform(
0083       sizes.begin(), sizes.end(), sizes_units.begin(), this->sizes_params_.begin(), std::multiplies<size_t>());
0084   return this->sizes_params_;
0085 }
0086
0087 HeterogeneousHGCalHEFConditionsWrapper::~HeterogeneousHGCalHEFConditionsWrapper() {
0088   cudaCheck(cudaFreeHost(this->params_.cellFineX_));
0089 }
0090
0091 //I could use template specializations
0092 //try to use std::variant in the future to avoid similar functions with different return values
0093 double*& HeterogeneousHGCalHEFConditionsWrapper::select_pointer_d_(cpar::HeterogeneousHGCalHEFParameters* cpuObject,
0094                                                                    const unsigned int& item) const {
0095   switch (item) {
0096     case 0:
0097       return cpuObject->cellFineX_;
0098     case 1:
0099       return cpuObject->cellFineY_;
0100     case 2:
0101       return cpuObject->cellCoarseX_;
0102     case 3:
0103       return cpuObject->cellCoarseY_;
0104     default:
0105       edm::LogError("HeterogeneousHGCalHEFConditionsWrapper") << "select_pointer_d(heterogeneous): no item.";
0106       return cpuObject->cellCoarseY_;
0107   }
0108 }
0109
0110 std::vector<double> HeterogeneousHGCalHEFConditionsWrapper::select_pointer_d_(const HGCalParameters* cpuObject,
0111                                                                               const unsigned int& item) const {
0112   switch (item) {
0113     case 0:
0114       return cpuObject->cellFineX_;
0115     case 1:
0116       return cpuObject->cellFineY_;
0117     case 2:
0118       return cpuObject->cellCoarseX_;
0119     case 3:
0120       return cpuObject->cellCoarseY_;
0121     default:
0122       edm::LogError("HeterogeneousHGCalHEFConditionsWrapper") << "select_pointer_d(non-heterogeneous): no item.";
0123       return cpuObject->cellCoarseY_;
0124   }
0125 }
0126
0127 int32_t*& HeterogeneousHGCalHEFConditionsWrapper::select_pointer_i_(cpar::HeterogeneousHGCalHEFParameters* cpuObject,
0128                                                                     const unsigned int& item) const {
0129   switch (item) {
0130     case 4:
0131       return cpuObject->waferTypeL_;
0132     default:
0133       edm::LogError("HeterogeneousHGCalHEFConditionsWrapper") << "select_pointer_i(heterogeneous): no item.";
0134       return cpuObject->waferTypeL_;
0135   }
0136 }
0137
0138 std::vector<int32_t> HeterogeneousHGCalHEFConditionsWrapper::select_pointer_i_(const HGCalParameters* cpuObject,
0139                                                                                const unsigned int& item) const {
0140   switch (item) {
0141     case 4:
0142       return cpuObject->waferTypeL_;
0143     default:
0144       edm::LogError("HeterogeneousHGCalHEFConditionsWrapper") << "select_pointer_i(non-heterogeneous): no item.";
0145       return cpuObject->waferTypeL_;
0146   }
0147 }
0148
0149 hgcal_conditions::HeterogeneousHEFConditionsESProduct const*
0150 HeterogeneousHGCalHEFConditionsWrapper::getHeterogeneousConditionsESProductAsync(cudaStream_t stream) const {
0151   // cms::cuda::ESProduct<T> essentially holds an array of GPUData objects,
0152   // one per device. If the data have already been transferred to the
0153   // current device (or the transfer has been queued), the helper just
0154   // returns a reference to that GPUData object. Otherwise, i.e. data are
0155   // not yet on the current device, the helper calls the lambda to do the
0156   // necessary memory allocations and to queue the transfers.
0157   auto const& data = gpuData_.dataForCurrentDeviceAsync(stream, [this](GPUData& data, cudaStream_t stream) {
0158     // Allocate the payload object on pinned host memory.
0159     cudaCheck(cudaMallocHost(&data.host, sizeof(hgcal_conditions::HeterogeneousHEFConditionsESProduct)));
0160     // Allocate the payload array(s) on device memory.
0161     cudaCheck(cudaMalloc(&(data.host->params.cellFineX_), chunk_params_));
0162
0163     // Complete the host-side information on the payload
0164
0165     //(set the pointers of the parameters)
0166     size_t sdouble = sizeof(double);
0167     for (unsigned int j = 0; j < this->sizes_params_.size() - 1; ++j) {
0168       if (cpar::typesHEF[j] == cpar::HeterogeneousHGCalHEFParametersType::Double and
0169           cpar::typesHEF[j + 1] == cpar::HeterogeneousHGCalHEFParametersType::Double)
0170         select_pointer_d_(&(data.host->params), j + 1) =
0171             select_pointer_d_(&(data.host->params), j) + (this->sizes_params_[j] / sdouble);
0172       else if (cpar::typesHEF[j] == cpar::HeterogeneousHGCalHEFParametersType::Double and
0173                cpar::typesHEF[j + 1] == cpar::HeterogeneousHGCalHEFParametersType::Int32_t)
0174         select_pointer_i_(&(data.host->params), j + 1) =
0175             reinterpret_cast<int32_t*>(select_pointer_d_(&(data.host->params), j) + (this->sizes_params_[j] / sdouble));
0176       else
0177         throw cms::Exception("HeterogeneousHGCalHEFConditionsWrapper")
0178             << "compare this functions' logic with hgcal_conditions::parameters::typesHEF";
0179     }
0180
0181     // Allocate the payload object on the device memory.
0182     cudaCheck(cudaMalloc(&data.device, sizeof(hgcal_conditions::HeterogeneousHEFConditionsESProduct)));
0183     // Transfer the payload, first the array(s) ...
0184     cudaCheck(cudaMemcpyAsync(
0185         data.host->params.cellFineX_, this->params_.cellFineX_, chunk_params_, cudaMemcpyHostToDevice, stream));
0186
0187     // ... and then the payload object
0188     cudaCheck(cudaMemcpyAsync(data.device,
0189                               data.host,
0190                               sizeof(hgcal_conditions::HeterogeneousHEFConditionsESProduct),
0191                               cudaMemcpyHostToDevice,
0192                               stream));
0193   });  //gpuData_.dataForCurrentDeviceAsync
0194
0195   // Returns the payload object on the memory of the current device
0196   return data.device;
0197 }
0198
0199 // Destructor frees all member pointers
0200 HeterogeneousHGCalHEFConditionsWrapper::GPUData::~GPUData() {
0201   if (host != nullptr) {
0202     cudaCheck(cudaFree(host->params.cellFineX_));
0203     cudaCheck(cudaFreeHost(host));
0204   }
0205   cudaCheck(cudaFree(device));
0206 }