Portable/interface/PortableDeviceCollection.h

0001 #ifndef DataFormats_Portable_interface_PortableDeviceCollection_h
0002 #define DataFormats_Portable_interface_PortableDeviceCollection_h
0003
0004 #include <cassert>
0005 #include <optional>
0006 #include <type_traits>
0007
0008 #include <alpaka/alpaka.hpp>
0009
0010 #include "DataFormats/Common/interface/Uninitialized.h"
0011 #include "DataFormats/Portable/interface/PortableCollectionCommon.h"
0012 #include "HeterogeneousCore/AlpakaInterface/interface/config.h"
0013 #include "HeterogeneousCore/AlpakaInterface/interface/memory.h"
0014
0015 // generic SoA-based product in device memory
0016 template <typename T, typename TDev, typename = std::enable_if_t<alpaka::isDevice<TDev>>>
0017 class PortableDeviceCollection {
0018   static_assert(not std::is_same_v<TDev, alpaka_common::DevHost>,
0019                 "Use PortableHostCollection<T> instead of PortableDeviceCollection<T, DevHost>");
0020
0021 public:
0022   using Layout = T;
0023   using View = typename Layout::View;
0024   using ConstView = typename Layout::ConstView;
0025   using Descriptor = typename Layout::Descriptor;
0026   using ConstDescriptor = typename Layout::ConstDescriptor;
0027   using Buffer = cms::alpakatools::device_buffer<TDev, std::byte[]>;
0028   using ConstBuffer = cms::alpakatools::const_device_buffer<TDev, std::byte[]>;
0029
0030   PortableDeviceCollection() = delete;
0031
0032   explicit PortableDeviceCollection(edm::Uninitialized) noexcept {}
0033
0034   PortableDeviceCollection(int32_t elements, TDev const& device)
0035       : buffer_{cms::alpakatools::make_device_buffer<std::byte[]>(device, Layout::computeDataSize(elements))},
0036         layout_{buffer_->data(), elements},
0037         view_{layout_} {
0038     // Alpaka set to a default alignment of 128 bytes defining ALPAKA_DEFAULT_HOST_MEMORY_ALIGNMENT=128
0039     assert(reinterpret_cast<uintptr_t>(buffer_->data()) % Layout::alignment == 0);
0040   }
0041
0042   template <typename TQueue, typename = std::enable_if_t<alpaka::isQueue<TQueue>>>
0043   PortableDeviceCollection(int32_t elements, TQueue const& queue)
0044       : buffer_{cms::alpakatools::make_device_buffer<std::byte[]>(queue, Layout::computeDataSize(elements))},
0045         layout_{buffer_->data(), elements},
0046         view_{layout_} {
0047     // Alpaka set to a default alignment of 128 bytes defining ALPAKA_DEFAULT_HOST_MEMORY_ALIGNMENT=128
0048     assert(reinterpret_cast<uintptr_t>(buffer_->data()) % Layout::alignment == 0);
0049   }
0050
0051   // non-copyable
0052   PortableDeviceCollection(PortableDeviceCollection const&) = delete;
0053   PortableDeviceCollection& operator=(PortableDeviceCollection const&) = delete;
0054
0055   // movable
0056   PortableDeviceCollection(PortableDeviceCollection&&) = default;
0057   PortableDeviceCollection& operator=(PortableDeviceCollection&&) = default;
0058
0059   // default destructor
0060   ~PortableDeviceCollection() = default;
0061
0062   // access the View
0063   View& view() { return view_; }
0064   ConstView const& view() const { return view_; }
0065   ConstView const& const_view() const { return view_; }
0066
0067   View& operator*() { return view_; }
0068   ConstView const& operator*() const { return view_; }
0069
0070   View* operator->() { return &view_; }
0071   ConstView const* operator->() const { return &view_; }
0072
0073   // access the Buffer
0074   Buffer buffer() { return *buffer_; }
0075   ConstBuffer buffer() const { return *buffer_; }
0076   ConstBuffer const_buffer() const { return *buffer_; }
0077
0078   // erases the data in the Buffer by writing zeros (bytes containing '\0') to it
0079   template <typename TQueue, typename = std::enable_if_t<alpaka::isQueue<TQueue>>>
0080   void zeroInitialise(TQueue&& queue) {
0081     alpaka::memset(std::forward<TQueue>(queue), *buffer_, 0x00);
0082   }
0083
0084   // Copy column by column heterogeneously for device to host/device data transfer.
0085   template <typename TQueue>
0086   void deepCopy(ConstView const& view, TQueue& queue) {
0087     ConstDescriptor desc{view};
0088     Descriptor desc_{view_};
0089     _deepCopy<0>(desc_, desc, queue);
0090   }
0091
0092 private:
0093   // Helper function implementing the recursive deep copy
0094   template <int I, typename TQueue>
0095   void _deepCopy(Descriptor& dest, ConstDescriptor const& src, TQueue& queue) {
0096     if constexpr (I < ConstDescriptor::num_cols) {
0097       assert(std::get<I>(dest.buff).size_bytes() == std::get<I>(src.buff).size_bytes());
0098       alpaka::memcpy(
0099           queue,
0100           alpaka::createView(alpaka::getDev(queue), std::get<I>(dest.buff).data(), std::get<I>(dest.buff).size()),
0101           alpaka::createView(alpaka::getDev(queue), std::get<I>(src.buff).data(), std::get<I>(src.buff).size()));
0102       _deepCopy<I + 1>(dest, src, queue);
0103     }
0104   }
0105
0106   // Data members
0107   std::optional<Buffer> buffer_;  //!
0108   Layout layout_;                 //
0109   View view_;                     //!
0110 };
0111
0112 // generic SoA-based product in device memory
0113 template <typename TDev, typename T0, typename... Args>
0114 class PortableDeviceMultiCollection {
0115   //static_assert(alpaka::isDevice<TDev>);
0116   static_assert(not std::is_same_v<TDev, alpaka_common::DevHost>,
0117                 "Use PortableHostCollection<T> instead of PortableDeviceCollection<T, DevHost>");
0118
0119   template <typename T>
0120   static constexpr std::size_t count_t_ = portablecollection::typeCount<T, T0, Args...>;
0121
0122   template <typename T>
0123   static constexpr std::size_t index_t_ = portablecollection::typeIndex<T, T0, Args...>;
0124
0125   static constexpr std::size_t members_ = sizeof...(Args) + 1;
0126
0127 public:
0128   using Buffer = cms::alpakatools::device_buffer<TDev, std::byte[]>;
0129   using ConstBuffer = cms::alpakatools::const_device_buffer<TDev, std::byte[]>;
0130   using Implementation = portablecollection::CollectionImpl<0, T0, Args...>;
0131
0132   using SizesArray = std::array<int32_t, members_>;
0133
0134   template <std::size_t Idx = 0>
0135   using Layout = portablecollection::TypeResolver<Idx, T0, Args...>;
0136
0137   //template <std::size_t Idx = 0>
0138   //using View = typename Layout<Idx>::View;
0139   // Workaround for flaky expansion of tempaltes by nvcc (expanding with "Args" instead of "Args...
0140   template <std::size_t Idx = 0UL>
0141   using View = typename std::tuple_element<Idx, std::tuple<T0, Args...>>::type::View;
0142
0143   //template <std::size_t Idx = 0>
0144   //using ConstView = typename Layout<Idx>::ConstView;
0145   // Workaround for flaky expansion of tempaltes by nvcc (expanding with "Args" instead of "Args..."
0146   template <std::size_t Idx = 0UL>
0147   using ConstView = typename std::tuple_element<Idx, std::tuple<T0, Args...>>::type::ConstView;
0148
0149 private:
0150   template <std::size_t Idx>
0151   using Leaf = portablecollection::CollectionLeaf<Idx, Layout<Idx>>;
0152
0153   template <std::size_t Idx>
0154   Leaf<Idx>& get() {
0155     return static_cast<Leaf<Idx>&>(impl_);
0156   }
0157
0158   template <std::size_t Idx>
0159   Leaf<Idx> const& get() const {
0160     return static_cast<Leaf<Idx> const&>(impl_);
0161   }
0162
0163   template <typename T>
0164   Leaf<index_t_<T>>& get() {
0165     return static_cast<Leaf<index_t_<T>>&>(impl_);
0166   }
0167
0168   template <typename T>
0169   Leaf<index_t_<T>> const& get() const {
0170     return static_cast<Leaf<index_t_<T>> const&>(impl_);
0171   }
0172
0173 public:
0174   PortableDeviceMultiCollection() = delete;
0175
0176   explicit PortableDeviceMultiCollection(edm::Uninitialized) noexcept {};
0177
0178   PortableDeviceMultiCollection(int32_t elements, TDev const& device)
0179       : buffer_{cms::alpakatools::make_device_buffer<std::byte[]>(device, Layout<>::computeDataSize(elements))},
0180         impl_{buffer_->data(), elements} {
0181     // Alpaka set to a default alignment of 128 bytes defining ALPAKA_DEFAULT_HOST_MEMORY_ALIGNMENT=128
0182     assert(reinterpret_cast<uintptr_t>(buffer_->data()) % Layout<>::alignment == 0);
0183     static_assert(members_ == 1);
0184   }
0185
0186   template <typename TQueue, typename = std::enable_if_t<alpaka::isQueue<TQueue>>>
0187   PortableDeviceMultiCollection(int32_t elements, TQueue const& queue)
0188       : buffer_{cms::alpakatools::make_device_buffer<std::byte[]>(queue, Layout<>::computeDataSize(elements))},
0189         impl_{buffer_->data(), elements} {
0190     // Alpaka set to a default alignment of 128 bytes defining ALPAKA_DEFAULT_HOST_MEMORY_ALIGNMENT=128
0191     assert(reinterpret_cast<uintptr_t>(buffer_->data()) % Layout<>::alignment == 0);
0192     static_assert(members_ == 1);
0193   }
0194
0195   static int32_t computeDataSize(const SizesArray& sizes) {
0196     int32_t ret = 0;
0197     portablecollection::constexpr_for<0, members_>(
0198         [&sizes, &ret](auto i) { ret += Layout<i>::computeDataSize(sizes[i]); });
0199     return ret;
0200   }
0201
0202   PortableDeviceMultiCollection(const SizesArray& sizes, TDev const& device)
0203       // allocate device memory
0204       : buffer_{cms::alpakatools::make_device_buffer<std::byte[]>(device, computeDataSize(sizes))},
0205         impl_{buffer_->data(), sizes} {
0206     portablecollection::constexpr_for<0, members_>(
0207         [&](auto i) { assert(reinterpret_cast<uintptr_t>(buffer_->data()) % Layout<i>::alignment == 0); });
0208     constexpr auto alignment = Layout<0>::alignment;
0209     portablecollection::constexpr_for<1, members_>(
0210         [&alignment](auto i) { static_assert(alignment == Layout<i>::alignment); });
0211   }
0212
0213   template <typename TQueue, typename = std::enable_if_t<alpaka::isQueue<TQueue>>>
0214   PortableDeviceMultiCollection(const SizesArray& sizes, TQueue const& queue)
0215       // allocate device memory asynchronously on the given work queue
0216       : buffer_{cms::alpakatools::make_device_buffer<std::byte[]>(queue, computeDataSize(sizes))},
0217         impl_{buffer_->data(), sizes} {
0218     portablecollection::constexpr_for<0, members_>(
0219         [&](auto i) { assert(reinterpret_cast<uintptr_t>(buffer_->data()) % Layout<i>::alignment == 0); });
0220     constexpr auto alignment = Layout<0>::alignment;
0221     portablecollection::constexpr_for<1, members_>(
0222         [&alignment](auto i) { static_assert(alignment == Layout<i>::alignment); });
0223   }
0224
0225   // non-copyable
0226   PortableDeviceMultiCollection(PortableDeviceMultiCollection const&) = delete;
0227   PortableDeviceMultiCollection& operator=(PortableDeviceMultiCollection const&) = delete;
0228
0229   // movable
0230   PortableDeviceMultiCollection(PortableDeviceMultiCollection&&) = default;
0231   PortableDeviceMultiCollection& operator=(PortableDeviceMultiCollection&&) = default;
0232
0233   // default destructor
0234   ~PortableDeviceMultiCollection() = default;
0235
0236   // access the View by index
0237   template <std::size_t Idx = 0, typename = std::enable_if_t<(members_ > Idx)>>
0238   View<Idx>& view() {
0239     return get<Idx>().view_;
0240   }
0241
0242   template <std::size_t Idx = 0, typename = std::enable_if_t<(members_ > Idx)>>
0243   ConstView<Idx> const& view() const {
0244     return get<Idx>().view_;
0245   }
0246
0247   template <std::size_t Idx = 0, typename = std::enable_if_t<(members_ > Idx)>>
0248   ConstView<Idx> const& const_view() const {
0249     return get<Idx>().view_;
0250   }
0251
0252   template <std::size_t Idx = 0, typename = std::enable_if_t<(members_ > Idx)>>
0253   View<Idx>& operator*() {
0254     return get<Idx>().view_;
0255   }
0256
0257   template <std::size_t Idx = 0, typename = std::enable_if_t<(members_ > Idx)>>
0258   ConstView<Idx> const& operator*() const {
0259     return get<Idx>().view_;
0260   }
0261
0262   template <std::size_t Idx = 0, typename = std::enable_if_t<(members_ > Idx)>>
0263   View<Idx>* operator->() {
0264     return &get<Idx>().view_;
0265   }
0266
0267   template <std::size_t Idx = 0, typename = std::enable_if_t<(members_ > Idx)>>
0268   ConstView<Idx> const* operator->() const {
0269     return &get<Idx>().view_;
0270   }
0271
0272   // access the View by type
0273   template <typename T>
0274   typename T::View& view() {
0275     return get<T>().view_;
0276   }
0277
0278   template <typename T>
0279   typename T::ConstView const& view() const {
0280     return get<T>().view_;
0281   }
0282
0283   template <typename T>
0284   typename T::ConstView const& const_view() const {
0285     return get<T>().view_;
0286   }
0287
0288   template <typename T>
0289   typename T::View& operator*() {
0290     return get<T>().view_;
0291   }
0292
0293   template <typename T>
0294   typename T::ConstView const& operator*() const {
0295     return get<T>().view_;
0296   }
0297
0298   template <typename T>
0299   typename T::View* operator->() {
0300     return &get<T>().view_;
0301   }
0302
0303   template <typename T>
0304   typename T::ConstView const* operator->() const {
0305     return &get<T>().view_;
0306   }
0307
0308   // access the Buffer
0309   Buffer buffer() { return *buffer_; }
0310   ConstBuffer buffer() const { return *buffer_; }
0311   ConstBuffer const_buffer() const { return *buffer_; }
0312
0313   // erases the data in the Buffer by writing zeros (bytes containing '\0') to it
0314   template <typename TQueue, typename = std::enable_if_t<alpaka::isQueue<TQueue>>>
0315   void zeroInitialise(TQueue&& queue) {
0316     alpaka::memset(std::forward<TQueue>(queue), *buffer_, 0x00);
0317   }
0318
0319   // extract the sizes array
0320   SizesArray sizes() const {
0321     SizesArray ret;
0322     portablecollection::constexpr_for<0, members_>([&](auto i) { ret[i] = get<i>().layout_.metadata().size(); });
0323     return ret;
0324   }
0325
0326 private:
0327   std::optional<Buffer> buffer_;  //!
0328   Implementation impl_;           // (serialized: this is where the layouts live)
0329 };
0330
0331 #endif  // DataFormats_Portable_interface_PortableDeviceCollection_h