File indexing completed on 2024-06-06 04:26:43
0001 #ifndef DataFormats_SoATemplate_interface_SoACommon_h
0002 #define DataFormats_SoATemplate_interface_SoACommon_h
0003
0004
0005
0006
0007
0008 #include <cstdint>
0009 #include <cassert>
0010 #include <ostream>
0011 #include <tuple>
0012 #include <type_traits>
0013
0014 #include <boost/preprocessor.hpp>
0015
0016 #include "FWCore/Utilities/interface/typedefs.h"
0017
0018
0019 #if defined(__CUDACC__) || defined(__HIPCC__)
0020 #define SOA_HOST_ONLY __host__
0021 #define SOA_DEVICE_ONLY __device__
0022 #define SOA_HOST_DEVICE __host__ __device__
0023 #define SOA_INLINE __forceinline__
0024 #else
0025 #define SOA_HOST_ONLY
0026 #define SOA_DEVICE_ONLY
0027 #define SOA_HOST_DEVICE
0028 #define SOA_INLINE inline __attribute__((always_inline))
0029 #endif
0030
0031
0032 #if defined(__CUDACC__) && defined(__CUDA_ARCH__)
0033 #define SOA_THROW_OUT_OF_RANGE(A) \
0034 { \
0035 printf("%s\n", (A)); \
0036 __trap(); \
0037 }
0038 #elif defined(__HIPCC__) && defined(__HIP_DEVICE_COMPILE__)
0039 #define SOA_THROW_OUT_OF_RANGE(A) \
0040 { \
0041 printf("%s\n", (A)); \
0042 abort(); \
0043 }
0044 #else
0045 #define SOA_THROW_OUT_OF_RANGE(A) \
0046 { throw std::out_of_range(A); }
0047 #endif
0048
0049
0050 #define _VALUE_TYPE_SCALAR 0
0051 #define _VALUE_TYPE_COLUMN 1
0052 #define _VALUE_TYPE_EIGEN_COLUMN 2
0053
0054
0055
0056
0057 #define CMS_SOA_BYTE_SIZE_TYPE std::size_t
0058
0059 namespace cms::soa {
0060
0061
0062 using size_type = cms_int32_t;
0063
0064 using byte_size_type = CMS_SOA_BYTE_SIZE_TYPE;
0065
0066 enum class SoAColumnType {
0067 scalar = _VALUE_TYPE_SCALAR,
0068 column = _VALUE_TYPE_COLUMN,
0069 eigen = _VALUE_TYPE_EIGEN_COLUMN
0070 };
0071
0072 namespace RestrictQualify {
0073 constexpr bool enabled = true;
0074 constexpr bool disabled = false;
0075 constexpr bool Default = enabled;
0076 }
0077
0078 namespace RangeChecking {
0079 constexpr bool enabled = true;
0080 constexpr bool disabled = false;
0081 constexpr bool Default = enabled;
0082 }
0083
0084 template <typename T, bool RESTRICT_QUALIFY>
0085 struct add_restrict {};
0086
0087 template <typename T>
0088 struct add_restrict<T, RestrictQualify::enabled> {
0089 using Value = T;
0090 using Pointer = T* __restrict__;
0091 using Reference = T& __restrict__;
0092 using ConstValue = const T;
0093 using PointerToConst = const T* __restrict__;
0094 using ReferenceToConst = const T& __restrict__;
0095 };
0096
0097 template <typename T>
0098 struct add_restrict<T, RestrictQualify::disabled> {
0099 using Value = T;
0100 using Pointer = T*;
0101 using Reference = T&;
0102 using ConstValue = const T;
0103 using PointerToConst = const T*;
0104 using ReferenceToConst = const T&;
0105 };
0106
0107
0108 template <SoAColumnType COLUMN_TYPE, typename T>
0109 struct SoAConstParametersImpl;
0110
0111 template <SoAColumnType COLUMN_TYPE, typename T>
0112 struct SoAParametersImpl;
0113
0114
0115 template <SoAColumnType COLUMN_TYPE, typename T>
0116 struct SoAConstParametersImpl {
0117 static constexpr SoAColumnType columnType = COLUMN_TYPE;
0118
0119 using ValueType = T;
0120 using ScalarType = T;
0121 using TupleOrPointerType = const ValueType*;
0122
0123
0124 SoAConstParametersImpl() = default;
0125
0126
0127 SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(ValueType const* addr) : addr_(addr) {}
0128
0129
0130 SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(SoAParametersImpl<columnType, ValueType> const& o)
0131 : addr_{o.addr_} {}
0132
0133 static constexpr bool checkAlignment(ValueType* addr, byte_size_type alignment) {
0134 return reinterpret_cast<intptr_t>(addr) % alignment;
0135 }
0136
0137 TupleOrPointerType tupleOrPointer() { return addr_; }
0138
0139 public:
0140
0141 ValueType const* addr_ = nullptr;
0142 };
0143
0144
0145 template <typename T>
0146 struct SoAConstParametersImpl<SoAColumnType::eigen, T> {
0147 static constexpr SoAColumnType columnType = SoAColumnType::eigen;
0148
0149 using ValueType = T;
0150 using ScalarType = typename T::Scalar;
0151 using TupleOrPointerType = std::tuple<ScalarType*, byte_size_type>;
0152
0153
0154 SoAConstParametersImpl() = default;
0155
0156
0157 SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(ScalarType const* addr, byte_size_type stride)
0158 : addr_(addr), stride_(stride) {}
0159
0160
0161 SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(TupleOrPointerType const& tuple)
0162 : addr_(std::get<0>(tuple)), stride_(std::get<1>(tuple)) {}
0163
0164
0165 SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(SoAParametersImpl<columnType, ValueType> const& o)
0166 : addr_{o.addr_}, stride_{o.stride_} {}
0167
0168 static constexpr bool checkAlignment(TupleOrPointerType const& tuple, byte_size_type alignment) {
0169 const auto& [addr, stride] = tuple;
0170 return reinterpret_cast<intptr_t>(addr) % alignment;
0171 }
0172
0173 TupleOrPointerType tupleOrPointer() { return {addr_, stride_}; }
0174
0175 public:
0176
0177 ScalarType const* addr_ = nullptr;
0178 byte_size_type stride_ = 0;
0179 };
0180
0181
0182 template <SoAColumnType COLUMN_TYPE>
0183 struct SoAConstParameters_ColumnType {
0184 template <typename T>
0185 using DataType = SoAConstParametersImpl<COLUMN_TYPE, T>;
0186 };
0187
0188
0189 template <SoAColumnType COLUMN_TYPE, typename T>
0190 struct SoAParametersImpl {
0191 static constexpr SoAColumnType columnType = COLUMN_TYPE;
0192
0193 using ValueType = T;
0194 using ScalarType = T;
0195 using TupleOrPointerType = ValueType*;
0196
0197 using ConstType = SoAConstParametersImpl<columnType, ValueType>;
0198 friend ConstType;
0199
0200
0201 SoAParametersImpl() = default;
0202
0203
0204 SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl(ValueType* addr) : addr_(addr) {}
0205
0206 static constexpr bool checkAlignment(ValueType* addr, byte_size_type alignment) {
0207 return reinterpret_cast<intptr_t>(addr) % alignment;
0208 }
0209
0210 TupleOrPointerType tupleOrPointer() { return addr_; }
0211
0212 public:
0213
0214 ValueType* addr_ = nullptr;
0215 };
0216
0217
0218 template <typename T>
0219 struct SoAParametersImpl<SoAColumnType::eigen, T> {
0220 static constexpr SoAColumnType columnType = SoAColumnType::eigen;
0221
0222 using ValueType = T;
0223 using ScalarType = typename T::Scalar;
0224 using TupleOrPointerType = std::tuple<ScalarType*, byte_size_type>;
0225
0226 using ConstType = SoAConstParametersImpl<columnType, ValueType>;
0227 friend ConstType;
0228
0229
0230 SoAParametersImpl() = default;
0231
0232
0233 SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl(ScalarType* addr, byte_size_type stride)
0234 : addr_(addr), stride_(stride) {}
0235
0236
0237 SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl(TupleOrPointerType const& tuple)
0238 : addr_(std::get<0>(tuple)), stride_(std::get<1>(tuple)) {}
0239
0240 static constexpr bool checkAlignment(TupleOrPointerType const& tuple, byte_size_type alignment) {
0241 const auto& [addr, stride] = tuple;
0242 return reinterpret_cast<intptr_t>(addr) % alignment;
0243 }
0244
0245 TupleOrPointerType tupleOrPointer() { return {addr_, stride_}; }
0246
0247 public:
0248
0249 ScalarType* addr_ = nullptr;
0250 byte_size_type stride_ = 0;
0251 };
0252
0253
0254 template <SoAColumnType COLUMN_TYPE>
0255 struct SoAParameters_ColumnType {
0256 template <typename T>
0257 using DataType = SoAParametersImpl<COLUMN_TYPE, T>;
0258 };
0259
0260
0261 namespace {
0262 template <typename T>
0263 constexpr inline std::remove_const_t<T>* non_const_ptr(T* p) {
0264 return const_cast<std::remove_const_t<T>*>(p);
0265 }
0266 }
0267
0268 template <SoAColumnType COLUMN_TYPE, typename T>
0269 SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl<COLUMN_TYPE, T> const_cast_SoAParametersImpl(
0270 SoAConstParametersImpl<COLUMN_TYPE, T> const& o) {
0271 return SoAParametersImpl<COLUMN_TYPE, T>{non_const_ptr(o.addr_)};
0272 }
0273
0274 template <typename T>
0275 SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl<SoAColumnType::eigen, T> const_cast_SoAParametersImpl(
0276 SoAConstParametersImpl<SoAColumnType::eigen, T> const& o) {
0277 return SoAParametersImpl<SoAColumnType::eigen, T>{non_const_ptr(o.addr_), o.stride_};
0278 }
0279
0280
0281
0282
0283 template <SoAColumnType COLUMN_TYPE,
0284 typename T,
0285 byte_size_type ALIGNMENT,
0286 bool RESTRICT_QUALIFY = RestrictQualify::disabled>
0287 class SoAValue {
0288
0289 static_assert(COLUMN_TYPE != SoAColumnType::eigen);
0290
0291 public:
0292 using Restr = add_restrict<T, RESTRICT_QUALIFY>;
0293 using Val = typename Restr::Value;
0294 using Ptr = typename Restr::Pointer;
0295 using Ref = typename Restr::Reference;
0296 using PtrToConst = typename Restr::PointerToConst;
0297 using RefToConst = typename Restr::ReferenceToConst;
0298
0299 SOA_HOST_DEVICE SOA_INLINE SoAValue(size_type i, T* col) : idx_(i), col_(col) {}
0300
0301 SOA_HOST_DEVICE SOA_INLINE SoAValue(size_type i, SoAParametersImpl<COLUMN_TYPE, T> params)
0302 : idx_(i), col_(params.addr_) {}
0303
0304 SOA_HOST_DEVICE SOA_INLINE Ref operator()() {
0305
0306 Ptr col = col_;
0307 return col[idx_];
0308 }
0309
0310 SOA_HOST_DEVICE SOA_INLINE RefToConst operator()() const {
0311
0312 PtrToConst col = col_;
0313 return col[idx_];
0314 }
0315
0316 SOA_HOST_DEVICE SOA_INLINE Ptr operator&() { return &col_[idx_]; }
0317
0318 SOA_HOST_DEVICE SOA_INLINE PtrToConst operator&() const { return &col_[idx_]; }
0319
0320
0321
0322
0323
0324
0325
0326
0327
0328
0329
0330
0331
0332
0333
0334
0335
0336
0337
0338
0339
0340
0341 using valueType = Val;
0342
0343 static constexpr auto valueSize = sizeof(T);
0344
0345 private:
0346 size_type idx_;
0347 T* col_;
0348 };
0349
0350
0351 #ifdef EIGEN_WORLD_VERSION
0352
0353 template <class C, byte_size_type ALIGNMENT, bool RESTRICT_QUALIFY>
0354 class SoAValue<SoAColumnType::eigen, C, ALIGNMENT, RESTRICT_QUALIFY> {
0355 public:
0356 using Type = C;
0357 using MapType = Eigen::Map<C, 0, Eigen::InnerStride<Eigen::Dynamic>>;
0358 using CMapType = const Eigen::Map<const C, 0, Eigen::InnerStride<Eigen::Dynamic>>;
0359 using Restr = add_restrict<typename C::Scalar, RESTRICT_QUALIFY>;
0360 using Val = typename Restr::Value;
0361 using Ptr = typename Restr::Pointer;
0362 using Ref = typename Restr::Reference;
0363 using PtrToConst = typename Restr::PointerToConst;
0364 using RefToConst = typename Restr::ReferenceToConst;
0365
0366 SOA_HOST_DEVICE SOA_INLINE SoAValue(size_type i, typename C::Scalar* col, byte_size_type stride)
0367 : val_(col + i, C::RowsAtCompileTime, C::ColsAtCompileTime, Eigen::InnerStride<Eigen::Dynamic>(stride)),
0368 crCol_(col),
0369 cVal_(crCol_ + i, C::RowsAtCompileTime, C::ColsAtCompileTime, Eigen::InnerStride<Eigen::Dynamic>(stride)),
0370 stride_(stride) {}
0371
0372 SOA_HOST_DEVICE SOA_INLINE SoAValue(size_type i, SoAParametersImpl<SoAColumnType::eigen, C> params)
0373 : val_(params.addr_ + i,
0374 C::RowsAtCompileTime,
0375 C::ColsAtCompileTime,
0376 Eigen::InnerStride<Eigen::Dynamic>(params.stride_)),
0377 crCol_(params.addr_),
0378 cVal_(crCol_ + i,
0379 C::RowsAtCompileTime,
0380 C::ColsAtCompileTime,
0381 Eigen::InnerStride<Eigen::Dynamic>(params.stride_)),
0382 stride_(params.stride_) {}
0383
0384 SOA_HOST_DEVICE SOA_INLINE MapType& operator()() { return val_; }
0385
0386 SOA_HOST_DEVICE SOA_INLINE const CMapType& operator()() const { return cVal_; }
0387
0388 SOA_HOST_DEVICE SOA_INLINE operator C() { return val_; }
0389
0390 SOA_HOST_DEVICE SOA_INLINE operator const C() const { return cVal_; }
0391
0392 SOA_HOST_DEVICE SOA_INLINE C* operator&() { return &val_; }
0393
0394 SOA_HOST_DEVICE SOA_INLINE const C* operator&() const { return &cVal_; }
0395
0396 template <class C2>
0397 SOA_HOST_DEVICE SOA_INLINE MapType& operator=(const C2& v) {
0398 return val_ = v;
0399 }
0400
0401 using ValueType = typename C::Scalar;
0402 static constexpr auto valueSize = sizeof(typename C::Scalar);
0403 SOA_HOST_DEVICE SOA_INLINE byte_size_type stride() const { return stride_; }
0404
0405 private:
0406 MapType val_;
0407 const Ptr crCol_;
0408 CMapType cVal_;
0409 byte_size_type stride_;
0410 };
0411 #else
0412
0413 template <class C, byte_size_type ALIGNMENT, bool RESTRICT_QUALIFY>
0414 class SoAValue<SoAColumnType::eigen, C, ALIGNMENT, RESTRICT_QUALIFY> {
0415 static_assert(!sizeof(C),
0416 "Eigen/Core should be pre-included before the SoA headers to enable support for Eigen columns.");
0417 };
0418 #endif
0419
0420
0421 template <SoAColumnType COLUMN_TYPE,
0422 typename T,
0423 byte_size_type ALIGNMENT,
0424 bool RESTRICT_QUALIFY = RestrictQualify::disabled>
0425 class SoAConstValue {
0426
0427 static_assert(COLUMN_TYPE != SoAColumnType::eigen);
0428
0429 public:
0430 using Restr = add_restrict<T, RESTRICT_QUALIFY>;
0431 using Val = typename Restr::Value;
0432 using Ptr = typename Restr::Pointer;
0433 using Ref = typename Restr::Reference;
0434 using PtrToConst = typename Restr::PointerToConst;
0435 using RefToConst = typename Restr::ReferenceToConst;
0436 using Params = SoAParametersImpl<COLUMN_TYPE, T>;
0437 using ConstParams = SoAConstParametersImpl<COLUMN_TYPE, T>;
0438
0439 SOA_HOST_DEVICE SOA_INLINE SoAConstValue(size_type i, const T* col) : idx_(i), col_(col) {}
0440
0441 SOA_HOST_DEVICE SOA_INLINE SoAConstValue(size_type i, SoAParametersImpl<COLUMN_TYPE, T> params)
0442 : idx_(i), col_(params.addr_) {}
0443
0444 SOA_HOST_DEVICE SOA_INLINE SoAConstValue(size_type i, SoAConstParametersImpl<COLUMN_TYPE, T> params)
0445 : idx_(i), col_(params.addr_) {}
0446
0447 SOA_HOST_DEVICE SOA_INLINE RefToConst operator()() const {
0448
0449 PtrToConst col = col_;
0450 return col[idx_];
0451 }
0452
0453 SOA_HOST_DEVICE SOA_INLINE const T* operator&() const { return &col_[idx_]; }
0454
0455
0456
0457
0458
0459
0460
0461
0462
0463
0464
0465
0466
0467
0468
0469 using valueType = T;
0470 static constexpr auto valueSize = sizeof(T);
0471
0472 private:
0473 size_type idx_;
0474 const T* col_;
0475 };
0476
0477
0478 #ifdef EIGEN_WORLD_VERSION
0479
0480 template <class C, byte_size_type ALIGNMENT, bool RESTRICT_QUALIFY>
0481 class SoAConstValue<SoAColumnType::eigen, C, ALIGNMENT, RESTRICT_QUALIFY> {
0482 public:
0483 using Type = C;
0484 using CMapType = Eigen::Map<const C, 0, Eigen::InnerStride<Eigen::Dynamic>>;
0485 using RefToConst = const CMapType&;
0486 using ConstParams = SoAConstParametersImpl<SoAColumnType::eigen, C>;
0487
0488 SOA_HOST_DEVICE SOA_INLINE SoAConstValue(size_type i, typename C::Scalar* col, byte_size_type stride)
0489 : crCol_(col),
0490 cVal_(crCol_ + i, C::RowsAtCompileTime, C::ColsAtCompileTime, Eigen::InnerStride<Eigen::Dynamic>(stride)),
0491 stride_(stride) {}
0492
0493 SOA_HOST_DEVICE SOA_INLINE SoAConstValue(size_type i, SoAConstParametersImpl<SoAColumnType::eigen, C> params)
0494 : crCol_(params.addr_),
0495 cVal_(crCol_ + i,
0496 C::RowsAtCompileTime,
0497 C::ColsAtCompileTime,
0498 Eigen::InnerStride<Eigen::Dynamic>(params.stride_)),
0499 stride_(params.stride_) {}
0500
0501 SOA_HOST_DEVICE SOA_INLINE const CMapType& operator()() const { return cVal_; }
0502
0503 SOA_HOST_DEVICE SOA_INLINE operator const C() const { return cVal_; }
0504
0505 SOA_HOST_DEVICE SOA_INLINE const C* operator&() const { return &cVal_; }
0506
0507 using ValueType = typename C::Scalar;
0508 static constexpr auto valueSize = sizeof(typename C::Scalar);
0509
0510 SOA_HOST_DEVICE SOA_INLINE byte_size_type stride() const { return stride_; }
0511
0512 private:
0513 const typename C::Scalar* __restrict__ crCol_;
0514 CMapType cVal_;
0515 byte_size_type stride_;
0516 };
0517 #else
0518
0519 template <class C, byte_size_type ALIGNMENT, bool RESTRICT_QUALIFY>
0520 class SoAConstValue<SoAColumnType::eigen, C, ALIGNMENT, RESTRICT_QUALIFY> {
0521 static_assert(!sizeof(C),
0522 "Eigen/Core should be pre-included before the SoA headers to enable support for Eigen columns.");
0523 };
0524 #endif
0525
0526
0527 #ifdef EIGEN_WORLD_VERSION
0528 template <class C>
0529 struct EigenConstMapMaker {
0530 using Type = Eigen::Map<const C, Eigen::AlignmentType::Unaligned, Eigen::InnerStride<Eigen::Dynamic>>;
0531
0532 class DataHolder {
0533 public:
0534 DataHolder(const typename C::Scalar* data) : data_(data) {}
0535
0536 EigenConstMapMaker::Type withStride(byte_size_type stride) {
0537 return EigenConstMapMaker::Type(
0538 data_, C::RowsAtCompileTime, C::ColsAtCompileTime, Eigen::InnerStride<Eigen::Dynamic>(stride));
0539 }
0540
0541 private:
0542 const typename C::Scalar* const data_;
0543 };
0544
0545 static DataHolder withData(const typename C::Scalar* data) { return DataHolder(data); }
0546 };
0547 #else
0548 template <class C>
0549 struct EigenConstMapMaker {
0550
0551 static_assert(!sizeof(C),
0552 "Eigen/Core should be pre-included before the SoA headers to enable support for Eigen columns.");
0553 };
0554 #endif
0555
0556
0557 constexpr inline byte_size_type alignSize(byte_size_type size, byte_size_type alignment) {
0558 return ((size + alignment - 1) / alignment) * alignment;
0559 }
0560
0561 }
0562
0563 #define SOA_SCALAR(TYPE, NAME) (_VALUE_TYPE_SCALAR, TYPE, NAME)
0564 #define SOA_COLUMN(TYPE, NAME) (_VALUE_TYPE_COLUMN, TYPE, NAME)
0565 #define SOA_EIGEN_COLUMN(TYPE, NAME) (_VALUE_TYPE_EIGEN_COLUMN, TYPE, NAME)
0566
0567
0568 #define _ITERATE_ON_ALL_COMMA(MACRO, DATA, ...) \
0569 BOOST_PP_TUPLE_ENUM(BOOST_PP_SEQ_TO_TUPLE(_ITERATE_ON_ALL(MACRO, DATA, __VA_ARGS__)))
0570
0571
0572 #define _ITERATE_ON_ALL(MACRO, DATA, ...) BOOST_PP_SEQ_FOR_EACH(MACRO, DATA, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__))
0573
0574
0575 #define _SWITCH_ON_TYPE(VALUE_TYPE, IF_SCALAR, IF_COLUMN, IF_EIGEN_COLUMN) \
0576 BOOST_PP_IF( \
0577 BOOST_PP_EQUAL(VALUE_TYPE, _VALUE_TYPE_SCALAR), \
0578 IF_SCALAR, \
0579 BOOST_PP_IF( \
0580 BOOST_PP_EQUAL(VALUE_TYPE, _VALUE_TYPE_COLUMN), \
0581 IF_COLUMN, \
0582 BOOST_PP_IF(BOOST_PP_EQUAL(VALUE_TYPE, _VALUE_TYPE_EIGEN_COLUMN), IF_EIGEN_COLUMN, BOOST_PP_EMPTY())))
0583
0584 namespace cms::soa {
0585
0586
0587 enum class SoAAccessType : bool { mutableAccess, constAccess };
0588
0589 template <typename, SoAColumnType, SoAAccessType, byte_size_type, bool>
0590 struct SoAColumnAccessorsImpl {};
0591
0592
0593
0594
0595
0596
0597 template <typename T, byte_size_type alignment, bool restrictQualify>
0598 struct SoAColumnAccessorsImpl<T, SoAColumnType::column, SoAAccessType::mutableAccess, alignment, restrictQualify> {
0599 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAParametersImpl<SoAColumnType::column, T>& params)
0600 : params_(params) {}
0601 SOA_HOST_DEVICE SOA_INLINE T* operator()() { return params_.addr_; }
0602 using NoParamReturnType = T*;
0603 using ParamReturnType = T&;
0604 SOA_HOST_DEVICE SOA_INLINE T& operator()(size_type index) { return params_.addr_[index]; }
0605
0606 private:
0607 SoAParametersImpl<SoAColumnType::column, T> params_;
0608 };
0609
0610
0611 template <typename T, byte_size_type alignment, bool restrictQualify>
0612 struct SoAColumnAccessorsImpl<T, SoAColumnType::column, SoAAccessType::constAccess, alignment, restrictQualify> {
0613 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAConstParametersImpl<SoAColumnType::column, T>& params)
0614 : params_(params) {}
0615 SOA_HOST_DEVICE SOA_INLINE const T* operator()() const { return params_.addr_; }
0616 using NoParamReturnType = const T*;
0617 using ParamReturnType = const T&;
0618 SOA_HOST_DEVICE SOA_INLINE T const& operator()(size_type index) const { return params_.addr_[index]; }
0619
0620 private:
0621 SoAConstParametersImpl<SoAColumnType::column, T> params_;
0622 };
0623
0624
0625 template <typename T, byte_size_type alignment, bool restrictQualify>
0626 struct SoAColumnAccessorsImpl<T, SoAColumnType::scalar, SoAAccessType::mutableAccess, alignment, restrictQualify> {
0627 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAParametersImpl<SoAColumnType::scalar, T>& params)
0628 : params_(params) {}
0629 SOA_HOST_DEVICE SOA_INLINE T& operator()() { return *params_.addr_; }
0630 using NoParamReturnType = T&;
0631 using ParamReturnType = void;
0632 SOA_HOST_DEVICE SOA_INLINE void operator()(size_type index) const {
0633 assert(false && "Indexed access impossible for SoA scalars.");
0634 }
0635
0636 private:
0637 SoAParametersImpl<SoAColumnType::scalar, T> params_;
0638 };
0639
0640
0641 template <typename T, byte_size_type alignment, bool restrictQualify>
0642 struct SoAColumnAccessorsImpl<T, SoAColumnType::scalar, SoAAccessType::constAccess, alignment, restrictQualify> {
0643 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAConstParametersImpl<SoAColumnType::scalar, T>& params)
0644 : params_(params) {}
0645 SOA_HOST_DEVICE SOA_INLINE T const& operator()() const { return *params_.addr_; }
0646 using NoParamReturnType = T const&;
0647 using ParamReturnType = void;
0648 SOA_HOST_DEVICE SOA_INLINE void operator()(size_type index) const {
0649 assert(false && "Indexed access impossible for SoA scalars.");
0650 }
0651
0652 private:
0653 SoAConstParametersImpl<SoAColumnType::scalar, T> params_;
0654 };
0655
0656
0657 template <typename T, byte_size_type alignment, bool restrictQualify>
0658 struct SoAColumnAccessorsImpl<T, SoAColumnType::eigen, SoAAccessType::mutableAccess, alignment, restrictQualify> {
0659 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAParametersImpl<SoAColumnType::eigen, T>& params)
0660 : params_(params) {}
0661 SOA_HOST_DEVICE SOA_INLINE typename T::Scalar* operator()() { return params_.addr_; }
0662 using NoParamReturnType = typename T::Scalar*;
0663 using ParamReturnType = typename SoAValue<SoAColumnType::eigen, T, alignment, restrictQualify>::MapType;
0664 SOA_HOST_DEVICE SOA_INLINE ParamReturnType operator()(size_type index) {
0665 return SoAValue<SoAColumnType::eigen, T, alignment, restrictQualify>(index, params_)();
0666 }
0667
0668 private:
0669 SoAParametersImpl<SoAColumnType::eigen, T> params_;
0670 };
0671
0672
0673 template <typename T, byte_size_type alignment, bool restrictQualify>
0674 struct SoAColumnAccessorsImpl<T, SoAColumnType::eigen, SoAAccessType::constAccess, alignment, restrictQualify> {
0675 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAConstParametersImpl<SoAColumnType::eigen, T>& params)
0676 : params_(params) {}
0677 SOA_HOST_DEVICE SOA_INLINE typename T::Scalar const* operator()() const { return params_.addr_; }
0678 using NoParamReturnType = typename T::Scalar const*;
0679 using ParamReturnType = typename SoAValue<SoAColumnType::eigen, T, alignment, restrictQualify>::CMapType;
0680 SOA_HOST_DEVICE SOA_INLINE ParamReturnType operator()(size_type index) const {
0681 return SoAConstValue<SoAColumnType::eigen, T, alignment, restrictQualify>(index, params_)();
0682 }
0683
0684 private:
0685 SoAConstParametersImpl<SoAColumnType::eigen, T> params_;
0686 };
0687
0688
0689 template <typename T>
0690 struct SoAAccessors {
0691 template <auto columnType>
0692 struct ColumnType {
0693 template <auto accessType>
0694 struct AccessType {
0695 template <auto alignment>
0696 struct Alignment {
0697 template <auto restrictQualify>
0698 struct RestrictQualifier
0699 : public SoAColumnAccessorsImpl<T, columnType, accessType, alignment, restrictQualify> {
0700 using SoAColumnAccessorsImpl<T, columnType, accessType, alignment, restrictQualify>::SoAColumnAccessorsImpl;
0701 };
0702 };
0703 };
0704 };
0705 };
0706
0707
0708
0709
0710 struct AlignmentEnforcement {
0711 static constexpr bool relaxed = false;
0712 static constexpr bool enforced = true;
0713 };
0714
0715 struct CacheLineSize {
0716 static constexpr byte_size_type NvidiaGPU = 128;
0717 static constexpr byte_size_type IntelCPU = 64;
0718 static constexpr byte_size_type AMDCPU = 64;
0719 static constexpr byte_size_type ARMCPU = 64;
0720 static constexpr byte_size_type defaultSize = NvidiaGPU;
0721 };
0722
0723 }
0724
0725
0726 template <typename SOA,
0727 typename SFINAE =
0728 typename std::enable_if_t<std::is_invocable_v<decltype(&SOA::soaToStreamInternal), SOA&, std::ostream&>>>
0729 SOA_HOST_ONLY std::ostream& operator<<(std::ostream& os, const SOA& soa) {
0730 soa.soaToStreamInternal(os);
0731 return os;
0732 }
0733
0734 #endif