File indexing completed on 2023-03-17 10:51:25
0001 #ifndef DataFormats_SoATemplate_interface_SoACommon_h
0002 #define DataFormats_SoATemplate_interface_SoACommon_h
0003
0004
0005
0006
0007
0008 #include <cstdint>
0009 #include <cassert>
0010 #include <ostream>
0011 #include <tuple>
0012 #include <type_traits>
0013
0014 #include <boost/preprocessor.hpp>
0015
0016 #include "FWCore/Utilities/interface/typedefs.h"
0017
0018
0019 #if defined(__CUDACC__) || defined(__HIPCC__)
0020 #define SOA_HOST_ONLY __host__
0021 #define SOA_DEVICE_ONLY __device__
0022 #define SOA_HOST_DEVICE __host__ __device__
0023 #define SOA_INLINE __forceinline__
0024 #else
0025 #define SOA_HOST_ONLY
0026 #define SOA_DEVICE_ONLY
0027 #define SOA_HOST_DEVICE
0028 #define SOA_INLINE inline __attribute__((always_inline))
0029 #endif
0030
0031
0032 #if defined(__CUDACC__) && defined(__CUDA_ARCH__)
0033 #define SOA_THROW_OUT_OF_RANGE(A) \
0034 { \
0035 printf("%s\n", (A)); \
0036 __trap(); \
0037 }
0038 #elif defined(__HIPCC__) && defined(__HIP_DEVICE_COMPILE__)
0039 #define SOA_THROW_OUT_OF_RANGE(A) \
0040 { \
0041 printf("%s\n", (A)); \
0042 abort(); \
0043 }
0044 #else
0045 #define SOA_THROW_OUT_OF_RANGE(A) \
0046 { throw std::out_of_range(A); }
0047 #endif
0048
0049
0050 #define _VALUE_TYPE_SCALAR 0
0051 #define _VALUE_TYPE_COLUMN 1
0052 #define _VALUE_TYPE_EIGEN_COLUMN 2
0053
0054
0055 #define CMS_SOA_BYTE_SIZE_TYPE std::size_t
0056
0057 namespace cms::soa {
0058
0059
0060 using size_type = cms_int32_t;
0061
0062 using byte_size_type = CMS_SOA_BYTE_SIZE_TYPE;
0063
0064 enum class SoAColumnType {
0065 scalar = _VALUE_TYPE_SCALAR,
0066 column = _VALUE_TYPE_COLUMN,
0067 eigen = _VALUE_TYPE_EIGEN_COLUMN
0068 };
0069
0070 namespace RestrictQualify {
0071 constexpr bool enabled = true;
0072 constexpr bool disabled = false;
0073 constexpr bool Default = disabled;
0074 }
0075
0076 namespace RangeChecking {
0077 constexpr bool enabled = true;
0078 constexpr bool disabled = false;
0079 constexpr bool Default = disabled;
0080 }
0081
0082 template <typename T, bool RESTRICT_QUALIFY>
0083 struct add_restrict {};
0084
0085 template <typename T>
0086 struct add_restrict<T, RestrictQualify::enabled> {
0087 using Value = T;
0088 using Pointer = T* __restrict__;
0089 using Reference = T& __restrict__;
0090 using ConstValue = const T;
0091 using PointerToConst = const T* __restrict__;
0092 using ReferenceToConst = const T& __restrict__;
0093 };
0094
0095 template <typename T>
0096 struct add_restrict<T, RestrictQualify::disabled> {
0097 using Value = T;
0098 using Pointer = T*;
0099 using Reference = T&;
0100 using ConstValue = const T;
0101 using PointerToConst = const T*;
0102 using ReferenceToConst = const T&;
0103 };
0104
0105
0106 template <SoAColumnType COLUMN_TYPE, typename T>
0107 struct SoAConstParametersImpl;
0108
0109 template <SoAColumnType COLUMN_TYPE, typename T>
0110 struct SoAParametersImpl;
0111
0112
0113 template <SoAColumnType COLUMN_TYPE, typename T>
0114 struct SoAConstParametersImpl {
0115 static constexpr SoAColumnType columnType = COLUMN_TYPE;
0116
0117 using ValueType = T;
0118 using ScalarType = T;
0119 using TupleOrPointerType = const ValueType*;
0120
0121
0122 SoAConstParametersImpl() = default;
0123
0124
0125 SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(ValueType const* addr) : addr_(addr) {}
0126
0127
0128 SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(SoAParametersImpl<columnType, ValueType> const& o)
0129 : addr_{o.addr_} {}
0130
0131 static constexpr bool checkAlignment(ValueType* addr, byte_size_type alignment) {
0132 return reinterpret_cast<intptr_t>(addr) % alignment;
0133 }
0134
0135 public:
0136
0137 ValueType const* addr_ = nullptr;
0138 };
0139
0140
0141 template <typename T>
0142 struct SoAConstParametersImpl<SoAColumnType::eigen, T> {
0143 static constexpr SoAColumnType columnType = SoAColumnType::eigen;
0144
0145 using ValueType = T;
0146 using ScalarType = typename T::Scalar;
0147 using TupleOrPointerType = std::tuple<ScalarType*, byte_size_type>;
0148
0149
0150 SoAConstParametersImpl() = default;
0151
0152
0153 SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(ScalarType const* addr, byte_size_type stride)
0154 : addr_(addr), stride_(stride) {}
0155
0156
0157 SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(TupleOrPointerType const& tuple)
0158 : addr_(std::get<0>(tuple)), stride_(std::get<1>(tuple)) {}
0159
0160
0161 SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(SoAParametersImpl<columnType, ValueType> const& o)
0162 : addr_{o.addr_}, stride_{o.stride_} {}
0163
0164 static constexpr bool checkAlignment(TupleOrPointerType const& tuple, byte_size_type alignment) {
0165 const auto& [addr, stride] = tuple;
0166 return reinterpret_cast<intptr_t>(addr) % alignment;
0167 }
0168
0169 public:
0170
0171 ScalarType const* addr_ = nullptr;
0172 byte_size_type stride_ = 0;
0173 };
0174
0175
0176 template <SoAColumnType COLUMN_TYPE>
0177 struct SoAConstParameters_ColumnType {
0178 template <typename T>
0179 using DataType = SoAConstParametersImpl<COLUMN_TYPE, T>;
0180 };
0181
0182
0183 template <SoAColumnType COLUMN_TYPE, typename T>
0184 struct SoAParametersImpl {
0185 static constexpr SoAColumnType columnType = COLUMN_TYPE;
0186
0187 using ValueType = T;
0188 using ScalarType = T;
0189 using TupleOrPointerType = ValueType*;
0190
0191 using ConstType = SoAConstParametersImpl<columnType, ValueType>;
0192 friend ConstType;
0193
0194
0195 SoAParametersImpl() = default;
0196
0197
0198 SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl(ValueType* addr) : addr_(addr) {}
0199
0200 static constexpr bool checkAlignment(ValueType* addr, byte_size_type alignment) {
0201 return reinterpret_cast<intptr_t>(addr) % alignment;
0202 }
0203
0204 public:
0205
0206 ValueType* addr_ = nullptr;
0207 };
0208
0209
0210 template <typename T>
0211 struct SoAParametersImpl<SoAColumnType::eigen, T> {
0212 static constexpr SoAColumnType columnType = SoAColumnType::eigen;
0213
0214 using ValueType = T;
0215 using ScalarType = typename T::Scalar;
0216 using TupleOrPointerType = std::tuple<ScalarType*, byte_size_type>;
0217
0218 using ConstType = SoAConstParametersImpl<columnType, ValueType>;
0219 friend ConstType;
0220
0221
0222 SoAParametersImpl() = default;
0223
0224
0225 SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl(ScalarType* addr, byte_size_type stride)
0226 : addr_(addr), stride_(stride) {}
0227
0228
0229 SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl(TupleOrPointerType const& tuple)
0230 : addr_(std::get<0>(tuple)), stride_(std::get<1>(tuple)) {}
0231
0232 static constexpr bool checkAlignment(TupleOrPointerType const& tuple, byte_size_type alignment) {
0233 const auto& [addr, stride] = tuple;
0234 return reinterpret_cast<intptr_t>(addr) % alignment;
0235 }
0236
0237 public:
0238
0239 ScalarType* addr_ = nullptr;
0240 byte_size_type stride_ = 0;
0241 };
0242
0243
0244 template <SoAColumnType COLUMN_TYPE>
0245 struct SoAParameters_ColumnType {
0246 template <typename T>
0247 using DataType = SoAParametersImpl<COLUMN_TYPE, T>;
0248 };
0249
0250
0251 namespace {
0252 template <typename T>
0253 constexpr inline std::remove_const_t<T>* non_const_ptr(T* p) {
0254 return const_cast<std::remove_const_t<T>*>(p);
0255 }
0256 }
0257
0258 template <SoAColumnType COLUMN_TYPE, typename T>
0259 SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl<COLUMN_TYPE, T> const_cast_SoAParametersImpl(
0260 SoAConstParametersImpl<COLUMN_TYPE, T> const& o) {
0261 return SoAParametersImpl<COLUMN_TYPE, T>{non_const_ptr(o.addr_)};
0262 }
0263
0264 template <typename T>
0265 SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl<SoAColumnType::eigen, T> const_cast_SoAParametersImpl(
0266 SoAConstParametersImpl<SoAColumnType::eigen, T> const& o) {
0267 return SoAParametersImpl<SoAColumnType::eigen, T>{non_const_ptr(o.addr_), o.stride_};
0268 }
0269
0270
0271
0272
0273 template <SoAColumnType COLUMN_TYPE,
0274 typename T,
0275 byte_size_type ALIGNMENT,
0276 bool RESTRICT_QUALIFY = RestrictQualify::disabled>
0277 class SoAValue {
0278
0279 static_assert(COLUMN_TYPE != SoAColumnType::eigen);
0280
0281 public:
0282 using Restr = add_restrict<T, RESTRICT_QUALIFY>;
0283 using Val = typename Restr::Value;
0284 using Ptr = typename Restr::Pointer;
0285 using Ref = typename Restr::Reference;
0286 using PtrToConst = typename Restr::PointerToConst;
0287 using RefToConst = typename Restr::ReferenceToConst;
0288
0289 SOA_HOST_DEVICE SOA_INLINE SoAValue(size_type i, T* col) : idx_(i), col_(col) {}
0290
0291 SOA_HOST_DEVICE SOA_INLINE SoAValue(size_type i, SoAParametersImpl<COLUMN_TYPE, T> params)
0292 : idx_(i), col_(params.addr_) {}
0293
0294 SOA_HOST_DEVICE SOA_INLINE Ref operator()() {
0295
0296 Ptr col = col_;
0297 return col[idx_];
0298 }
0299
0300 SOA_HOST_DEVICE SOA_INLINE RefToConst operator()() const {
0301
0302 PtrToConst col = col_();
0303 return col[idx_];
0304 }
0305
0306 SOA_HOST_DEVICE SOA_INLINE Ptr operator&() { return &col_[idx_]; }
0307
0308 SOA_HOST_DEVICE SOA_INLINE PtrToConst operator&() const { return &col_[idx_]; }
0309
0310
0311
0312
0313
0314
0315
0316
0317
0318
0319
0320
0321
0322
0323
0324
0325
0326
0327
0328
0329
0330
0331 using valueType = Val;
0332
0333 static constexpr auto valueSize = sizeof(T);
0334
0335 private:
0336 size_type idx_;
0337 T* col_;
0338 };
0339
0340
0341 #ifdef EIGEN_WORLD_VERSION
0342
0343 template <class C, byte_size_type ALIGNMENT, bool RESTRICT_QUALIFY>
0344 class SoAValue<SoAColumnType::eigen, C, ALIGNMENT, RESTRICT_QUALIFY> {
0345 public:
0346 using Type = C;
0347 using MapType = Eigen::Map<C, 0, Eigen::InnerStride<Eigen::Dynamic>>;
0348 using CMapType = const Eigen::Map<const C, 0, Eigen::InnerStride<Eigen::Dynamic>>;
0349 using Restr = add_restrict<typename C::Scalar, RESTRICT_QUALIFY>;
0350 using Val = typename Restr::Value;
0351 using Ptr = typename Restr::Pointer;
0352 using Ref = typename Restr::Reference;
0353 using PtrToConst = typename Restr::PointerToConst;
0354 using RefToConst = typename Restr::ReferenceToConst;
0355
0356 SOA_HOST_DEVICE SOA_INLINE SoAValue(size_type i, typename C::Scalar* col, byte_size_type stride)
0357 : val_(col + i, C::RowsAtCompileTime, C::ColsAtCompileTime, Eigen::InnerStride<Eigen::Dynamic>(stride)),
0358 crCol_(col),
0359 cVal_(crCol_ + i, C::RowsAtCompileTime, C::ColsAtCompileTime, Eigen::InnerStride<Eigen::Dynamic>(stride)),
0360 stride_(stride) {}
0361
0362 SOA_HOST_DEVICE SOA_INLINE SoAValue(size_type i, SoAParametersImpl<SoAColumnType::eigen, C> params)
0363 : val_(params.addr_ + i,
0364 C::RowsAtCompileTime,
0365 C::ColsAtCompileTime,
0366 Eigen::InnerStride<Eigen::Dynamic>(params.stride_)),
0367 crCol_(params.addr_),
0368 cVal_(crCol_ + i,
0369 C::RowsAtCompileTime,
0370 C::ColsAtCompileTime,
0371 Eigen::InnerStride<Eigen::Dynamic>(params.stride_)),
0372 stride_(params.stride_) {}
0373
0374 SOA_HOST_DEVICE SOA_INLINE MapType& operator()() { return val_; }
0375
0376 SOA_HOST_DEVICE SOA_INLINE const CMapType& operator()() const { return cVal_; }
0377
0378 SOA_HOST_DEVICE SOA_INLINE operator C() { return val_; }
0379
0380 SOA_HOST_DEVICE SOA_INLINE operator const C() const { return cVal_; }
0381
0382 SOA_HOST_DEVICE SOA_INLINE C* operator&() { return &val_; }
0383
0384 SOA_HOST_DEVICE SOA_INLINE const C* operator&() const { return &cVal_; }
0385
0386 template <class C2>
0387 SOA_HOST_DEVICE SOA_INLINE MapType& operator=(const C2& v) {
0388 return val_ = v;
0389 }
0390
0391 using ValueType = typename C::Scalar;
0392 static constexpr auto valueSize = sizeof(typename C::Scalar);
0393 SOA_HOST_DEVICE SOA_INLINE byte_size_type stride() const { return stride_; }
0394
0395 private:
0396 MapType val_;
0397 const Ptr crCol_;
0398 CMapType cVal_;
0399 byte_size_type stride_;
0400 };
0401 #else
0402
0403 template <class C, byte_size_type ALIGNMENT, bool RESTRICT_QUALIFY>
0404 class SoAValue<SoAColumnType::eigen, C, ALIGNMENT, RESTRICT_QUALIFY> {
0405 static_assert(!sizeof(C),
0406 "Eigen/Core should be pre-included before the SoA headers to enable support for Eigen columns.");
0407 };
0408 #endif
0409
0410
0411 template <SoAColumnType COLUMN_TYPE,
0412 typename T,
0413 byte_size_type ALIGNMENT,
0414 bool RESTRICT_QUALIFY = RestrictQualify::disabled>
0415 class SoAConstValue {
0416
0417 static_assert(COLUMN_TYPE != SoAColumnType::eigen);
0418
0419 public:
0420 using Restr = add_restrict<T, RESTRICT_QUALIFY>;
0421 using Val = typename Restr::Value;
0422 using Ptr = typename Restr::Pointer;
0423 using Ref = typename Restr::Reference;
0424 using PtrToConst = typename Restr::PointerToConst;
0425 using RefToConst = typename Restr::ReferenceToConst;
0426 using Params = SoAParametersImpl<COLUMN_TYPE, T>;
0427 using ConstParams = SoAConstParametersImpl<COLUMN_TYPE, T>;
0428
0429 SOA_HOST_DEVICE SOA_INLINE SoAConstValue(size_type i, const T* col) : idx_(i), col_(col) {}
0430
0431 SOA_HOST_DEVICE SOA_INLINE SoAConstValue(size_type i, SoAParametersImpl<COLUMN_TYPE, T> params)
0432 : idx_(i), col_(params.addr_) {}
0433
0434 SOA_HOST_DEVICE SOA_INLINE SoAConstValue(size_type i, SoAConstParametersImpl<COLUMN_TYPE, T> params)
0435 : idx_(i), col_(params.addr_) {}
0436
0437 SOA_HOST_DEVICE SOA_INLINE RefToConst operator()() const {
0438
0439 PtrToConst col = col_;
0440 return col[idx_];
0441 }
0442
0443 SOA_HOST_DEVICE SOA_INLINE const T* operator&() const { return &col_[idx_]; }
0444
0445
0446
0447
0448
0449
0450
0451
0452
0453
0454
0455
0456
0457
0458
0459 using valueType = T;
0460 static constexpr auto valueSize = sizeof(T);
0461
0462 private:
0463 size_type idx_;
0464 const T* col_;
0465 };
0466
0467
0468 #ifdef EIGEN_WORLD_VERSION
0469
0470 template <class C, byte_size_type ALIGNMENT, bool RESTRICT_QUALIFY>
0471 class SoAConstValue<SoAColumnType::eigen, C, ALIGNMENT, RESTRICT_QUALIFY> {
0472 public:
0473 using Type = C;
0474 using CMapType = Eigen::Map<const C, 0, Eigen::InnerStride<Eigen::Dynamic>>;
0475 using RefToConst = const CMapType&;
0476 using ConstParams = SoAConstParametersImpl<SoAColumnType::eigen, C>;
0477
0478 SOA_HOST_DEVICE SOA_INLINE SoAConstValue(size_type i, typename C::Scalar* col, byte_size_type stride)
0479 : crCol_(col),
0480 cVal_(crCol_ + i, C::RowsAtCompileTime, C::ColsAtCompileTime, Eigen::InnerStride<Eigen::Dynamic>(stride)),
0481 stride_(stride) {}
0482
0483 SOA_HOST_DEVICE SOA_INLINE SoAConstValue(size_type i, SoAConstParametersImpl<SoAColumnType::eigen, C> params)
0484 : crCol_(params.addr_),
0485 cVal_(crCol_ + i,
0486 C::RowsAtCompileTime,
0487 C::ColsAtCompileTime,
0488 Eigen::InnerStride<Eigen::Dynamic>(params.stride_)),
0489 stride_(params.stride_) {}
0490
0491 SOA_HOST_DEVICE SOA_INLINE const CMapType& operator()() const { return cVal_; }
0492
0493 SOA_HOST_DEVICE SOA_INLINE operator const C() const { return cVal_; }
0494
0495 SOA_HOST_DEVICE SOA_INLINE const C* operator&() const { return &cVal_; }
0496
0497 using ValueType = typename C::Scalar;
0498 static constexpr auto valueSize = sizeof(typename C::Scalar);
0499
0500 SOA_HOST_DEVICE SOA_INLINE byte_size_type stride() const { return stride_; }
0501
0502 private:
0503 const typename C::Scalar* __restrict__ crCol_;
0504 CMapType cVal_;
0505 byte_size_type stride_;
0506 };
0507 #else
0508
0509 template <class C, byte_size_type ALIGNMENT, bool RESTRICT_QUALIFY>
0510 class SoAConstValue<SoAColumnType::eigen, C, ALIGNMENT, RESTRICT_QUALIFY> {
0511 static_assert(!sizeof(C),
0512 "Eigen/Core should be pre-included before the SoA headers to enable support for Eigen columns.");
0513 };
0514 #endif
0515
0516
0517 #ifdef EIGEN_WORLD_VERSION
0518 template <class C>
0519 struct EigenConstMapMaker {
0520 using Type = Eigen::Map<const C, Eigen::AlignmentType::Unaligned, Eigen::InnerStride<Eigen::Dynamic>>;
0521
0522 class DataHolder {
0523 public:
0524 DataHolder(const typename C::Scalar* data) : data_(data) {}
0525
0526 EigenConstMapMaker::Type withStride(byte_size_type stride) {
0527 return EigenConstMapMaker::Type(
0528 data_, C::RowsAtCompileTime, C::ColsAtCompileTime, Eigen::InnerStride<Eigen::Dynamic>(stride));
0529 }
0530
0531 private:
0532 const typename C::Scalar* const data_;
0533 };
0534
0535 static DataHolder withData(const typename C::Scalar* data) { return DataHolder(data); }
0536 };
0537 #else
0538 template <class C>
0539 struct EigenConstMapMaker {
0540
0541 static_assert(!sizeof(C),
0542 "Eigen/Core should be pre-included before the SoA headers to enable support for Eigen columns.");
0543 };
0544 #endif
0545
0546
0547 constexpr inline byte_size_type alignSize(byte_size_type size, byte_size_type alignment) {
0548 return ((size + alignment - 1) / alignment) * alignment;
0549 }
0550
0551 }
0552
0553 #define SOA_SCALAR(TYPE, NAME) (_VALUE_TYPE_SCALAR, TYPE, NAME)
0554 #define SOA_COLUMN(TYPE, NAME) (_VALUE_TYPE_COLUMN, TYPE, NAME)
0555 #define SOA_EIGEN_COLUMN(TYPE, NAME) (_VALUE_TYPE_EIGEN_COLUMN, TYPE, NAME)
0556
0557
0558 #define _ITERATE_ON_ALL_COMMA(MACRO, DATA, ...) \
0559 BOOST_PP_TUPLE_ENUM(BOOST_PP_SEQ_TO_TUPLE(_ITERATE_ON_ALL(MACRO, DATA, __VA_ARGS__)))
0560
0561
0562 #define _ITERATE_ON_ALL(MACRO, DATA, ...) BOOST_PP_SEQ_FOR_EACH(MACRO, DATA, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__))
0563
0564
0565 #define _SWITCH_ON_TYPE(VALUE_TYPE, IF_SCALAR, IF_COLUMN, IF_EIGEN_COLUMN) \
0566 BOOST_PP_IF( \
0567 BOOST_PP_EQUAL(VALUE_TYPE, _VALUE_TYPE_SCALAR), \
0568 IF_SCALAR, \
0569 BOOST_PP_IF( \
0570 BOOST_PP_EQUAL(VALUE_TYPE, _VALUE_TYPE_COLUMN), \
0571 IF_COLUMN, \
0572 BOOST_PP_IF(BOOST_PP_EQUAL(VALUE_TYPE, _VALUE_TYPE_EIGEN_COLUMN), IF_EIGEN_COLUMN, BOOST_PP_EMPTY())))
0573
0574 namespace cms::soa {
0575
0576
0577 enum class SoAAccessType : bool { mutableAccess, constAccess };
0578
0579 template <typename, SoAColumnType, SoAAccessType, byte_size_type, bool>
0580 struct SoAColumnAccessorsImpl {};
0581
0582
0583
0584
0585
0586
0587 template <typename T, byte_size_type alignment, bool restrictQualify>
0588 struct SoAColumnAccessorsImpl<T, SoAColumnType::column, SoAAccessType::mutableAccess, alignment, restrictQualify> {
0589 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAParametersImpl<SoAColumnType::column, T>& params)
0590 : params_(params) {}
0591 SOA_HOST_DEVICE SOA_INLINE T* operator()() { return params_.addr_; }
0592 using NoParamReturnType = T*;
0593 using ParamReturnType = T&;
0594 SOA_HOST_DEVICE SOA_INLINE T& operator()(size_type index) { return params_.addr_[index]; }
0595
0596 private:
0597 SoAParametersImpl<SoAColumnType::column, T> params_;
0598 };
0599
0600
0601 template <typename T, byte_size_type alignment, bool restrictQualify>
0602 struct SoAColumnAccessorsImpl<T, SoAColumnType::column, SoAAccessType::constAccess, alignment, restrictQualify> {
0603 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAConstParametersImpl<SoAColumnType::column, T>& params)
0604 : params_(params) {}
0605 SOA_HOST_DEVICE SOA_INLINE const T* operator()() const { return params_.addr_; }
0606 using NoParamReturnType = const T*;
0607 using ParamReturnType = const T&;
0608 SOA_HOST_DEVICE SOA_INLINE T const& operator()(size_type index) const { return params_.addr_[index]; }
0609
0610 private:
0611 SoAConstParametersImpl<SoAColumnType::column, T> params_;
0612 };
0613
0614
0615 template <typename T, byte_size_type alignment, bool restrictQualify>
0616 struct SoAColumnAccessorsImpl<T, SoAColumnType::scalar, SoAAccessType::mutableAccess, alignment, restrictQualify> {
0617 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAParametersImpl<SoAColumnType::scalar, T>& params)
0618 : params_(params) {}
0619 SOA_HOST_DEVICE SOA_INLINE T& operator()() { return *params_.addr_; }
0620 using NoParamReturnType = T&;
0621 using ParamReturnType = void;
0622 SOA_HOST_DEVICE SOA_INLINE void operator()(size_type index) const {
0623 assert(false && "Indexed access impossible for SoA scalars.");
0624 }
0625
0626 private:
0627 SoAParametersImpl<SoAColumnType::scalar, T> params_;
0628 };
0629
0630
0631 template <typename T, byte_size_type alignment, bool restrictQualify>
0632 struct SoAColumnAccessorsImpl<T, SoAColumnType::scalar, SoAAccessType::constAccess, alignment, restrictQualify> {
0633 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAConstParametersImpl<SoAColumnType::scalar, T>& params)
0634 : params_(params) {}
0635 SOA_HOST_DEVICE SOA_INLINE T const& operator()() const { return *params_.addr_; }
0636 using NoParamReturnType = T const&;
0637 using ParamReturnType = void;
0638 SOA_HOST_DEVICE SOA_INLINE void operator()(size_type index) const {
0639 assert(false && "Indexed access impossible for SoA scalars.");
0640 }
0641
0642 private:
0643 SoAConstParametersImpl<SoAColumnType::scalar, T> params_;
0644 };
0645
0646
0647 template <typename T, byte_size_type alignment, bool restrictQualify>
0648 struct SoAColumnAccessorsImpl<T, SoAColumnType::eigen, SoAAccessType::mutableAccess, alignment, restrictQualify> {
0649 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAParametersImpl<SoAColumnType::eigen, T>& params)
0650 : params_(params) {}
0651 SOA_HOST_DEVICE SOA_INLINE typename T::Scalar* operator()() { return params_.addr_; }
0652 using NoParamReturnType = typename T::Scalar*;
0653 using ParamReturnType = typename SoAValue<SoAColumnType::eigen, T, alignment, restrictQualify>::MapType;
0654 SOA_HOST_DEVICE SOA_INLINE ParamReturnType operator()(size_type index) {
0655 return SoAValue<SoAColumnType::eigen, T, alignment, restrictQualify>(index, params_)();
0656 }
0657
0658 private:
0659 SoAParametersImpl<SoAColumnType::eigen, T> params_;
0660 };
0661
0662
0663 template <typename T, byte_size_type alignment, bool restrictQualify>
0664 struct SoAColumnAccessorsImpl<T, SoAColumnType::eigen, SoAAccessType::constAccess, alignment, restrictQualify> {
0665 SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAConstParametersImpl<SoAColumnType::eigen, T>& params)
0666 : params_(params) {}
0667 SOA_HOST_DEVICE SOA_INLINE typename T::Scalar const* operator()() const { return params_.addr_; }
0668 using NoParamReturnType = typename T::Scalar const*;
0669 using ParamReturnType = typename SoAValue<SoAColumnType::eigen, T, alignment, restrictQualify>::CMapType;
0670 SOA_HOST_DEVICE SOA_INLINE ParamReturnType operator()(size_type index) const {
0671 return SoAConstValue<SoAColumnType::eigen, T, alignment, restrictQualify>(index, params_)();
0672 }
0673
0674 private:
0675 SoAConstParametersImpl<SoAColumnType::eigen, T> params_;
0676 };
0677
0678
0679 template <typename T>
0680 struct SoAAccessors {
0681 template <auto columnType>
0682 struct ColumnType {
0683 template <auto accessType>
0684 struct AccessType {
0685 template <auto alignment>
0686 struct Alignment {
0687 template <auto restrictQualify>
0688 struct RestrictQualifier
0689 : public SoAColumnAccessorsImpl<T, columnType, accessType, alignment, restrictQualify> {
0690 using SoAColumnAccessorsImpl<T, columnType, accessType, alignment, restrictQualify>::SoAColumnAccessorsImpl;
0691 };
0692 };
0693 };
0694 };
0695 };
0696
0697
0698
0699
0700 struct AlignmentEnforcement {
0701 static constexpr bool relaxed = false;
0702 static constexpr bool enforced = true;
0703 };
0704
0705 struct CacheLineSize {
0706 static constexpr byte_size_type NvidiaGPU = 128;
0707 static constexpr byte_size_type IntelCPU = 64;
0708 static constexpr byte_size_type AMDCPU = 64;
0709 static constexpr byte_size_type ARMCPU = 64;
0710 static constexpr byte_size_type defaultSize = NvidiaGPU;
0711 };
0712
0713 }
0714
0715
0716 template <typename SOA,
0717 typename SFINAE =
0718 typename std::enable_if_t<std::is_invocable_v<decltype(&SOA::soaToStreamInternal), SOA&, std::ostream&>>>
0719 SOA_HOST_ONLY std::ostream& operator<<(std::ostream& os, const SOA& soa) {
0720 soa.soaToStreamInternal(os);
0721 return os;
0722 }
0723
0724 #endif