File indexing completed on 2023-10-25 09:57:43
0001 #include <memory>
0002 #include <cassert>
0003 #include <atomic>
0004 #include <cstddef>
0005 #include <malloc.h>
0006
0007 #include "PerfTools/AllocMonitor/interface/AllocMonitorRegistry.h"
0008 #include "FWCore/Utilities/interface/thread_safety_macros.h"
0009
0010 #include <dlfcn.h> // dlsym
0011
0012 #if !defined(__x86_64__) && !defined(__i386__)
0013 #define USE_LOCAL_MALLOC
0014 #endif
0015 #if defined(__GLIBC__) && (__GLIBC__ == 2) && (__GLIBC_MINOR__ < 28)
0016
0017 #define USE_LOCAL_MALLOC
0018 #endif
0019
0020 namespace {
0021 std::atomic<bool>& alloc_monitor_running_state() {
0022 static std::atomic<bool> s_state = false;
0023 return s_state;
0024 }
0025
0026 template <typename T>
0027 T get(const char* iName) {
0028 void* original = dlsym(RTLD_NEXT, iName);
0029 assert(original);
0030 return reinterpret_cast<T>(original);
0031 }
0032
0033 #ifdef USE_LOCAL_MALLOC
0034
0035
0036
0037
0038
0039 constexpr auto max_align = alignof(std::max_align_t);
0040 alignas(max_align) char tmpbuff[131072];
0041 unsigned long tmppos = 0;
0042 unsigned long tmpallocs = 0;
0043
0044 void* local_malloc(size_t size) noexcept {
0045
0046 size = ((size + max_align - 1) / max_align) * max_align;
0047 if (tmppos + size < sizeof(tmpbuff)) {
0048 void* retptr = tmpbuff + tmppos;
0049 tmppos += size;
0050 ++tmpallocs;
0051 return retptr;
0052 } else {
0053 return nullptr;
0054 }
0055 }
0056
0057
0058 void* local_calloc(size_t nitems, size_t item_size) noexcept { return local_malloc(nitems * item_size); }
0059
0060 inline bool is_local_alloc(void* ptr) noexcept { return ptr >= (void*)tmpbuff && ptr <= (void*)(tmpbuff + tmppos); }
0061
0062
0063
0064
0065 struct Originals {
0066 inline static void init() noexcept {
0067 if (not set) {
0068 set = true;
0069 malloc = get<decltype(&::malloc)>("malloc");
0070 calloc = get<decltype(&::calloc)>("calloc");
0071 }
0072 }
0073 CMS_SA_ALLOW static decltype(&::malloc) malloc;
0074 CMS_SA_ALLOW static decltype(&::calloc) calloc;
0075 CMS_SA_ALLOW static bool set;
0076 };
0077
0078 decltype(&::malloc) Originals::malloc = local_malloc;
0079 decltype(&::calloc) Originals::calloc = local_calloc;
0080 bool Originals::set = false;
0081 #else
0082 constexpr inline bool is_local_alloc(void* ptr) noexcept { return false; }
0083 #endif
0084 }
0085
0086 using namespace cms::perftools;
0087
0088 extern "C" {
0089 void alloc_monitor_start() { alloc_monitor_running_state() = true; }
0090 void alloc_monitor_stop() { alloc_monitor_running_state() = false; }
0091
0092
0093
0094
0095 #ifdef USE_LOCAL_MALLOC
0096 void* malloc(size_t size) noexcept {
0097 const auto original = Originals::malloc;
0098 Originals::init();
0099 if (not alloc_monitor_running_state()) {
0100 return original(size);
0101 }
0102 auto& reg = AllocMonitorRegistry::instance();
0103 return reg.allocCalled(
0104 size, [size, original]() { return original(size); }, [](auto ret) { return malloc_usable_size(ret); });
0105 }
0106
0107 void* calloc(size_t nitems, size_t item_size) noexcept {
0108 const auto original = Originals::calloc;
0109 Originals::init();
0110 if (not alloc_monitor_running_state()) {
0111 return original(nitems, item_size);
0112 }
0113 auto& reg = AllocMonitorRegistry::instance();
0114 return reg.allocCalled(
0115 nitems * item_size,
0116 [nitems, item_size, original]() { return original(nitems, item_size); },
0117 [](auto ret) { return malloc_usable_size(ret); });
0118 }
0119 #else
0120 void* malloc(size_t size) noexcept {
0121 CMS_SA_ALLOW static const auto original = get<decltype(&::malloc)>("malloc");
0122 if (not alloc_monitor_running_state()) {
0123 return original(size);
0124 }
0125 auto& reg = AllocMonitorRegistry::instance();
0126 return reg.allocCalled(
0127 size, [size]() { return original(size); }, [](auto ret) { return malloc_usable_size(ret); });
0128 }
0129
0130 void* calloc(size_t nitems, size_t item_size) noexcept {
0131 CMS_SA_ALLOW static const auto original = get<decltype(&::calloc)>("calloc");
0132 if (not alloc_monitor_running_state()) {
0133 return original(nitems, item_size);
0134 }
0135 auto& reg = AllocMonitorRegistry::instance();
0136 return reg.allocCalled(
0137 nitems * item_size,
0138 [nitems, item_size]() { return original(nitems, item_size); },
0139 [](auto ret) { return malloc_usable_size(ret); });
0140 }
0141 #endif
0142
0143 void* realloc(void* ptr, size_t size) noexcept {
0144 CMS_SA_ALLOW static const auto original = get<decltype(&::realloc)>("realloc");
0145 if (not alloc_monitor_running_state()) {
0146 return original(ptr, size);
0147 }
0148 size_t oldsize = malloc_usable_size(ptr);
0149 void* ret;
0150 auto& reg = AllocMonitorRegistry::instance();
0151 {
0152
0153 [[maybe_unused]] auto g = reg.makeGuard();
0154 ret = original(ptr, size);
0155 }
0156 size_t used = malloc_usable_size(ret);
0157 if (used != oldsize) {
0158 reg.deallocCalled(
0159 ptr, [](auto) {}, [oldsize](auto) { return oldsize; });
0160 reg.allocCalled(
0161 size, []() { return nullptr; }, [used](auto) { return used; });
0162 }
0163 return ret;
0164 }
0165
0166 void* aligned_alloc(size_t alignment, size_t size) noexcept {
0167 CMS_SA_ALLOW static const auto original = get<decltype(&::aligned_alloc)>("aligned_alloc");
0168 if (not alloc_monitor_running_state()) {
0169 return original(alignment, size);
0170 }
0171
0172 auto& reg = AllocMonitorRegistry::instance();
0173 return reg.allocCalled(
0174 size,
0175 [alignment, size]() { return original(alignment, size); },
0176 [](auto ret) { return malloc_usable_size(ret); });
0177 }
0178
0179
0180 int posix_memalign(void** memptr, size_t alignment, size_t size) noexcept {
0181 CMS_SA_ALLOW static const auto original = get<decltype(&::posix_memalign)>("posix_memalign");
0182 if (not alloc_monitor_running_state()) {
0183 return original(memptr, alignment, size);
0184 }
0185
0186 auto& reg = AllocMonitorRegistry::instance();
0187 int ret;
0188 reg.allocCalled(
0189 size,
0190 [&ret, memptr, alignment, size]() {
0191 ret = original(memptr, alignment, size);
0192 return *memptr;
0193 },
0194 [](auto ret) { return malloc_usable_size(ret); });
0195 return ret;
0196 }
0197
0198
0199 void* memalign(size_t alignment, size_t size) noexcept {
0200 CMS_SA_ALLOW static const auto original = get<decltype(&::memalign)>("memalign");
0201 if (not alloc_monitor_running_state()) {
0202 return original(alignment, size);
0203 }
0204
0205 auto& reg = AllocMonitorRegistry::instance();
0206 return reg.allocCalled(
0207 size,
0208 [alignment, size]() { return original(alignment, size); },
0209 [](auto ret) { return malloc_usable_size(ret); });
0210 }
0211
0212 void free(void* ptr) noexcept {
0213 CMS_SA_ALLOW static const auto original = get<decltype(&::free)>("free");
0214
0215 if (not is_local_alloc(ptr)) {
0216 if (not alloc_monitor_running_state()) {
0217 original(ptr);
0218 return;
0219 }
0220
0221 auto& reg = AllocMonitorRegistry::instance();
0222 reg.deallocCalled(
0223 ptr, [](auto ptr) { original(ptr); }, [](auto ptr) { return malloc_usable_size(ptr); });
0224 }
0225 }
0226 }
0227
0228
0229
0230
0231 #define CPP_MEM_OVERRIDE
0232
0233 #if defined(CPP_MEM_OVERRIDE)
0234 #include <new>
0235
0236 void* operator new(std::size_t size) {
0237 CMS_SA_ALLOW static const auto original = get<void* (*)(std::size_t)>("_Znwm");
0238 if (not alloc_monitor_running_state()) {
0239 return original(size);
0240 }
0241
0242 auto& reg = AllocMonitorRegistry::instance();
0243 return reg.allocCalled(
0244 size, [size]() { return original(size); }, [](auto ret) { return malloc_usable_size(ret); });
0245 }
0246
0247 void operator delete(void* ptr) noexcept {
0248 CMS_SA_ALLOW static const auto original = get<void (*)(void*)>("_ZdlPv");
0249 if (not alloc_monitor_running_state()) {
0250 original(ptr);
0251 return;
0252 }
0253
0254 auto& reg = AllocMonitorRegistry::instance();
0255 reg.deallocCalled(
0256 ptr, [](auto ptr) { original(ptr); }, [](auto ptr) { return malloc_usable_size(ptr); });
0257 }
0258
0259 void* operator new[](std::size_t size) {
0260 CMS_SA_ALLOW static const auto original = get<void* (*)(std::size_t)>("_Znam");
0261 if (not alloc_monitor_running_state()) {
0262 return original(size);
0263 }
0264
0265 auto& reg = AllocMonitorRegistry::instance();
0266 return reg.allocCalled(
0267 size, [size]() { return original(size); }, [](auto ret) { return malloc_usable_size(ret); });
0268 }
0269
0270 void operator delete[](void* ptr) noexcept {
0271 CMS_SA_ALLOW static const auto original = get<void (*)(void*)>("_ZdaPv");
0272
0273 if (not alloc_monitor_running_state()) {
0274 original(ptr);
0275 return;
0276 }
0277 auto& reg = AllocMonitorRegistry::instance();
0278 reg.deallocCalled(
0279 ptr, [](auto ptr) { original(ptr); }, [](auto ptr) { return malloc_usable_size(ptr); });
0280 }
0281
0282 void* operator new(std::size_t size, std::align_val_t al) {
0283 CMS_SA_ALLOW static const auto original = get<void* (*)(std::size_t, std::align_val_t)>("_ZnwmSt11align_val_t");
0284 if (not alloc_monitor_running_state()) {
0285 return original(size, al);
0286 }
0287
0288 auto& reg = AllocMonitorRegistry::instance();
0289 return reg.allocCalled(
0290 size, [size, al]() { return original(size, al); }, [](auto ret) { return malloc_usable_size(ret); });
0291 }
0292
0293 void* operator new[](std::size_t size, std::align_val_t al) {
0294 CMS_SA_ALLOW static const auto original = get<void* (*)(std::size_t, std::align_val_t)>("_ZnamSt11align_val_t");
0295
0296 if (not alloc_monitor_running_state()) {
0297 return original(size, al);
0298 }
0299
0300 auto& reg = AllocMonitorRegistry::instance();
0301 return reg.allocCalled(
0302 size, [size, al]() { return original(size, al); }, [](auto ret) { return malloc_usable_size(ret); });
0303 }
0304
0305 void* operator new(std::size_t size, const std::nothrow_t& tag) noexcept {
0306 CMS_SA_ALLOW static const auto original =
0307 get<void* (*)(std::size_t, const std::nothrow_t&) noexcept>("_ZnwmRKSt9nothrow_t");
0308
0309 if (not alloc_monitor_running_state()) {
0310 return original(size, tag);
0311 }
0312
0313 auto& reg = AllocMonitorRegistry::instance();
0314 return reg.allocCalled(
0315 size, [size, &tag]() { return original(size, tag); }, [](auto ret) { return malloc_usable_size(ret); });
0316 }
0317
0318 void* operator new[](std::size_t size, const std::nothrow_t& tag) noexcept {
0319 CMS_SA_ALLOW static const auto original =
0320 get<void* (*)(std::size_t, const std::nothrow_t&) noexcept>("_ZnamRKSt9nothrow_t");
0321
0322 if (not alloc_monitor_running_state()) {
0323 return original(size, tag);
0324 }
0325
0326 auto& reg = AllocMonitorRegistry::instance();
0327 return reg.allocCalled(
0328 size, [size, &tag]() { return original(size, tag); }, [](auto ret) { return malloc_usable_size(ret); });
0329 }
0330
0331 void* operator new(std::size_t size, std::align_val_t al, const std::nothrow_t& tag) noexcept {
0332 CMS_SA_ALLOW static const auto original =
0333 get<void* (*)(std::size_t, std::align_val_t, const std::nothrow_t&) noexcept>(
0334 "_ZnwmSt11align_val_tRKSt9nothrow_t");
0335
0336 if (not alloc_monitor_running_state()) {
0337 return original(size, al, tag);
0338 }
0339
0340 auto& reg = AllocMonitorRegistry::instance();
0341 return reg.allocCalled(
0342 size, [size, al, &tag]() { return original(size, al, tag); }, [](auto ret) { return malloc_usable_size(ret); });
0343 }
0344
0345 void* operator new[](std::size_t size, std::align_val_t al, const std::nothrow_t& tag) noexcept {
0346 CMS_SA_ALLOW static const auto original =
0347 get<void* (*)(std::size_t, std::align_val_t, const std::nothrow_t&) noexcept>(
0348 "_ZnamSt11align_val_tRKSt9nothrow_t");
0349
0350 if (not alloc_monitor_running_state()) {
0351 return original(size, al, tag);
0352 }
0353
0354 auto& reg = AllocMonitorRegistry::instance();
0355 return reg.allocCalled(
0356 size, [size, al, &tag]() { return original(size, al, tag); }, [](auto ret) { return malloc_usable_size(ret); });
0357 }
0358
0359 void operator delete(void* ptr, std::align_val_t al) noexcept {
0360 CMS_SA_ALLOW static const auto original = get<void (*)(void*, std::align_val_t) noexcept>("_ZdlPvSt11align_val_t");
0361
0362 if (not alloc_monitor_running_state()) {
0363 original(ptr, al);
0364 return;
0365 }
0366 auto& reg = AllocMonitorRegistry::instance();
0367 reg.deallocCalled(
0368 ptr, [al](auto ptr) { original(ptr, al); }, [](auto ptr) { return malloc_usable_size(ptr); });
0369 }
0370
0371 void operator delete[](void* ptr, std::align_val_t al) noexcept {
0372 CMS_SA_ALLOW static const auto original = get<void (*)(void*, std::align_val_t) noexcept>("_ZdaPvSt11align_val_t");
0373
0374 if (not alloc_monitor_running_state()) {
0375 original(ptr, al);
0376 return;
0377 }
0378 auto& reg = AllocMonitorRegistry::instance();
0379 reg.deallocCalled(
0380 ptr, [al](auto ptr) { original(ptr, al); }, [](auto ptr) { return malloc_usable_size(ptr); });
0381 }
0382
0383 void operator delete(void* ptr, std::size_t sz) noexcept {
0384 CMS_SA_ALLOW static const auto original = get<void (*)(void*, std::size_t) noexcept>("_ZdlPvm");
0385
0386 if (not alloc_monitor_running_state()) {
0387 original(ptr, sz);
0388 return;
0389 }
0390 auto& reg = AllocMonitorRegistry::instance();
0391 reg.deallocCalled(
0392 ptr, [sz](auto ptr) { original(ptr, sz); }, [](auto ptr) { return malloc_usable_size(ptr); });
0393 }
0394
0395 void operator delete[](void* ptr, std::size_t sz) noexcept {
0396 CMS_SA_ALLOW static const auto original = get<void (*)(void*, std::size_t) noexcept>("_ZdaPvm");
0397
0398 if (not alloc_monitor_running_state()) {
0399 original(ptr, sz);
0400 return;
0401 }
0402 auto& reg = AllocMonitorRegistry::instance();
0403 reg.deallocCalled(
0404 ptr, [sz](auto ptr) { original(ptr, sz); }, [](auto ptr) { return malloc_usable_size(ptr); });
0405 }
0406
0407 void operator delete(void* ptr, std::size_t sz, std::align_val_t al) noexcept {
0408 CMS_SA_ALLOW static const auto original =
0409 get<void (*)(void*, std::size_t, std::align_val_t) noexcept>("_ZdlPvmSt11align_val_t");
0410
0411 if (not alloc_monitor_running_state()) {
0412 original(ptr, sz, al);
0413 return;
0414 }
0415 auto& reg = AllocMonitorRegistry::instance();
0416 reg.deallocCalled(
0417 ptr, [sz, al](auto ptr) { original(ptr, sz, al); }, [](auto ptr) { return malloc_usable_size(ptr); });
0418 }
0419
0420 void operator delete[](void* ptr, std::size_t sz, std::align_val_t al) noexcept {
0421 CMS_SA_ALLOW static const auto original =
0422 get<void (*)(void*, std::size_t, std::align_val_t) noexcept>("_ZdaPvmSt11align_val_t");
0423
0424 if (not alloc_monitor_running_state()) {
0425 original(ptr, sz, al);
0426 return;
0427 }
0428 auto& reg = AllocMonitorRegistry::instance();
0429 reg.deallocCalled(
0430 ptr, [sz, al](auto ptr) { original(ptr, sz, al); }, [](auto ptr) { return malloc_usable_size(ptr); });
0431 }
0432
0433 void operator delete(void* ptr, const std::nothrow_t& tag) noexcept {
0434 CMS_SA_ALLOW static const auto original =
0435 get<void (*)(void*, const std::nothrow_t&) noexcept>("_ZdlPvRKSt9nothrow_t");
0436
0437 if (not alloc_monitor_running_state()) {
0438 original(ptr, tag);
0439 return;
0440 }
0441 auto& reg = AllocMonitorRegistry::instance();
0442 reg.deallocCalled(
0443 ptr, [&tag](auto ptr) { original(ptr, tag); }, [](auto ptr) { return malloc_usable_size(ptr); });
0444 }
0445
0446 void operator delete[](void* ptr, const std::nothrow_t& tag) noexcept {
0447 CMS_SA_ALLOW static const auto original =
0448 get<void (*)(void*, const std::nothrow_t&) noexcept>("_ZdaPvRKSt9nothrow_t");
0449
0450 if (not alloc_monitor_running_state()) {
0451 original(ptr, tag);
0452 return;
0453 }
0454 auto& reg = AllocMonitorRegistry::instance();
0455 reg.deallocCalled(
0456 ptr, [&tag](auto ptr) { original(ptr, tag); }, [](auto ptr) { return malloc_usable_size(ptr); });
0457 }
0458
0459 void operator delete(void* ptr, std::align_val_t al, const std::nothrow_t& tag) noexcept {
0460 CMS_SA_ALLOW static const auto original =
0461 get<void (*)(void*, std::align_val_t, const std::nothrow_t&) noexcept>("_ZdlPvSt11align_val_tRKSt9nothrow_t");
0462
0463 if (not alloc_monitor_running_state()) {
0464 original(ptr, al, tag);
0465 return;
0466 }
0467 auto& reg = AllocMonitorRegistry::instance();
0468 reg.deallocCalled(
0469 ptr, [al, &tag](auto ptr) { original(ptr, al, tag); }, [](auto ptr) { return malloc_usable_size(ptr); });
0470 }
0471
0472 void operator delete[](void* ptr, std::align_val_t al, const std::nothrow_t& tag) noexcept {
0473 CMS_SA_ALLOW static const auto original =
0474 get<void (*)(void*, std::align_val_t, const std::nothrow_t&) noexcept>("_ZdaPvSt11align_val_tRKSt9nothrow_t");
0475
0476 if (not alloc_monitor_running_state()) {
0477 original(ptr, al, tag);
0478 return;
0479 }
0480 auto& reg = AllocMonitorRegistry::instance();
0481 reg.deallocCalled(
0482 ptr, [al, &tag](auto ptr) { original(ptr, al, tag); }, [](auto ptr) { return malloc_usable_size(ptr); });
0483 }
0484
0485 #endif