Back to home page

Project CMSSW displayed by LXR
 
 

    

File indexing completed on 2023-10-25 09:57:43 

0001 #include <memory>
0002 #include <cassert>
0003 #include <atomic>
0004 #include <cstddef>
0005 #include <malloc.h>
0006 
0007 #include "PerfTools/AllocMonitor/interface/AllocMonitorRegistry.h"
0008 #include "FWCore/Utilities/interface/thread_safety_macros.h"
0009 
0010 #include <dlfcn.h>  // dlsym
0011 
0012 #if !defined(__x86_64__) && !defined(__i386__)
0013 #define USE_LOCAL_MALLOC
0014 #endif
0015 #if defined(__GLIBC__) && (__GLIBC__ == 2) && (__GLIBC_MINOR__ < 28)
0016 //needed for sl7
0017 #define USE_LOCAL_MALLOC
0018 #endif
0019 
0020 namespace {
0021   std::atomic<bool>& alloc_monitor_running_state() {
0022     static std::atomic<bool> s_state = false;
0023     return s_state;
0024   }
0025 
0026   template <typename T>
0027   T get(const char* iName) {
0028     void* original = dlsym(RTLD_NEXT, iName);
0029     assert(original);
0030     return reinterpret_cast<T>(original);
0031   }
0032 
0033 #ifdef USE_LOCAL_MALLOC
0034   // this is a very simple-minded allocator used for any allocations
0035   // before we've finished our setup.  In particular, this avoids a
0036   // chicken/egg problem if dlsym() allocates any memory.
0037   // Size was chosen to be 2x what ARM64 uses as an emergency buffer
0038   // for libstdc++ exception handling.
0039   constexpr auto max_align = alignof(std::max_align_t);
0040   alignas(max_align) char tmpbuff[131072];
0041   unsigned long tmppos = 0;
0042   unsigned long tmpallocs = 0;
0043 
0044   void* local_malloc(size_t size) noexcept {
0045     // round up so next alloc is aligned
0046     size = ((size + max_align - 1) / max_align) * max_align;
0047     if (tmppos + size < sizeof(tmpbuff)) {
0048       void* retptr = tmpbuff + tmppos;
0049       tmppos += size;
0050       ++tmpallocs;
0051       return retptr;
0052     } else {
0053       return nullptr;
0054     }
0055   }
0056 
0057   //can use local_malloc since static memory buffers are guaranteed to be zero initialized
0058   void* local_calloc(size_t nitems, size_t item_size) noexcept { return local_malloc(nitems * item_size); }
0059 
0060   inline bool is_local_alloc(void* ptr) noexcept { return ptr >= (void*)tmpbuff && ptr <= (void*)(tmpbuff + tmppos); }
0061 
0062   // the pointers in this struct should only be modified during
0063   // global construction at program startup, so thread safety
0064   // should not be an issue.
0065   struct Originals {
0066     inline static void init() noexcept {
0067       if (not set) {
0068         set = true;  // must be first to avoid recursion
0069         malloc = get<decltype(&::malloc)>("malloc");
0070         calloc = get<decltype(&::calloc)>("calloc");
0071       }
0072     }
0073     CMS_SA_ALLOW static decltype(&::malloc) malloc;
0074     CMS_SA_ALLOW static decltype(&::calloc) calloc;
0075     CMS_SA_ALLOW static bool set;
0076   };
0077 
0078   decltype(&::malloc) Originals::malloc = local_malloc;
0079   decltype(&::calloc) Originals::calloc = local_calloc;
0080   bool Originals::set = false;
0081 #else
0082   constexpr inline bool is_local_alloc(void* ptr) noexcept { return false; }
0083 #endif
0084 }  // namespace
0085 
0086 using namespace cms::perftools;
0087 
0088 extern "C" {
0089 void alloc_monitor_start() { alloc_monitor_running_state() = true; }
0090 void alloc_monitor_stop() { alloc_monitor_running_state() = false; }
0091 
0092 //----------------------------------------------------------------
0093 //C memory functions
0094 
0095 #ifdef USE_LOCAL_MALLOC
0096 void* malloc(size_t size) noexcept {
0097   const auto original = Originals::malloc;
0098   Originals::init();
0099   if (not alloc_monitor_running_state()) {
0100     return original(size);
0101   }
0102   auto& reg = AllocMonitorRegistry::instance();
0103   return reg.allocCalled(
0104       size, [size, original]() { return original(size); }, [](auto ret) { return malloc_usable_size(ret); });
0105 }
0106 
0107 void* calloc(size_t nitems, size_t item_size) noexcept {
0108   const auto original = Originals::calloc;
0109   Originals::init();
0110   if (not alloc_monitor_running_state()) {
0111     return original(nitems, item_size);
0112   }
0113   auto& reg = AllocMonitorRegistry::instance();
0114   return reg.allocCalled(
0115       nitems * item_size,
0116       [nitems, item_size, original]() { return original(nitems, item_size); },
0117       [](auto ret) { return malloc_usable_size(ret); });
0118 }
0119 #else
0120 void* malloc(size_t size) noexcept {
0121   CMS_SA_ALLOW static const auto original = get<decltype(&::malloc)>("malloc");
0122   if (not alloc_monitor_running_state()) {
0123     return original(size);
0124   }
0125   auto& reg = AllocMonitorRegistry::instance();
0126   return reg.allocCalled(
0127       size, [size]() { return original(size); }, [](auto ret) { return malloc_usable_size(ret); });
0128 }
0129 
0130 void* calloc(size_t nitems, size_t item_size) noexcept {
0131   CMS_SA_ALLOW static const auto original = get<decltype(&::calloc)>("calloc");
0132   if (not alloc_monitor_running_state()) {
0133     return original(nitems, item_size);
0134   }
0135   auto& reg = AllocMonitorRegistry::instance();
0136   return reg.allocCalled(
0137       nitems * item_size,
0138       [nitems, item_size]() { return original(nitems, item_size); },
0139       [](auto ret) { return malloc_usable_size(ret); });
0140 }
0141 #endif
0142 
0143 void* realloc(void* ptr, size_t size) noexcept {
0144   CMS_SA_ALLOW static const auto original = get<decltype(&::realloc)>("realloc");
0145   if (not alloc_monitor_running_state()) {
0146     return original(ptr, size);
0147   }
0148   size_t oldsize = malloc_usable_size(ptr);
0149   void* ret;
0150   auto& reg = AllocMonitorRegistry::instance();
0151   {
0152     //incase this calls malloc/free
0153     [[maybe_unused]] auto g = reg.makeGuard();
0154     ret = original(ptr, size);
0155   }
0156   size_t used = malloc_usable_size(ret);
0157   if (used != oldsize) {
0158     reg.deallocCalled(
0159         ptr, [](auto) {}, [oldsize](auto) { return oldsize; });
0160     reg.allocCalled(
0161         size, []() { return nullptr; }, [used](auto) { return used; });
0162   }
0163   return ret;
0164 }
0165 
0166 void* aligned_alloc(size_t alignment, size_t size) noexcept {
0167   CMS_SA_ALLOW static const auto original = get<decltype(&::aligned_alloc)>("aligned_alloc");
0168   if (not alloc_monitor_running_state()) {
0169     return original(alignment, size);
0170   }
0171 
0172   auto& reg = AllocMonitorRegistry::instance();
0173   return reg.allocCalled(
0174       size,
0175       [alignment, size]() { return original(alignment, size); },
0176       [](auto ret) { return malloc_usable_size(ret); });
0177 }
0178 
0179 //used by tensorflow
0180 int posix_memalign(void** memptr, size_t alignment, size_t size) noexcept {
0181   CMS_SA_ALLOW static const auto original = get<decltype(&::posix_memalign)>("posix_memalign");
0182   if (not alloc_monitor_running_state()) {
0183     return original(memptr, alignment, size);
0184   }
0185 
0186   auto& reg = AllocMonitorRegistry::instance();
0187   int ret;
0188   reg.allocCalled(
0189       size,
0190       [&ret, memptr, alignment, size]() {
0191         ret = original(memptr, alignment, size);
0192         return *memptr;
0193       },
0194       [](auto ret) { return malloc_usable_size(ret); });
0195   return ret;
0196 }
0197 
0198 //used by libc
0199 void* memalign(size_t alignment, size_t size) noexcept {
0200   CMS_SA_ALLOW static const auto original = get<decltype(&::memalign)>("memalign");
0201   if (not alloc_monitor_running_state()) {
0202     return original(alignment, size);
0203   }
0204 
0205   auto& reg = AllocMonitorRegistry::instance();
0206   return reg.allocCalled(
0207       size,
0208       [alignment, size]() { return original(alignment, size); },
0209       [](auto ret) { return malloc_usable_size(ret); });
0210 }
0211 
0212 void free(void* ptr) noexcept {
0213   CMS_SA_ALLOW static const auto original = get<decltype(&::free)>("free");
0214   // ignore memory allocated from our static array at startup
0215   if (not is_local_alloc(ptr)) {
0216     if (not alloc_monitor_running_state()) {
0217       original(ptr);
0218       return;
0219     }
0220 
0221     auto& reg = AllocMonitorRegistry::instance();
0222     reg.deallocCalled(
0223         ptr, [](auto ptr) { original(ptr); }, [](auto ptr) { return malloc_usable_size(ptr); });
0224   }
0225 }
0226 }  // extern "C"
0227 
0228 //----------------------------------------------------------------
0229 //C++ memory functions
0230 
0231 #define CPP_MEM_OVERRIDE
0232 
0233 #if defined(CPP_MEM_OVERRIDE)
0234 #include <new>
0235 
0236 void* operator new(std::size_t size) {
0237   CMS_SA_ALLOW static const auto original = get<void* (*)(std::size_t)>("_Znwm");
0238   if (not alloc_monitor_running_state()) {
0239     return original(size);
0240   }
0241 
0242   auto& reg = AllocMonitorRegistry::instance();
0243   return reg.allocCalled(
0244       size, [size]() { return original(size); }, [](auto ret) { return malloc_usable_size(ret); });
0245 }  //_Znwm
0246 
0247 void operator delete(void* ptr) noexcept {
0248   CMS_SA_ALLOW static const auto original = get<void (*)(void*)>("_ZdlPv");
0249   if (not alloc_monitor_running_state()) {
0250     original(ptr);
0251     return;
0252   }
0253 
0254   auto& reg = AllocMonitorRegistry::instance();
0255   reg.deallocCalled(
0256       ptr, [](auto ptr) { original(ptr); }, [](auto ptr) { return malloc_usable_size(ptr); });
0257 }  //_ZdlPv
0258 
0259 void* operator new[](std::size_t size) {
0260   CMS_SA_ALLOW static const auto original = get<void* (*)(std::size_t)>("_Znam");
0261   if (not alloc_monitor_running_state()) {
0262     return original(size);
0263   }
0264 
0265   auto& reg = AllocMonitorRegistry::instance();
0266   return reg.allocCalled(
0267       size, [size]() { return original(size); }, [](auto ret) { return malloc_usable_size(ret); });
0268 }  //_Znam
0269 
0270 void operator delete[](void* ptr) noexcept {
0271   CMS_SA_ALLOW static const auto original = get<void (*)(void*)>("_ZdaPv");
0272 
0273   if (not alloc_monitor_running_state()) {
0274     original(ptr);
0275     return;
0276   }
0277   auto& reg = AllocMonitorRegistry::instance();
0278   reg.deallocCalled(
0279       ptr, [](auto ptr) { original(ptr); }, [](auto ptr) { return malloc_usable_size(ptr); });
0280 }  //_ZdaPv
0281 
0282 void* operator new(std::size_t size, std::align_val_t al) {
0283   CMS_SA_ALLOW static const auto original = get<void* (*)(std::size_t, std::align_val_t)>("_ZnwmSt11align_val_t");
0284   if (not alloc_monitor_running_state()) {
0285     return original(size, al);
0286   }
0287 
0288   auto& reg = AllocMonitorRegistry::instance();
0289   return reg.allocCalled(
0290       size, [size, al]() { return original(size, al); }, [](auto ret) { return malloc_usable_size(ret); });
0291 }  //_ZnwmSt11align_val_t
0292 
0293 void* operator new[](std::size_t size, std::align_val_t al) {
0294   CMS_SA_ALLOW static const auto original = get<void* (*)(std::size_t, std::align_val_t)>("_ZnamSt11align_val_t");
0295 
0296   if (not alloc_monitor_running_state()) {
0297     return original(size, al);
0298   }
0299 
0300   auto& reg = AllocMonitorRegistry::instance();
0301   return reg.allocCalled(
0302       size, [size, al]() { return original(size, al); }, [](auto ret) { return malloc_usable_size(ret); });
0303 }  //_ZnamSt11align_val_t
0304 
0305 void* operator new(std::size_t size, const std::nothrow_t& tag) noexcept {
0306   CMS_SA_ALLOW static const auto original =
0307       get<void* (*)(std::size_t, const std::nothrow_t&) noexcept>("_ZnwmRKSt9nothrow_t");
0308 
0309   if (not alloc_monitor_running_state()) {
0310     return original(size, tag);
0311   }
0312 
0313   auto& reg = AllocMonitorRegistry::instance();
0314   return reg.allocCalled(
0315       size, [size, &tag]() { return original(size, tag); }, [](auto ret) { return malloc_usable_size(ret); });
0316 }  //_ZnwmRKSt9nothrow_t
0317 
0318 void* operator new[](std::size_t size, const std::nothrow_t& tag) noexcept {
0319   CMS_SA_ALLOW static const auto original =
0320       get<void* (*)(std::size_t, const std::nothrow_t&) noexcept>("_ZnamRKSt9nothrow_t");
0321 
0322   if (not alloc_monitor_running_state()) {
0323     return original(size, tag);
0324   }
0325 
0326   auto& reg = AllocMonitorRegistry::instance();
0327   return reg.allocCalled(
0328       size, [size, &tag]() { return original(size, tag); }, [](auto ret) { return malloc_usable_size(ret); });
0329 }  //_ZnamRKSt9nothrow_t
0330 
0331 void* operator new(std::size_t size, std::align_val_t al, const std::nothrow_t& tag) noexcept {
0332   CMS_SA_ALLOW static const auto original =
0333       get<void* (*)(std::size_t, std::align_val_t, const std::nothrow_t&) noexcept>(
0334           "_ZnwmSt11align_val_tRKSt9nothrow_t");
0335 
0336   if (not alloc_monitor_running_state()) {
0337     return original(size, al, tag);
0338   }
0339 
0340   auto& reg = AllocMonitorRegistry::instance();
0341   return reg.allocCalled(
0342       size, [size, al, &tag]() { return original(size, al, tag); }, [](auto ret) { return malloc_usable_size(ret); });
0343 }  //_ZnwmSt11align_val_tRKSt9nothrow_t
0344 
0345 void* operator new[](std::size_t size, std::align_val_t al, const std::nothrow_t& tag) noexcept {
0346   CMS_SA_ALLOW static const auto original =
0347       get<void* (*)(std::size_t, std::align_val_t, const std::nothrow_t&) noexcept>(
0348           "_ZnamSt11align_val_tRKSt9nothrow_t");
0349 
0350   if (not alloc_monitor_running_state()) {
0351     return original(size, al, tag);
0352   }
0353 
0354   auto& reg = AllocMonitorRegistry::instance();
0355   return reg.allocCalled(
0356       size, [size, al, &tag]() { return original(size, al, tag); }, [](auto ret) { return malloc_usable_size(ret); });
0357 }  //_ZnamSt11align_val_tRKSt9nothrow_t
0358 
0359 void operator delete(void* ptr, std::align_val_t al) noexcept {
0360   CMS_SA_ALLOW static const auto original = get<void (*)(void*, std::align_val_t) noexcept>("_ZdlPvSt11align_val_t");
0361 
0362   if (not alloc_monitor_running_state()) {
0363     original(ptr, al);
0364     return;
0365   }
0366   auto& reg = AllocMonitorRegistry::instance();
0367   reg.deallocCalled(
0368       ptr, [al](auto ptr) { original(ptr, al); }, [](auto ptr) { return malloc_usable_size(ptr); });
0369 }  //_ZdlPvSt11align_val_t
0370 
0371 void operator delete[](void* ptr, std::align_val_t al) noexcept {
0372   CMS_SA_ALLOW static const auto original = get<void (*)(void*, std::align_val_t) noexcept>("_ZdaPvSt11align_val_t");
0373 
0374   if (not alloc_monitor_running_state()) {
0375     original(ptr, al);
0376     return;
0377   }
0378   auto& reg = AllocMonitorRegistry::instance();
0379   reg.deallocCalled(
0380       ptr, [al](auto ptr) { original(ptr, al); }, [](auto ptr) { return malloc_usable_size(ptr); });
0381 }  //_ZdaPvSt11align_val_t
0382 
0383 void operator delete(void* ptr, std::size_t sz) noexcept {
0384   CMS_SA_ALLOW static const auto original = get<void (*)(void*, std::size_t) noexcept>("_ZdlPvm");
0385 
0386   if (not alloc_monitor_running_state()) {
0387     original(ptr, sz);
0388     return;
0389   }
0390   auto& reg = AllocMonitorRegistry::instance();
0391   reg.deallocCalled(
0392       ptr, [sz](auto ptr) { original(ptr, sz); }, [](auto ptr) { return malloc_usable_size(ptr); });
0393 }  //_ZdlPvm
0394 
0395 void operator delete[](void* ptr, std::size_t sz) noexcept {
0396   CMS_SA_ALLOW static const auto original = get<void (*)(void*, std::size_t) noexcept>("_ZdaPvm");
0397 
0398   if (not alloc_monitor_running_state()) {
0399     original(ptr, sz);
0400     return;
0401   }
0402   auto& reg = AllocMonitorRegistry::instance();
0403   reg.deallocCalled(
0404       ptr, [sz](auto ptr) { original(ptr, sz); }, [](auto ptr) { return malloc_usable_size(ptr); });
0405 }  //_ZdaPvm
0406 
0407 void operator delete(void* ptr, std::size_t sz, std::align_val_t al) noexcept {
0408   CMS_SA_ALLOW static const auto original =
0409       get<void (*)(void*, std::size_t, std::align_val_t) noexcept>("_ZdlPvmSt11align_val_t");
0410 
0411   if (not alloc_monitor_running_state()) {
0412     original(ptr, sz, al);
0413     return;
0414   }
0415   auto& reg = AllocMonitorRegistry::instance();
0416   reg.deallocCalled(
0417       ptr, [sz, al](auto ptr) { original(ptr, sz, al); }, [](auto ptr) { return malloc_usable_size(ptr); });
0418 }  //_ZdlPvmSt11align_val_t
0419 
0420 void operator delete[](void* ptr, std::size_t sz, std::align_val_t al) noexcept {
0421   CMS_SA_ALLOW static const auto original =
0422       get<void (*)(void*, std::size_t, std::align_val_t) noexcept>("_ZdaPvmSt11align_val_t");
0423 
0424   if (not alloc_monitor_running_state()) {
0425     original(ptr, sz, al);
0426     return;
0427   }
0428   auto& reg = AllocMonitorRegistry::instance();
0429   reg.deallocCalled(
0430       ptr, [sz, al](auto ptr) { original(ptr, sz, al); }, [](auto ptr) { return malloc_usable_size(ptr); });
0431 }  //_ZdaPvmSt11align_val_t
0432 
0433 void operator delete(void* ptr, const std::nothrow_t& tag) noexcept {
0434   CMS_SA_ALLOW static const auto original =
0435       get<void (*)(void*, const std::nothrow_t&) noexcept>("_ZdlPvRKSt9nothrow_t");
0436 
0437   if (not alloc_monitor_running_state()) {
0438     original(ptr, tag);
0439     return;
0440   }
0441   auto& reg = AllocMonitorRegistry::instance();
0442   reg.deallocCalled(
0443       ptr, [&tag](auto ptr) { original(ptr, tag); }, [](auto ptr) { return malloc_usable_size(ptr); });
0444 }  //_ZdlPvRKSt9nothrow_t
0445 
0446 void operator delete[](void* ptr, const std::nothrow_t& tag) noexcept {
0447   CMS_SA_ALLOW static const auto original =
0448       get<void (*)(void*, const std::nothrow_t&) noexcept>("_ZdaPvRKSt9nothrow_t");
0449 
0450   if (not alloc_monitor_running_state()) {
0451     original(ptr, tag);
0452     return;
0453   }
0454   auto& reg = AllocMonitorRegistry::instance();
0455   reg.deallocCalled(
0456       ptr, [&tag](auto ptr) { original(ptr, tag); }, [](auto ptr) { return malloc_usable_size(ptr); });
0457 }  //_ZdaPvRKSt9nothrow_t
0458 
0459 void operator delete(void* ptr, std::align_val_t al, const std::nothrow_t& tag) noexcept {
0460   CMS_SA_ALLOW static const auto original =
0461       get<void (*)(void*, std::align_val_t, const std::nothrow_t&) noexcept>("_ZdlPvSt11align_val_tRKSt9nothrow_t");
0462 
0463   if (not alloc_monitor_running_state()) {
0464     original(ptr, al, tag);
0465     return;
0466   }
0467   auto& reg = AllocMonitorRegistry::instance();
0468   reg.deallocCalled(
0469       ptr, [al, &tag](auto ptr) { original(ptr, al, tag); }, [](auto ptr) { return malloc_usable_size(ptr); });
0470 }  //_ZdlPvSt11align_val_tRKSt9nothrow_t
0471 
0472 void operator delete[](void* ptr, std::align_val_t al, const std::nothrow_t& tag) noexcept {
0473   CMS_SA_ALLOW static const auto original =
0474       get<void (*)(void*, std::align_val_t, const std::nothrow_t&) noexcept>("_ZdaPvSt11align_val_tRKSt9nothrow_t");
0475 
0476   if (not alloc_monitor_running_state()) {
0477     original(ptr, al, tag);
0478     return;
0479   }
0480   auto& reg = AllocMonitorRegistry::instance();
0481   reg.deallocCalled(
0482       ptr, [al, &tag](auto ptr) { original(ptr, al, tag); }, [](auto ptr) { return malloc_usable_size(ptr); });
0483 }  //_ZdaPvSt11align_val_tRKSt9nothrow_t
0484 
0485 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.2.1 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!