Back to home page

Project CMSSW displayed by LXR
 
 

    

File indexing completed on 2025-01-28 23:24:28 

0001 // -*- C++ -*-
0002 //
0003 // Package:     Services
0004 // Class  :     Memory
0005 //
0006 // Implementation:
0007 //
0008 // Original Author:  Jim Kowalkowski
0009 //
0010 // Change Log
0011 //
0012 // 1 - Apr 25, 2008 M. Fischler
0013 //        Collect event summary information and output to XML file and logger
0014 //        at the end of the job.  Involves split-up of updateAndPrint method.
0015 //
0016 // 2 - May 7, 2008 M. Fischler
0017 //      Collect module summary information and output to XML file and logger
0018 //        at the end of the job.
0019 //
0020 // 3 - Jan 14, 2009 Natalia Garcia Nebot
0021 //        Added:        - Average rate of growth in RSS and peak value attained.
0022 //                - Average rate of growth in VSize over time, Peak VSize
0023 //
0024 //
0025 #include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
0026 
0027 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0028 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0029 #include "DataFormats/Provenance/interface/EventID.h"
0030 #include "FWCore/Services/plugins/ProcInfoFetcher.h"
0031 
0032 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0033 #include "FWCore/Framework/interface/Event.h"
0034 #include "FWCore/MessageLogger/interface/JobReport.h"
0035 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0036 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0037 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0038 #include "FWCore/ServiceRegistry/interface/Service.h"
0039 #include "FWCore/ServiceRegistry/interface/StreamContext.h"
0040 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0041 #include "FWCore/Utilities/interface/Exception.h"
0042 #include "FWCore/Utilities/interface/get_underlying_safe.h"
0043 
0044 #include <array>
0045 #include <cstring>
0046 #include <iostream>
0047 #include <memory>
0048 #ifdef __linux__
0049 #include <malloc.h>
0050 #endif
0051 #include <sstream>
0052 //#include <stdio.h>
0053 #include <string>
0054 //#include <string.h>
0055 
0056 #include <cstdio>
0057 #include <atomic>
0058 #include <optional>
0059 
0060 // for jemalloc queries
0061 #include <dlfcn.h>
0062 extern "C" {
0063 typedef int (*mallctl_t)(const char* name, void* oldp, size_t* oldlenp, void* newp, size_t newlen);
0064 }
0065 
0066 namespace edm {
0067   namespace service {
0068     enum class SmapsSection {
0069       kSharedObject = 0,
0070       kPcm = 1,
0071       kOtherFile = 2,
0072       kStack = 3,
0073       kMmap = 4,
0074       kOther = 5,
0075       kSize = 6
0076     };
0077     struct smapsInfo {
0078       double private_ = 0;        // in MB
0079       double pss_ = 0;            // in MB
0080       double anonHugePages_ = 0;  // in MB
0081 
0082       static constexpr auto sectionsSize_ = static_cast<unsigned>(SmapsSection::kSize);
0083       std::array<double, sectionsSize_> sectionRss_{};    // in MB
0084       std::array<double, sectionsSize_> sectionVSize_{};  // in MB
0085     };
0086     struct JemallocInfo {
0087       double allocated = 0;  // in MB
0088       double active = 0;     // in MB
0089       double resident = 0;   // in MB
0090       double mapped = 0;     // in MB
0091       double metadata = 0;   // in MB
0092     };
0093 
0094     class SimpleMemoryCheck {
0095     public:
0096       SimpleMemoryCheck(const ParameterSet&, ActivityRegistry&);
0097       ~SimpleMemoryCheck();
0098 
0099       static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0100 
0101       void preSourceConstruction(const ModuleDescription&);
0102       void postSourceConstruction(const ModuleDescription&);
0103       void postSourceEvent(StreamID);
0104 
0105       void postBeginJob();
0106 
0107       void postEvent(StreamContext const&);
0108 
0109       void postModuleBeginJob(const ModuleDescription&);
0110       void postModuleConstruction(const ModuleDescription&);
0111 
0112       void preModule(StreamContext const&, ModuleCallingContext const&);
0113       void postModule(StreamContext const&, ModuleCallingContext const&);
0114 
0115       void earlyTermination();
0116 
0117       void postEndJob();
0118 
0119       void startSamplingThread();
0120       void stopSamplingThread();
0121 
0122     private:
0123       ProcInfo fetch();
0124       smapsInfo fetchSmaps();
0125       JemallocInfo fetchJemalloc() const;
0126       double pageSize() const { return pg_size_; }
0127       double averageGrowthRate(double current, double past, int count);
0128       void update();
0129       void updateMax();
0130       void andPrintAlways(const std::string& type,
0131                           const std::string& mdlabel,
0132                           const std::string& mdname,
0133                           bool includeSmapsAndJe = false) const;
0134       void andPrint(const std::string& type, const std::string& mdlabel, const std::string& mdname) const;
0135       void updateAndPrint(const std::string& type, const std::string& mdlabel, const std::string& mdname);
0136 
0137       // Upon success returns an optional without value
0138       // Upon failure returns the name of the file the function attempted to open
0139       std::optional<std::string> openFilesNoThrow();
0140       void openFiles();
0141 
0142       char const* smapsLineBuffer() const { return get_underlying_safe(smapsLineBuffer_); }
0143       char*& smapsLineBuffer() { return get_underlying_safe(smapsLineBuffer_); }
0144 
0145       ProcInfo a_;
0146       ProcInfo b_;
0147       ProcInfo max_;
0148       edm::propagate_const<ProcInfo*> current_;
0149       edm::propagate_const<ProcInfo*> previous_;
0150 
0151       smapsInfo currentSmaps_;
0152 
0153       ProcInfoFetcher piFetcher_;
0154       double pg_size_;
0155       int num_to_skip_;
0156       //options
0157       bool showMallocInfo_;
0158       bool showJemallocInfo_;
0159       bool oncePerEventMode_;
0160       bool printEachTime_;
0161       bool jobReportOutputOnly_;
0162       bool monitorPssAndPrivate_;
0163       std::atomic<int> count_;
0164       unsigned int sampleEveryNSeconds_;
0165       std::optional<std::thread> samplingThread_;
0166       std::atomic<bool> stopThread_ = false;
0167       std::atomic<edm::EventID> mostRecentlyStartedEvent_;
0168 
0169       mallctl_t je_mallctl = nullptr;
0170 
0171       //smaps
0172       edm::propagate_const<FILE*> smapsFile_ = nullptr;
0173       edm::propagate_const<char*> smapsLineBuffer_;
0174       size_t smapsLineBufferLen_;
0175 
0176       //Rates of growth
0177       double growthRateVsize_;
0178       double growthRateRss_;
0179 
0180       // Event summary statistics               changeLog 1
0181       struct SignificantEvent {
0182         edm::EventID event;
0183         double vsize = 0;
0184         double deltaVsize = 0;
0185         double rss = 0;
0186         double deltaRss = 0;
0187         double privateSize = 0;
0188         double pss = 0;
0189         double anonHugePages = 0;
0190         std::optional<JemallocInfo> jemalloc;
0191         int count = 0;
0192         bool monitorPssAndPrivate = false;
0193         SignificantEvent() = default;
0194         void set(double deltaV, double deltaR, edm::EventID const& e, SimpleMemoryCheck* t) {
0195           count = t->count_;
0196           vsize = t->current_->vsize;
0197           deltaVsize = deltaV;
0198           rss = t->current_->rss;
0199           deltaRss = deltaR;
0200           monitorPssAndPrivate = t->monitorPssAndPrivate_;
0201           if (monitorPssAndPrivate) {
0202             privateSize = t->currentSmaps_.private_;
0203             pss = t->currentSmaps_.pss_;
0204             anonHugePages = t->currentSmaps_.anonHugePages_;
0205           }
0206           if (t->showJemallocInfo_) {
0207             jemalloc = t->fetchJemalloc();
0208           }
0209           event = e;
0210         }
0211       };  // SignificantEvent
0212       friend struct SignificantEvent;
0213       friend std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantEvent const& se);
0214 
0215       /*
0216        Significative events for deltaVsize:
0217        - eventM_: Event which makes the biggest value for deltaVsize
0218        - eventL1_: Event which makes the second biggest value for deltaVsize
0219        - eventL2_: Event which makes the third biggest value for deltaVsize
0220        - eventR1_: Event which makes the second biggest value for deltaVsize
0221        - eventR2_: Event which makes the third biggest value for deltaVsize
0222        M>L1>L2 and M>R1>R2
0223        Unknown relation between Ls and Rs events ???????
0224        Significative events for vsize:
0225        - eventT1_: Event whith the biggest value for vsize
0226        - eventT2_: Event whith the second biggest value for vsize
0227        - eventT3_: Event whith the third biggest value for vsize
0228        T1>T2>T3
0229        */
0230       SignificantEvent eventM_;
0231       SignificantEvent eventL1_;
0232       SignificantEvent eventL2_;
0233       SignificantEvent eventR1_;
0234       SignificantEvent eventR2_;
0235       SignificantEvent eventT1_;
0236       SignificantEvent eventT2_;
0237       SignificantEvent eventT3_;
0238 
0239       /*
0240        Significative event for deltaRss:
0241        - eventRssT1_: Event whith the biggest value for rss
0242        - eventRssT2_: Event whith the second biggest value for rss
0243        - eventRssT3_: Event whith the third biggest value for rss
0244        T1>T2>T3
0245        Significative events for deltaRss:
0246        - eventDeltaRssT1_: Event whith the biggest value for deltaRss
0247        - eventDeltaRssT2_: Event whith the second biggest value for deltaRss
0248        - eventDeltaRssT3_: Event whith the third biggest value for deltaRss
0249        T1>T2>T3
0250        */
0251       SignificantEvent eventRssT1_;
0252       SignificantEvent eventRssT2_;
0253       SignificantEvent eventRssT3_;
0254       SignificantEvent eventDeltaRssT1_;
0255       SignificantEvent eventDeltaRssT2_;
0256       SignificantEvent eventDeltaRssT3_;
0257 
0258       void updateEventStats(edm::EventID const& e);
0259       std::string eventStatOutput(std::string title, SignificantEvent const& e) const;
0260       void eventStatOutput(std::string title, SignificantEvent const& e, std::map<std::string, std::string>& m) const;
0261       std::string mallOutput(std::string title, size_t const& n) const;
0262 
0263       // Module summary statistices
0264       struct SignificantModule {
0265         int postEarlyCount;
0266         double totalDeltaVsize;
0267         double maxDeltaVsize;
0268         edm::EventID eventMaxDeltaV;
0269         double totalEarlyVsize;
0270         double maxEarlyVsize;
0271         SignificantModule()
0272             : postEarlyCount(0),
0273               totalDeltaVsize(0),
0274               maxDeltaVsize(0),
0275               eventMaxDeltaV(),
0276               totalEarlyVsize(0),
0277               maxEarlyVsize(0) {}
0278         void set(double deltaV, bool early);
0279       };  // SignificantModule
0280       friend struct SignificantModule;
0281       friend std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantModule const& se);
0282       bool moduleSummaryRequested_;
0283       typedef std::map<std::string, SignificantModule> SignificantModulesMap;
0284       SignificantModulesMap modules_;
0285       double moduleEntryVsize_;
0286       void updateModuleMemoryStats(SignificantModule& m, double dv, edm::EventID const&);
0287 
0288       //Used to guarantee we only do one measurement at a time
0289       std::atomic<bool> measurementUnderway_;
0290       std::atomic<bool> moduleMeasurementUnderway_;
0291       std::atomic<unsigned int> moduleStreamID_;
0292       std::atomic<unsigned int> moduleID_;
0293 
0294     };  // SimpleMemoryCheck
0295 
0296     std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantEvent const& se);
0297 
0298     std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantModule const& se);
0299 
0300   }  // namespace service
0301 }  // namespace edm
0302 
0303 #ifdef __linux__
0304 #define LINUX 1
0305 #endif
0306 
0307 #include <fcntl.h>
0308 #include <unistd.h>
0309 
0310 namespace edm {
0311   namespace service {
0312 
0313     static std::string d2str(double d) {
0314       std::ostringstream t;
0315       t << d;
0316       return t.str();
0317     }
0318 
0319     static std::string i2str(int i) {
0320       std::ostringstream t;
0321       t << i;
0322       return t.str();
0323     }
0324 
0325     ProcInfo SimpleMemoryCheck::fetch() { return piFetcher_.fetch(); }
0326 
0327     smapsInfo SimpleMemoryCheck::fetchSmaps() {
0328       smapsInfo ret;
0329 #ifdef LINUX
0330       fseek(smapsFile_, 0, SEEK_SET);
0331       ssize_t read;
0332       SmapsSection section = SmapsSection::kOther;
0333 
0334       /*
0335        The format of the report is
0336        Private_Clean:        0 kB
0337        Private_Dirty:       72 kB
0338        Swap:                 0 kB
0339        Pss:                 72 kB
0340        AnonHugePages:    10240 kB
0341        */
0342 
0343       while ((read = getline(&smapsLineBuffer(), &smapsLineBufferLen_, smapsFile_)) != -1) {
0344         if (read > 14) {
0345           // Are we in a line that defines a mapping?
0346           // (a character following ':' is not a space)
0347           if (char const* ret = strchr(smapsLineBuffer_, ':'); ret != nullptr and *(ret + 1) != ' ') {
0348             ret = strrchr(smapsLineBuffer_, ' ');
0349             if (ret == nullptr) {
0350               // shouldn't happen, but let's protect anyway
0351               section = SmapsSection::kOther;
0352             } else if (*(ret + 1) == '\n') {
0353               // no "path" element
0354               section = SmapsSection::kMmap;
0355             } else if (*(ret + 1) == '/') {
0356               // "path" starts with '/', assume it's file
0357               // differentiate shared object and .pcm files
0358               auto len = strlen(ret);
0359               if (0 == strncmp(ret + len - 5, ".pcm", 4)) {
0360                 section = SmapsSection::kPcm;
0361               } else if (strstr(ret, ".so") != nullptr) {
0362                 section = SmapsSection::kSharedObject;
0363               } else {
0364                 section = SmapsSection::kOtherFile;
0365               }
0366             } else if (0 == strncmp("[stack]", ret + 1, 7)) {
0367               section = SmapsSection::kStack;
0368             } else {
0369               section = SmapsSection::kOther;
0370             }
0371             continue;
0372           }
0373 
0374           //Private
0375           if (0 == strncmp("Private_", smapsLineBuffer_, 8)) {
0376             unsigned int value = atoi(smapsLineBuffer_ + 14);
0377             //Convert from kB to MB
0378             ret.private_ += static_cast<double>(value) / 1024.;
0379           } else if (0 == strncmp("Pss:", smapsLineBuffer_, 4)) {
0380             unsigned int value = atoi(smapsLineBuffer_ + 4);
0381             //Convert from kB to MB
0382             ret.pss_ += static_cast<double>(value) / 1024.;
0383           } else if (0 == strncmp("AnonHugePages:", smapsLineBuffer_, 14)) {
0384             unsigned int value = atoi(smapsLineBuffer_ + 14);
0385             ret.anonHugePages_ += static_cast<double>(value) / 1024.;
0386           } else if (0 == strncmp("Rss:", smapsLineBuffer_, 4)) {
0387             unsigned int value = atoi(smapsLineBuffer_ + 4);
0388             //Convert from kB to MB
0389             ret.sectionRss_[static_cast<unsigned>(section)] += static_cast<double>(value) / 1024.;
0390           } else if (0 == strncmp("Size:", smapsLineBuffer_, 5)) {
0391             unsigned int value = atoi(smapsLineBuffer_ + 5);
0392             //Convert from kB to MB
0393             ret.sectionVSize_[static_cast<unsigned>(section)] += static_cast<double>(value) / 1024.;
0394           }
0395         }
0396       }
0397 #endif
0398       return ret;
0399     }
0400 
0401     JemallocInfo SimpleMemoryCheck::fetchJemalloc() const {
0402       JemallocInfo info;
0403       if (je_mallctl) {
0404         // refresh stats
0405         uint64_t epoch = 1;
0406         size_t e_len = sizeof(uint64_t);
0407         if (je_mallctl("epoch", &epoch, &e_len, &epoch, e_len) != 0) {
0408           return info;
0409         }
0410 
0411         // query values
0412         size_t allocated, active, resident, mapped, metadata;
0413         size_t len = sizeof(size_t);
0414         if (je_mallctl("stats.allocated", &allocated, &len, nullptr, 0) != 0) {
0415           return info;
0416         }
0417         if (je_mallctl("stats.active", &active, &len, nullptr, 0) != 0) {
0418           return info;
0419         }
0420         if (je_mallctl("stats.resident", &resident, &len, nullptr, 0) != 0) {
0421           return info;
0422         }
0423         if (je_mallctl("stats.mapped", &mapped, &len, nullptr, 0) != 0) {
0424           return info;
0425         }
0426         if (je_mallctl("stats.metadata", &metadata, &len, nullptr, 0) != 0) {
0427           return info;
0428         }
0429         info.allocated = allocated / 1024.0 / 1024.0;
0430         info.active = active / 1024.0 / 1024.0;
0431         info.resident = resident / 1024.0 / 1024.0;
0432         info.mapped = mapped / 1024.0 / 1024.0;
0433         info.metadata = metadata / 1024.0 / 1024.0;
0434       }
0435       return info;
0436     }
0437 
0438     double SimpleMemoryCheck::averageGrowthRate(double current, double past, int count) {
0439       return (current - past) / (double)std::max(count, 1);
0440     }
0441 
0442     SimpleMemoryCheck::SimpleMemoryCheck(ParameterSet const& iPS, ActivityRegistry& iReg)
0443         : a_(),
0444           b_(),
0445           current_(&a_),
0446           previous_(&b_),
0447           pg_size_(sysconf(_SC_PAGESIZE)),  // getpagesize()
0448           num_to_skip_(iPS.getUntrackedParameter<int>("ignoreTotal")),
0449           showMallocInfo_(iPS.getUntrackedParameter<bool>("showMallocInfo")),
0450           showJemallocInfo_(iPS.getUntrackedParameter<bool>("showJemallocInfo")),
0451           oncePerEventMode_(iPS.getUntrackedParameter<bool>("oncePerEventMode")),
0452           printEachTime_(oncePerEventMode_ or iPS.getUntrackedParameter<bool>("printEachSample")),
0453           jobReportOutputOnly_(iPS.getUntrackedParameter<bool>("jobReportOutputOnly")),
0454           monitorPssAndPrivate_(iPS.getUntrackedParameter<bool>("monitorPssAndPrivate")),
0455           count_(),
0456           sampleEveryNSeconds_(iPS.getUntrackedParameter<unsigned int>("sampleEveryNSeconds")),
0457           smapsFile_(nullptr),
0458           smapsLineBuffer_(nullptr),
0459           smapsLineBufferLen_(0),
0460           growthRateVsize_(),
0461           growthRateRss_(),
0462           moduleSummaryRequested_(iPS.getUntrackedParameter<bool>("moduleMemorySummary")),
0463           measurementUnderway_(false) {
0464       // changelog 2
0465       // pg_size = (double)getpagesize();
0466       std::ostringstream ost;
0467 
0468       if (monitorPssAndPrivate_) {
0469         openFiles();
0470       }
0471 
0472       iReg.watchPostEndJob(this, &SimpleMemoryCheck::postEndJob);
0473       // A possible source for early termination is a signal from WM
0474       // when the job's memory use exceeds their limit
0475       iReg.watchPreSourceEarlyTermination([this](TerminationOrigin) { earlyTermination(); });
0476       iReg.watchPreGlobalEarlyTermination([this](GlobalContext const&, TerminationOrigin) { earlyTermination(); });
0477       iReg.watchPreStreamEarlyTermination([this](StreamContext const&, TerminationOrigin) { earlyTermination(); });
0478 
0479       if (sampleEveryNSeconds_ > 0) {
0480         if (oncePerEventMode_) {
0481           throw edm::Exception(edm::errors::Configuration)
0482               << "'sampleEventNSeconds' and 'oncePerEventMode' cannot be used together";
0483         }
0484         if (moduleSummaryRequested_) {
0485           throw edm::Exception(edm::errors::Configuration)
0486               << "'sampleEventNSeconds' and 'moduleSummaryRequested' cannot be used together";
0487         }
0488         iReg.watchPostBeginJob(this, &SimpleMemoryCheck::startSamplingThread);
0489         iReg.watchPreEndJob(this, &SimpleMemoryCheck::stopSamplingThread);
0490         iReg.watchPreEvent([this](auto const& iContext) { mostRecentlyStartedEvent_.store(iContext.eventID()); });
0491         return;
0492       }
0493 
0494       iReg.watchPostEvent(this, &SimpleMemoryCheck::postEvent);
0495 
0496       if (!oncePerEventMode_) {  // default, prints on increases
0497         iReg.watchPreSourceConstruction(this, &SimpleMemoryCheck::preSourceConstruction);
0498         iReg.watchPostSourceConstruction(this, &SimpleMemoryCheck::postSourceConstruction);
0499         iReg.watchPostSourceEvent(this, &SimpleMemoryCheck::postSourceEvent);
0500         iReg.watchPostModuleConstruction(this, &SimpleMemoryCheck::postModuleConstruction);
0501         iReg.watchPostModuleBeginJob(this, &SimpleMemoryCheck::postModuleBeginJob);
0502         iReg.watchPostModuleEvent(this, &SimpleMemoryCheck::postModule);
0503         iReg.watchPostBeginJob(this, &SimpleMemoryCheck::postBeginJob);
0504       }
0505       if (moduleSummaryRequested_) {  // changelog 2
0506         iReg.watchPreModuleEvent(this, &SimpleMemoryCheck::preModule);
0507         if (oncePerEventMode_) {
0508           iReg.watchPostModuleEvent(this, &SimpleMemoryCheck::postModule);
0509         }
0510       }
0511 
0512       // The following are not currenty used/implemented below for either
0513       // of the print modes (but are left here for reference)
0514       //  iReg.watchPostBeginJob(this,
0515       //       &SimpleMemoryCheck::postBeginJob);
0516       //  iReg.watchPreProcessEvent(this,
0517       //       &SimpleMemoryCheck::preEventProcessing);
0518       //  iReg.watchPreModule(this,
0519       //       &SimpleMemoryCheck::preModule);
0520 
0521       if (showJemallocInfo_) {
0522         // jemalloc's mallctl(), if we use jemalloc
0523         je_mallctl = reinterpret_cast<mallctl_t>(::dlsym(RTLD_DEFAULT, "mallctl"));
0524         if (je_mallctl == nullptr) {
0525           showJemallocInfo_ = false;
0526         }
0527       }
0528     }
0529 
0530     SimpleMemoryCheck::~SimpleMemoryCheck() {
0531 #ifdef LINUX
0532       if (nullptr != smapsFile_) {
0533         fclose(smapsFile_);
0534       }
0535 #endif
0536       if (smapsLineBuffer_) {
0537         //getline will create the memory using malloc
0538         free(smapsLineBuffer_);
0539       }
0540     }
0541 
0542     void SimpleMemoryCheck::fillDescriptions(ConfigurationDescriptions& descriptions) {
0543       ParameterSetDescription desc;
0544       desc.addUntracked<int>("ignoreTotal", 1)
0545           ->setComment("Number of events/samples to finish before starting measuring and reporting.");
0546       desc.addUntracked<unsigned int>("sampleEveryNSeconds", 0)
0547           ->setComment(
0548               "Use a special thread to sample memory at the set rate. A value of 0 means no sampling. This option "
0549               "cannot be used with 'oncePerEventMode' or 'moduleMemorySummary'.");
0550       desc.addUntracked<bool>("printEachSample", false)
0551           ->setComment("If sampling on, print each sample taken else will print only when sample is the largest seen.");
0552       desc.addUntracked<bool>("showMallocInfo", false);
0553       desc.addUntracked<bool>("showJemallocInfo", true)
0554           ->setComment(
0555               "If enabled and jemalloc is being used, print high-level jemalloc statistics at the early termination "
0556               "and endJob printouts as well as for the peak VSIZE and RSS -using records.");
0557       desc.addUntracked<bool>("oncePerEventMode", false)
0558           ->setComment(
0559               "Only check memory at the end of each event. Not as useful in multi-threaded job as other running events "
0560               "contribute.");
0561       desc.addUntracked<bool>("jobReportOutputOnly", false);
0562       desc.addUntracked<bool>("monitorPssAndPrivate", false);
0563       desc.addUntracked<bool>("moduleMemorySummary", false)
0564           ->setComment(
0565               "Track significant memory events for each module. This does not work well in multi-threaded jobs.");
0566       descriptions.add("SimpleMemoryCheck", desc);
0567     }
0568 
0569     std::optional<std::string> SimpleMemoryCheck::openFilesNoThrow() {
0570 #ifdef LINUX
0571       std::ostringstream smapsNameOst;
0572       smapsNameOst << "/proc/" << getpid() << "/smaps";
0573       auto smapsName = smapsNameOst.str();
0574       if ((smapsFile_ = fopen(smapsName.c_str(), "r")) == nullptr) {
0575         return smapsName;
0576       }
0577 #endif
0578       return {};
0579     }
0580 
0581     void SimpleMemoryCheck::openFiles() {
0582       auto smapsFileNameIfFailed = openFilesNoThrow();
0583       if (smapsFileNameIfFailed.has_value()) {
0584         throw Exception(errors::Configuration) << "Failed to open smaps file " << *smapsFileNameIfFailed << std::endl;
0585       }
0586     }
0587 
0588     void SimpleMemoryCheck::postBeginJob() {
0589       growthRateVsize_ = current_->vsize;
0590       growthRateRss_ = current_->rss;
0591     }
0592 
0593     void SimpleMemoryCheck::preSourceConstruction(ModuleDescription const& md) {
0594       bool expected = false;
0595       if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0596         std::shared_ptr<void> guard(
0597             nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0598         updateAndPrint("pre-ctor", md.moduleLabel(), md.moduleName());
0599       }
0600     }
0601 
0602     void SimpleMemoryCheck::postSourceConstruction(ModuleDescription const& md) {
0603       bool expected = false;
0604       if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0605         std::shared_ptr<void> guard(
0606             nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0607         updateAndPrint("ctor", md.moduleLabel(), md.moduleName());
0608       }
0609     }
0610 
0611     void SimpleMemoryCheck::postSourceEvent(StreamID sid) {
0612       bool expected = false;
0613       if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0614         std::shared_ptr<void> guard(
0615             nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0616         updateAndPrint("module", "source", "source");
0617       }
0618     }
0619 
0620     void SimpleMemoryCheck::postModuleConstruction(ModuleDescription const& md) {
0621       bool expected = false;
0622       if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0623         std::shared_ptr<void> guard(
0624             nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0625         updateAndPrint("ctor", md.moduleLabel(), md.moduleName());
0626       }
0627     }
0628 
0629     void SimpleMemoryCheck::postModuleBeginJob(ModuleDescription const& md) {
0630       bool expected = false;
0631       if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0632         std::shared_ptr<void> guard(
0633             nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0634         updateAndPrint("beginJob", md.moduleLabel(), md.moduleName());
0635       }
0636     }
0637 
0638     void SimpleMemoryCheck::earlyTermination() {
0639       bool expected = false;
0640       while (not measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0641         expected = false;
0642       }
0643       std::shared_ptr<void> guard(
0644           nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0645       if (not smapsFile_) {
0646         openFilesNoThrow();
0647       }
0648       if (smapsFile_) {
0649         currentSmaps_ = fetchSmaps();
0650       }
0651       update();
0652       andPrintAlways("earlyTermination", "", "", true);
0653       updateMax();
0654     }
0655 
0656     void SimpleMemoryCheck::startSamplingThread() {
0657       samplingThread_ = std::thread{[this]() {
0658         while (not stopThread_) {
0659           std::this_thread::sleep_for(std::chrono::duration<unsigned int>(sampleEveryNSeconds_));
0660           ++count_;
0661           update();
0662           if (monitorPssAndPrivate_) {
0663             currentSmaps_ = fetchSmaps();
0664           }
0665           auto e = mostRecentlyStartedEvent_.load();
0666           andPrint("sampling", "", "");
0667           updateEventStats(e);
0668           updateMax();
0669         }
0670       }};
0671     }
0672     void SimpleMemoryCheck::stopSamplingThread() {
0673       stopThread_ = true;
0674       samplingThread_->join();
0675     }
0676 
0677     void SimpleMemoryCheck::postEndJob() {
0678       if (not jobReportOutputOnly_) {
0679         LogAbsolute log("MemoryReport");
0680 
0681         update();
0682         log << "MemoryReport> EndJob: virtual size " << current_->vsize << " Mbytes, RSS " << current_->rss
0683             << " Mbytes";
0684         // extract smaps information if file open succeeded
0685         if (not smapsFile_) {
0686           openFilesNoThrow();
0687         }
0688         if (smapsFile_) {
0689           currentSmaps_ = fetchSmaps();
0690           auto soRss = currentSmaps_.sectionRss_[static_cast<unsigned>(SmapsSection::kSharedObject)];
0691           auto pcmRss = currentSmaps_.sectionRss_[static_cast<unsigned>(SmapsSection::kPcm)];
0692           auto otherFileRss = currentSmaps_.sectionRss_[static_cast<unsigned>(SmapsSection::kOtherFile)];
0693           auto mmapRss = currentSmaps_.sectionRss_[static_cast<unsigned>(SmapsSection::kMmap)];
0694           auto soVSize = currentSmaps_.sectionVSize_[static_cast<unsigned>(SmapsSection::kSharedObject)];
0695           auto pcmVSize = currentSmaps_.sectionVSize_[static_cast<unsigned>(SmapsSection::kPcm)];
0696           auto otherFileVSize = currentSmaps_.sectionVSize_[static_cast<unsigned>(SmapsSection::kOtherFile)];
0697           auto mmapVSize = currentSmaps_.sectionVSize_[static_cast<unsigned>(SmapsSection::kMmap)];
0698           log << ", PSS " << currentSmaps_.pss_ << " MBytes, Private " << currentSmaps_.private_ << "\n AnonHugePages "
0699               << currentSmaps_.anonHugePages_ << " Mbytes\n"
0700               << " mmapped memory pages " << mmapVSize << " Mbytes (VSize), " << mmapRss << " MBytes (RSS)\n"
0701               << " mmapped file pages " << (soVSize + pcmVSize + otherFileVSize) << " Mbytes (VSize), "
0702               << (soRss + pcmRss + otherFileRss) << " MBytes (RSS)\n"
0703               << "  of which .so's " << soVSize << " Mbytes (VSize), " << soRss << " MBytes (RSS)\n"
0704               << "  of which PCM's " << pcmVSize << " Mbytes (VSize), " << pcmRss << " MBytes (RSS)\n"
0705               << "  of which other " << otherFileVSize << " Mbytes (VSize), " << otherFileRss << " MBytes (RSS)";
0706         }
0707         if (showJemallocInfo_) {
0708           auto info = fetchJemalloc();
0709           log << "\n Jemalloc allocated " << info.allocated << " MBytes, active " << info.active
0710               << " MBytes\n  resident " << info.resident << " Mbytes, mapped " << info.mapped << " Mbytes\n  metadata "
0711               << info.metadata << " Mbytes";
0712         }
0713         log << "\n";
0714 
0715         auto logJemalloc = [&log](std::optional<JemallocInfo> const& info) {
0716           if (info.has_value()) {
0717             log << "\n Jemalloc allocated " << info->allocated << " active " << info->active << " resident "
0718                 << info->resident << " mapped " << info->mapped << " metadata " << info->metadata;
0719           }
0720         };
0721 
0722         log << "MemoryReport> Peak virtual size " << eventT1_.vsize << " Mbytes (RSS " << eventT1_.rss << ")";
0723         logJemalloc(eventT1_.jemalloc);
0724         log << "\n Key events increasing vsize: \n"
0725             << eventL2_ << "\n"
0726             << eventL1_ << "\n"
0727             << eventM_ << "\n"
0728             << eventR1_ << "\n"
0729             << eventR2_ << "\n"
0730             << eventT3_ << "\n"
0731             << eventT2_ << "\n"
0732             << eventT1_ << "\nMemoryReport> Peak rss size " << eventRssT1_.rss << " Mbytes (VSIZE " << eventRssT1_.vsize
0733             << ")";
0734         ;
0735         logJemalloc(eventRssT1_.jemalloc);
0736         log << "\n Key events increasing rss:\n"
0737             << eventRssT3_ << "\n"
0738             << eventRssT2_ << "\n"
0739             << eventRssT1_ << "\n"
0740             << eventDeltaRssT3_ << "\n"
0741             << eventDeltaRssT2_ << "\n"
0742             << eventDeltaRssT1_;
0743       }
0744       if (moduleSummaryRequested_ and not jobReportOutputOnly_) {  // changelog 1
0745         LogAbsolute mmr("ModuleMemoryReport");                     // at end of if block, mmr
0746                                                                    // is destructed, causing
0747                                                                    // message to be logged
0748         mmr << "ModuleMemoryReport> Each line has module label and: \n";
0749         mmr << "  (after early ignored events) \n";
0750         mmr << "    count of times module executed; average increase in vsize \n";
0751         mmr << "    maximum increase in vsize; event on which maximum occurred \n";
0752         mmr << "  (during early ignored events) \n";
0753         mmr << "    total and maximum vsize increases \n \n";
0754         for (SignificantModulesMap::iterator im = modules_.begin(); im != modules_.end(); ++im) {
0755           SignificantModule const& m = im->second;
0756           if (m.totalDeltaVsize == 0 && m.totalEarlyVsize == 0)
0757             continue;
0758           mmr << im->first << ": ";
0759           mmr << "n = " << m.postEarlyCount;
0760           if (m.postEarlyCount > 0) {
0761             mmr << " avg = " << m.totalDeltaVsize / m.postEarlyCount;
0762           }
0763           mmr << " max = " << m.maxDeltaVsize << " " << m.eventMaxDeltaV;
0764           if (m.totalEarlyVsize > 0) {
0765             mmr << " early total: " << m.totalEarlyVsize;
0766             mmr << " max: " << m.maxEarlyVsize;
0767           }
0768           mmr << "\n";
0769         }
0770       }  // end of if; mmr goes out of scope; log message is queued
0771 
0772       Service<JobReport> reportSvc;
0773       // changelog 1
0774 #define SIMPLE_MEMORY_CHECK_ORIGINAL_XML_OUTPUT
0775 #ifdef SIMPLE_MEMORY_CHECK_ORIGINAL_XML_OUTPUT
0776       //     std::map<std::string, double> reportData;
0777       std::map<std::string, std::string> reportData;
0778 
0779       if (eventL2_.vsize > 0)
0780         eventStatOutput("LargeVsizeIncreaseEventL2", eventL2_, reportData);
0781       if (eventL1_.vsize > 0)
0782         eventStatOutput("LargeVsizeIncreaseEventL1", eventL1_, reportData);
0783       if (eventM_.vsize > 0)
0784         eventStatOutput("LargestVsizeIncreaseEvent", eventM_, reportData);
0785       if (eventR1_.vsize > 0)
0786         eventStatOutput("LargeVsizeIncreaseEventR1", eventR1_, reportData);
0787       if (eventR2_.vsize > 0)
0788         eventStatOutput("LargeVsizeIncreaseEventR2", eventR2_, reportData);
0789       if (eventT3_.vsize > 0)
0790         eventStatOutput("ThirdLargestVsizeEventT3", eventT3_, reportData);
0791       if (eventT2_.vsize > 0)
0792         eventStatOutput("SecondLargestVsizeEventT2", eventT2_, reportData);
0793       if (eventT1_.vsize > 0)
0794         eventStatOutput("LargestVsizeEventT1", eventT1_, reportData);
0795 
0796       if (eventRssT3_.rss > 0)
0797         eventStatOutput("ThirdLargestRssEvent", eventRssT3_, reportData);
0798       if (eventRssT2_.rss > 0)
0799         eventStatOutput("SecondLargestRssEvent", eventRssT2_, reportData);
0800       if (eventRssT1_.rss > 0)
0801         eventStatOutput("LargestRssEvent", eventRssT1_, reportData);
0802       if (eventDeltaRssT3_.deltaRss > 0)
0803         eventStatOutput("ThirdLargestIncreaseRssEvent", eventDeltaRssT3_, reportData);
0804       if (eventDeltaRssT2_.deltaRss > 0)
0805         eventStatOutput("SecondLargestIncreaseRssEvent", eventDeltaRssT2_, reportData);
0806       if (eventDeltaRssT1_.deltaRss > 0)
0807         eventStatOutput("LargestIncreaseRssEvent", eventDeltaRssT1_, reportData);
0808 
0809 #ifdef __linux__
0810 #if (__GLIBC__ > 2) || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33)
0811       struct mallinfo2 minfo = mallinfo2();
0812 #else
0813       struct mallinfo minfo = mallinfo();
0814 #endif
0815       reportData.insert(std::make_pair("HEAP_ARENA_SIZE_BYTES", std::to_string(minfo.arena)));
0816       reportData.insert(std::make_pair("HEAP_ARENA_N_UNUSED_CHUNKS", std::to_string(minfo.ordblks)));
0817       reportData.insert(std::make_pair("HEAP_TOP_FREE_BYTES", std::to_string(minfo.keepcost)));
0818       reportData.insert(std::make_pair("HEAP_MAPPED_SIZE_BYTES", std::to_string(minfo.hblkhd)));
0819       reportData.insert(std::make_pair("HEAP_MAPPED_N_CHUNKS", std::to_string(minfo.hblks)));
0820       reportData.insert(std::make_pair("HEAP_USED_BYTES", std::to_string(minfo.uordblks)));
0821       reportData.insert(std::make_pair("HEAP_UNUSED_BYTES", std::to_string(minfo.fordblks)));
0822 #endif
0823 
0824       // Report Growth rates for VSize and Rss
0825       reportData.insert(std::make_pair("AverageGrowthRateVsize",
0826                                        d2str(averageGrowthRate(current_->vsize, growthRateVsize_, count_))));
0827       reportData.insert(std::make_pair("PeakValueVsize", d2str(eventT1_.vsize)));
0828       reportData.insert(
0829           std::make_pair("AverageGrowthRateRss", d2str(averageGrowthRate(current_->rss, growthRateRss_, count_))));
0830       reportData.insert(std::make_pair("PeakValueRss", d2str(eventRssT1_.rss)));
0831 
0832       if (moduleSummaryRequested_) {  // changelog 2
0833         for (SignificantModulesMap::iterator im = modules_.begin(); im != modules_.end(); ++im) {
0834           SignificantModule const& m = im->second;
0835           if (m.totalDeltaVsize == 0 && m.totalEarlyVsize == 0)
0836             continue;
0837           std::string label = im->first + ":";
0838           reportData.insert(std::make_pair(label + "PostEarlyCount", i2str(m.postEarlyCount)));
0839           if (m.postEarlyCount > 0) {
0840             reportData.insert(std::make_pair(label + "AverageDeltaVsize", d2str(m.totalDeltaVsize / m.postEarlyCount)));
0841           }
0842           reportData.insert(std::make_pair(label + "MaxDeltaVsize", d2str(m.maxDeltaVsize)));
0843           if (m.totalEarlyVsize > 0) {
0844             reportData.insert(std::make_pair(label + "TotalEarlyVsize", d2str(m.totalEarlyVsize)));
0845             reportData.insert(std::make_pair(label + "MaxEarlyDeltaVsize", d2str(m.maxEarlyVsize)));
0846           }
0847         }
0848       }
0849 
0850       std::map<std::string, std::string> reportMemoryProperties;
0851 
0852       if (FILE* fmeminfo = fopen("/proc/meminfo", "r")) {
0853         char buf[128];
0854         char space[] = " ";
0855         size_t value;
0856         while (fgets(buf, sizeof(buf), fmeminfo)) {
0857           char* saveptr;
0858           char* token = nullptr;
0859           token = strtok_r(buf, space, &saveptr);
0860           if (token != nullptr) {
0861             value = atol(strtok_r(nullptr, space, &saveptr));
0862             std::string category = token;
0863             reportMemoryProperties.insert(std::make_pair(category.substr(0, strlen(token) - 1), i2str(value)));
0864           }
0865         }
0866 
0867         fclose(fmeminfo);
0868       }
0869 
0870       //      reportSvc->reportMemoryInfo(reportData, reportMemoryProperties);
0871       reportSvc->reportPerformanceSummary("ApplicationMemory", reportData);
0872       reportSvc->reportPerformanceSummary("SystemMemory", reportMemoryProperties);
0873 #endif
0874 
0875 #ifdef SIMPLE_MEMORY_CHECK_DIFFERENT_XML_OUTPUT
0876       std::vector<std::string> reportData;
0877 
0878       if (eventL2_.vsize > 0)
0879         reportData.push_back(eventStatOutput("LargeVsizeIncreaseEventL2", eventL2_));
0880       if (eventL1_.vsize > 0)
0881         reportData.push_back(eventStatOutput("LargeVsizeIncreaseEventL1", eventL1_));
0882       if (eventM_.vsize > 0)
0883         reportData.push_back(eventStatOutput("LargestVsizeIncreaseEvent", eventM_));
0884       if (eventR1_.vsize > 0)
0885         reportData.push_back(eventStatOutput("LargeVsizeIncreaseEventR1", eventR1_));
0886       if (eventR2_.vsize > 0)
0887         reportData.push_back(eventStatOutput("LargeVsizeIncreaseEventR2", eventR2_));
0888       if (eventT3_.vsize > 0)
0889         reportData.push_back(eventStatOutput("ThirdLargestVsizeEventT3", eventT3_));
0890       if (eventT2_.vsize > 0)
0891         reportData.push_back(eventStatOutput("SecondLargestVsizeEventT2", eventT2_));
0892       if (eventT1_.vsize > 0)
0893         reportData.push_back(eventStatOutput("LargestVsizeEventT1", eventT1_));
0894 
0895       if (eventRssT3_.rss > 0)
0896         eventStatOutput("ThirdLargestRssEvent", eventRssT3_, reportData);
0897       if (eventRssT2_.rss > 0)
0898         eventStatOutput("SecondLargestRssEvent", eventRssT2_, reportData);
0899       if (eventRssT1_.rss > 0)
0900         eventStatOutput("LargestRssEvent", eventRssT1_, reportData);
0901       if (eventDeltaRssT3_.deltaRss > 0)
0902         eventStatOutput("ThirdLargestIncreaseRssEvent", eventDeltaRssT3_, reportData);
0903       if (eventDeltaRssT2_.deltaRss > 0)
0904         eventStatOutput("SecondLargestIncreaseRssEvent", eventDeltaRssT2_, reportData);
0905       if (eventDeltaRssT1_.deltaRss > 0)
0906         eventStatOutput("LargestIncreaseRssEvent", eventDeltaRssT1_, reportData);
0907 
0908 #if (__GLIBC__ > 2) || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33)
0909       struct mallinfo2 minfo = mallinfo2();
0910 #else
0911       struct mallinfo minfo = mallinfo();
0912 #endif
0913       reportData.push_back(mallOutput("HEAP_ARENA_SIZE_BYTES", minfo.arena));
0914       reportData.push_back(mallOutput("HEAP_ARENA_N_UNUSED_CHUNKS", minfo.ordblks));
0915       reportData.push_back(mallOutput("HEAP_TOP_FREE_BYTES", minfo.keepcost));
0916       reportData.push_back(mallOutput("HEAP_MAPPED_SIZE_BYTES", minfo.hblkhd));
0917       reportData.push_back(mallOutput("HEAP_MAPPED_N_CHUNKS", minfo.hblks));
0918       reportData.push_back(mallOutput("HEAP_USED_BYTES", minfo.uordblks));
0919       reportData.push_back(mallOutput("HEAP_UNUSED_BYTES", minfo.fordblks));
0920 
0921       // Report Growth rates for VSize and Rss
0922       reportData.insert(std::make_pair("AverageGrowthRateVsize",
0923                                        d2str(averageGrowthRate(current_->vsize, growthRateVsize_, count_))));
0924       reportData.insert(std::make_pair("PeakValueVsize", d2str(eventT1_.vsize)));
0925       reportData.insert(
0926           std::make_pair("AverageGrowthRateRss", d2str(averageGrowthRate(current_->rss, growthRateRss_, count_))));
0927       reportData.insert(std::make_pair("PeakValueRss", d2str(eventRssT1_.rss)));
0928 
0929       reportSvc->reportMemoryInfo(reportData);
0930       // This is a form of reportMemoryInfo taking s vector, not a map
0931 #endif
0932     }  // postEndJob
0933 
0934     void SimpleMemoryCheck::postEvent(StreamContext const& iContext) {
0935       ++count_;
0936       bool expected = false;
0937       if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0938         std::shared_ptr<void> guard(
0939             nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0940         update();
0941         if (monitorPssAndPrivate_) {
0942           currentSmaps_ = fetchSmaps();
0943         }
0944         updateEventStats(iContext.eventID());
0945         if (oncePerEventMode_) {
0946           // should probably use be Run:Event or count_ for the label and name
0947           updateMax();
0948           andPrint("event", "", "");
0949         }
0950       }
0951     }
0952 
0953     void SimpleMemoryCheck::preModule(StreamContext const& iStreamContext, ModuleCallingContext const& iModuleContext) {
0954       bool expectedMeasurementUnderway = false;
0955       if (measurementUnderway_.compare_exchange_strong(expectedMeasurementUnderway, true, std::memory_order_acq_rel)) {
0956         std::shared_ptr<void> guard(
0957             nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0958         bool expectedModuleMeasurementUnderway = false;
0959         if (moduleMeasurementUnderway_.compare_exchange_strong(expectedModuleMeasurementUnderway, true)) {
0960           update();
0961           // changelog 2
0962           moduleEntryVsize_ = current_->vsize;
0963           moduleStreamID_.store(iStreamContext.streamID().value(), std::memory_order_release);
0964           moduleID_.store(iModuleContext.moduleDescription()->id(), std::memory_order_release);
0965         }
0966       }
0967     }
0968 
0969     void SimpleMemoryCheck::postModule(StreamContext const& iStreamContext,
0970                                        ModuleCallingContext const& iModuleContext) {
0971       if (!oncePerEventMode_) {
0972         bool expected = false;
0973         if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0974           std::shared_ptr<void> guard(
0975               nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0976           auto const md = iModuleContext.moduleDescription();
0977           updateAndPrint("module", md->moduleLabel(), md->moduleName());
0978         }
0979       }
0980 
0981       if (moduleSummaryRequested_) {
0982         //is this the module instance we are measuring?
0983         if (moduleMeasurementUnderway_.load(std::memory_order_acquire) and
0984             (iStreamContext.streamID().value() == moduleStreamID_.load(std::memory_order_acquire)) and
0985             (iModuleContext.moduleDescription()->id() == moduleID_.load(std::memory_order_acquire))) {
0986           //Need to release our module measurement lock
0987           std::shared_ptr<void> guardModuleMeasurementUnderway(
0988               nullptr, [this](void const*) { moduleMeasurementUnderway_.store(false, std::memory_order_release); });
0989           bool expected = false;
0990           if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0991             std::shared_ptr<void> guardMeasurementUnderway(
0992                 nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0993             if (oncePerEventMode_) {
0994               update();
0995             }
0996             // changelog 2
0997             double dv = current_->vsize - moduleEntryVsize_;
0998             std::string label = iModuleContext.moduleDescription()->moduleLabel();
0999             updateModuleMemoryStats(modules_[label], dv, iStreamContext.eventID());
1000           }
1001         }
1002       }
1003     }
1004 
1005     void SimpleMemoryCheck::update() {
1006       std::swap(current_, previous_);
1007       *current_ = fetch();
1008     }
1009 
1010     void SimpleMemoryCheck::updateMax() {
1011       auto v = *current_;
1012       if ((v > max_) || oncePerEventMode_) {
1013         if (max_.vsize < v.vsize) {
1014           max_.vsize = v.vsize;
1015         }
1016         if (max_.rss < v.rss) {
1017           max_.rss = v.rss;
1018         }
1019       }
1020     }
1021 
1022     void SimpleMemoryCheck::updateEventStats(EventID const& e) {
1023       if (count_ < num_to_skip_)
1024         return;
1025       if (count_ == num_to_skip_) {
1026         eventT1_.set(0, 0, e, this);
1027         eventM_.set(0, 0, e, this);
1028         eventRssT1_.set(0, 0, e, this);
1029         eventDeltaRssT1_.set(0, 0, e, this);
1030         return;
1031       }
1032       double vsize = current_->vsize;
1033       double deltaVsize = vsize - eventT1_.vsize;
1034 
1035       // Update significative events for Vsize
1036       if (vsize > eventT1_.vsize) {
1037         double deltaRss = current_->rss - eventT1_.rss;
1038         eventT3_ = eventT2_;
1039         eventT2_ = eventT1_;
1040         eventT1_.set(deltaVsize, deltaRss, e, this);
1041       } else if (vsize > eventT2_.vsize) {
1042         double deltaRss = current_->rss - eventT1_.rss;
1043         eventT3_ = eventT2_;
1044         eventT2_.set(deltaVsize, deltaRss, e, this);
1045       } else if (vsize > eventT3_.vsize) {
1046         double deltaRss = current_->rss - eventT1_.rss;
1047         eventT3_.set(deltaVsize, deltaRss, e, this);
1048       }
1049 
1050       if (deltaVsize > eventM_.deltaVsize) {
1051         double deltaRss = current_->rss - eventM_.rss;
1052         if (eventL1_.deltaVsize >= eventR1_.deltaVsize) {
1053           eventL2_ = eventL1_;
1054         } else {
1055           eventL2_ = eventR1_;
1056         }
1057         eventL1_ = eventM_;
1058         eventM_.set(deltaVsize, deltaRss, e, this);
1059         eventR1_ = SignificantEvent();
1060         eventR2_ = SignificantEvent();
1061       } else if (deltaVsize > eventR1_.deltaVsize) {
1062         double deltaRss = current_->rss - eventM_.rss;
1063         eventR2_ = eventR1_;
1064         eventR1_.set(deltaVsize, deltaRss, e, this);
1065       } else if (deltaVsize > eventR2_.deltaVsize) {
1066         double deltaRss = current_->rss - eventR1_.rss;
1067         eventR2_.set(deltaVsize, deltaRss, e, this);
1068       }
1069 
1070       // Update significative events for Rss
1071       double rss = current_->rss;
1072       double deltaRss = rss - eventRssT1_.rss;
1073 
1074       if (rss > eventRssT1_.rss) {
1075         eventRssT3_ = eventRssT2_;
1076         eventRssT2_ = eventRssT1_;
1077         eventRssT1_.set(deltaVsize, deltaRss, e, this);
1078       } else if (rss > eventRssT2_.rss) {
1079         eventRssT3_ = eventRssT2_;
1080         eventRssT2_.set(deltaVsize, deltaRss, e, this);
1081       } else if (rss > eventRssT3_.rss) {
1082         eventRssT3_.set(deltaVsize, deltaRss, e, this);
1083       }
1084       if (deltaRss > eventDeltaRssT1_.deltaRss) {
1085         eventDeltaRssT3_ = eventDeltaRssT2_;
1086         eventDeltaRssT2_ = eventDeltaRssT1_;
1087         eventDeltaRssT1_.set(deltaVsize, deltaRss, e, this);
1088       } else if (deltaRss > eventDeltaRssT2_.deltaRss) {
1089         eventDeltaRssT3_ = eventDeltaRssT2_;
1090         eventDeltaRssT2_.set(deltaVsize, deltaRss, e, this);
1091       } else if (deltaRss > eventDeltaRssT3_.deltaRss) {
1092         eventDeltaRssT3_.set(deltaVsize, deltaRss, e, this);
1093       }
1094     }  // updateEventStats
1095 
1096     void SimpleMemoryCheck::andPrintAlways(std::string const& type,
1097                                            std::string const& mdlabel,
1098                                            std::string const& mdname,
1099                                            bool includeSmapsAndJe) const {
1100       double deltaVSIZE = current_->vsize - max_.vsize;
1101       double deltaRSS = current_->rss - max_.rss;
1102 
1103       LogWarning log("MemoryCheck");
1104       // default
1105       log << "MemoryCheck: " << type << " " << mdname << ":" << mdlabel << " VSIZE " << current_->vsize << " "
1106           << deltaVSIZE << " RSS " << current_->rss << " " << deltaRSS;
1107       if (showMallocInfo_) {
1108 #ifdef __linux__
1109 #if (__GLIBC__ > 2) || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33)
1110         struct mallinfo2 minfo = mallinfo2();
1111 #else
1112         struct mallinfo minfo = mallinfo();
1113 #endif
1114         log << " HEAP-ARENA [ SIZE-BYTES " << minfo.arena << " N-UNUSED-CHUNKS " << minfo.ordblks << " TOP-FREE-BYTES "
1115             << minfo.keepcost << " ]"
1116             << " HEAP-MAPPED [ SIZE-BYTES " << minfo.hblkhd << " N-CHUNKS " << minfo.hblks << " ]"
1117             << " HEAP-USED-BYTES " << minfo.uordblks << " HEAP-UNUSED-BYTES " << minfo.fordblks;
1118 #endif
1119       }
1120       if (includeSmapsAndJe) {
1121         if (smapsFile_) {
1122           log << " PSS " << currentSmaps_.pss_ << " PRIVATE " << currentSmaps_.private_ << " ANONHUGEPAGES "
1123               << currentSmaps_.anonHugePages_;
1124         }
1125         if (je_mallctl) {
1126           auto info = fetchJemalloc();
1127           log << " JeMalloc allocated " << info.allocated << " active " << info.active << " resident " << info.resident
1128               << " mapped " << info.mapped << " metadata " << info.metadata;
1129         }
1130       }
1131     }
1132 
1133     void SimpleMemoryCheck::andPrint(std::string const& type,
1134                                      std::string const& mdlabel,
1135                                      std::string const& mdname) const {
1136       if (not jobReportOutputOnly_ && ((*current_ > max_) || printEachTime_)) {
1137         if (count_ >= num_to_skip_) {
1138           andPrintAlways(type, mdlabel, mdname);
1139         }
1140       }
1141     }
1142 
1143     void SimpleMemoryCheck::updateAndPrint(std::string const& type,
1144                                            std::string const& mdlabel,
1145                                            std::string const& mdname) {
1146       update();
1147       andPrint(type, mdlabel, mdname);
1148       updateMax();
1149     }
1150 
1151 #ifdef SIMPLE_MEMORY_CHECK_ORIGINAL_XML_OUTPUT
1152     void SimpleMemoryCheck::eventStatOutput(std::string title,
1153                                             SignificantEvent const& e,
1154                                             std::map<std::string, std::string>& m) const {
1155       {
1156         std::ostringstream os;
1157         os << title << "-a-COUNT";
1158         m.insert(std::make_pair(os.str(), i2str(e.count)));
1159       }
1160       {
1161         std::ostringstream os;
1162         os << title << "-b-RUN";
1163         m.insert(std::make_pair(os.str(), d2str(static_cast<double>(e.event.run()))));
1164       }
1165       {
1166         std::ostringstream os;
1167         os << title << "-c-EVENT";
1168         m.insert(std::make_pair(os.str(), d2str(static_cast<double>(e.event.event()))));
1169       }
1170       {
1171         std::ostringstream os;
1172         os << title << "-d-VSIZE";
1173         m.insert(std::make_pair(os.str(), d2str(e.vsize)));
1174       }
1175       {
1176         std::ostringstream os;
1177         os << title << "-e-DELTV";
1178         m.insert(std::make_pair(os.str(), d2str(e.deltaVsize)));
1179       }
1180       {
1181         std::ostringstream os;
1182         os << title << "-f-RSS";
1183         m.insert(std::make_pair(os.str(), d2str(e.rss)));
1184       }
1185       if (monitorPssAndPrivate_) {
1186         {
1187           std::ostringstream os;
1188           os << title << "-g-PRIVATE";
1189           m.insert(std::make_pair(os.str(), d2str(e.privateSize)));
1190         }
1191         {
1192           std::ostringstream os;
1193           os << title << "-h-PSS";
1194           m.insert(std::make_pair(os.str(), d2str(e.pss)));
1195         }
1196       }
1197     }  // eventStatOutput
1198 #endif
1199 
1200 #ifdef SIMPLE_MEMORY_CHECK_DIFFERENT_XML_OUTPUT
1201     std::string SimpleMemoryCheck::eventStatOutput(std::string title, SignificantEvent const& e) const {
1202       std::ostringstream os;
1203       os << "  <" << title << ">\n";
1204       os << "    " << e.count << ": " << e.event;
1205       os << " vsize " << e.vsize - e.deltaVsize << " + " << e.deltaVsize << " = " << e.vsize;
1206       os << "  rss: " << e.rss << "\n";
1207       os << "  </" << title << ">\n";
1208       return os.str();
1209     }  // eventStatOutput
1210 
1211     std::string SimpleMemoryCheck::mallOutput(std::string title, size_t const& n) const {
1212       std::ostringstream os;
1213       os << "  <" << title << ">\n";
1214       os << "    " << n << "\n";
1215       os << "  </" << title << ">\n";
1216       return os.str();
1217     }
1218 #endif
1219     // changelog 2
1220     void SimpleMemoryCheck::updateModuleMemoryStats(SignificantModule& m,
1221                                                     double dv,
1222                                                     edm::EventID const& currentEventID) {
1223       if (count_ < num_to_skip_) {
1224         m.totalEarlyVsize += dv;
1225         if (dv > m.maxEarlyVsize)
1226           m.maxEarlyVsize = dv;
1227       } else {
1228         ++m.postEarlyCount;
1229         m.totalDeltaVsize += dv;
1230         if (dv > m.maxDeltaVsize) {
1231           m.maxDeltaVsize = dv;
1232           m.eventMaxDeltaV = currentEventID;
1233         }
1234       }
1235     }  //updateModuleMemoryStats
1236 
1237     std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantEvent const& se) {
1238       os << "[" << se.count << "] " << se.event << "  vsize = " << se.vsize << " deltaVsize = " << se.deltaVsize
1239          << " rss = " << se.rss << " delta = " << se.deltaRss;
1240 
1241       if (se.monitorPssAndPrivate) {
1242         os << " private = " << se.privateSize << " pss = " << se.pss;
1243       }
1244       if (se.jemalloc.has_value()) {
1245         os << " allocated = " << se.jemalloc->allocated << " active = " << se.jemalloc->active;
1246       }
1247       return os;
1248     }
1249 
1250     std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantModule const& sm) {
1251       if (sm.postEarlyCount > 0) {
1252         os << "\nPost Early Events:  TotalDeltaVsize: " << sm.totalDeltaVsize
1253            << " (avg: " << sm.totalDeltaVsize / sm.postEarlyCount << "; max: " << sm.maxDeltaVsize << " during "
1254            << sm.eventMaxDeltaV << ")";
1255       }
1256       if (sm.totalEarlyVsize > 0) {
1257         os << "\n     Early Events:  TotalDeltaVsize: " << sm.totalEarlyVsize << " (max: " << sm.maxEarlyVsize << ")";
1258       }
1259 
1260       return os;
1261     }
1262 
1263   }  // end namespace service
1264 }  // end namespace edm
1265 
1266 #if defined(__linux__)
1267 using edm::service::SimpleMemoryCheck;
1268 DEFINE_FWK_SERVICE(SimpleMemoryCheck);
1269 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.2.1 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!