File indexing completed on 2025-01-28 23:24:28
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018
0019
0020
0021
0022
0023
0024
0025 #include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
0026
0027 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0028 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0029 #include "DataFormats/Provenance/interface/EventID.h"
0030 #include "FWCore/Services/plugins/ProcInfoFetcher.h"
0031
0032 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0033 #include "FWCore/Framework/interface/Event.h"
0034 #include "FWCore/MessageLogger/interface/JobReport.h"
0035 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0036 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0037 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0038 #include "FWCore/ServiceRegistry/interface/Service.h"
0039 #include "FWCore/ServiceRegistry/interface/StreamContext.h"
0040 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0041 #include "FWCore/Utilities/interface/Exception.h"
0042 #include "FWCore/Utilities/interface/get_underlying_safe.h"
0043
0044 #include <array>
0045 #include <cstring>
0046 #include <iostream>
0047 #include <memory>
0048 #ifdef __linux__
0049 #include <malloc.h>
0050 #endif
0051 #include <sstream>
0052
0053 #include <string>
0054
0055
0056 #include <cstdio>
0057 #include <atomic>
0058 #include <optional>
0059
0060
0061 #include <dlfcn.h>
0062 extern "C" {
0063 typedef int (*mallctl_t)(const char* name, void* oldp, size_t* oldlenp, void* newp, size_t newlen);
0064 }
0065
0066 namespace edm {
0067 namespace service {
0068 enum class SmapsSection {
0069 kSharedObject = 0,
0070 kPcm = 1,
0071 kOtherFile = 2,
0072 kStack = 3,
0073 kMmap = 4,
0074 kOther = 5,
0075 kSize = 6
0076 };
0077 struct smapsInfo {
0078 double private_ = 0;
0079 double pss_ = 0;
0080 double anonHugePages_ = 0;
0081
0082 static constexpr auto sectionsSize_ = static_cast<unsigned>(SmapsSection::kSize);
0083 std::array<double, sectionsSize_> sectionRss_{};
0084 std::array<double, sectionsSize_> sectionVSize_{};
0085 };
0086 struct JemallocInfo {
0087 double allocated = 0;
0088 double active = 0;
0089 double resident = 0;
0090 double mapped = 0;
0091 double metadata = 0;
0092 };
0093
0094 class SimpleMemoryCheck {
0095 public:
0096 SimpleMemoryCheck(const ParameterSet&, ActivityRegistry&);
0097 ~SimpleMemoryCheck();
0098
0099 static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0100
0101 void preSourceConstruction(const ModuleDescription&);
0102 void postSourceConstruction(const ModuleDescription&);
0103 void postSourceEvent(StreamID);
0104
0105 void postBeginJob();
0106
0107 void postEvent(StreamContext const&);
0108
0109 void postModuleBeginJob(const ModuleDescription&);
0110 void postModuleConstruction(const ModuleDescription&);
0111
0112 void preModule(StreamContext const&, ModuleCallingContext const&);
0113 void postModule(StreamContext const&, ModuleCallingContext const&);
0114
0115 void earlyTermination();
0116
0117 void postEndJob();
0118
0119 void startSamplingThread();
0120 void stopSamplingThread();
0121
0122 private:
0123 ProcInfo fetch();
0124 smapsInfo fetchSmaps();
0125 JemallocInfo fetchJemalloc() const;
0126 double pageSize() const { return pg_size_; }
0127 double averageGrowthRate(double current, double past, int count);
0128 void update();
0129 void updateMax();
0130 void andPrintAlways(const std::string& type,
0131 const std::string& mdlabel,
0132 const std::string& mdname,
0133 bool includeSmapsAndJe = false) const;
0134 void andPrint(const std::string& type, const std::string& mdlabel, const std::string& mdname) const;
0135 void updateAndPrint(const std::string& type, const std::string& mdlabel, const std::string& mdname);
0136
0137
0138
0139 std::optional<std::string> openFilesNoThrow();
0140 void openFiles();
0141
0142 char const* smapsLineBuffer() const { return get_underlying_safe(smapsLineBuffer_); }
0143 char*& smapsLineBuffer() { return get_underlying_safe(smapsLineBuffer_); }
0144
0145 ProcInfo a_;
0146 ProcInfo b_;
0147 ProcInfo max_;
0148 edm::propagate_const<ProcInfo*> current_;
0149 edm::propagate_const<ProcInfo*> previous_;
0150
0151 smapsInfo currentSmaps_;
0152
0153 ProcInfoFetcher piFetcher_;
0154 double pg_size_;
0155 int num_to_skip_;
0156
0157 bool showMallocInfo_;
0158 bool showJemallocInfo_;
0159 bool oncePerEventMode_;
0160 bool printEachTime_;
0161 bool jobReportOutputOnly_;
0162 bool monitorPssAndPrivate_;
0163 std::atomic<int> count_;
0164 unsigned int sampleEveryNSeconds_;
0165 std::optional<std::thread> samplingThread_;
0166 std::atomic<bool> stopThread_ = false;
0167 std::atomic<edm::EventID> mostRecentlyStartedEvent_;
0168
0169 mallctl_t je_mallctl = nullptr;
0170
0171
0172 edm::propagate_const<FILE*> smapsFile_ = nullptr;
0173 edm::propagate_const<char*> smapsLineBuffer_;
0174 size_t smapsLineBufferLen_;
0175
0176
0177 double growthRateVsize_;
0178 double growthRateRss_;
0179
0180
0181 struct SignificantEvent {
0182 edm::EventID event;
0183 double vsize = 0;
0184 double deltaVsize = 0;
0185 double rss = 0;
0186 double deltaRss = 0;
0187 double privateSize = 0;
0188 double pss = 0;
0189 double anonHugePages = 0;
0190 std::optional<JemallocInfo> jemalloc;
0191 int count = 0;
0192 bool monitorPssAndPrivate = false;
0193 SignificantEvent() = default;
0194 void set(double deltaV, double deltaR, edm::EventID const& e, SimpleMemoryCheck* t) {
0195 count = t->count_;
0196 vsize = t->current_->vsize;
0197 deltaVsize = deltaV;
0198 rss = t->current_->rss;
0199 deltaRss = deltaR;
0200 monitorPssAndPrivate = t->monitorPssAndPrivate_;
0201 if (monitorPssAndPrivate) {
0202 privateSize = t->currentSmaps_.private_;
0203 pss = t->currentSmaps_.pss_;
0204 anonHugePages = t->currentSmaps_.anonHugePages_;
0205 }
0206 if (t->showJemallocInfo_) {
0207 jemalloc = t->fetchJemalloc();
0208 }
0209 event = e;
0210 }
0211 };
0212 friend struct SignificantEvent;
0213 friend std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantEvent const& se);
0214
0215
0216
0217
0218
0219
0220
0221
0222
0223
0224
0225
0226
0227
0228
0229
0230 SignificantEvent eventM_;
0231 SignificantEvent eventL1_;
0232 SignificantEvent eventL2_;
0233 SignificantEvent eventR1_;
0234 SignificantEvent eventR2_;
0235 SignificantEvent eventT1_;
0236 SignificantEvent eventT2_;
0237 SignificantEvent eventT3_;
0238
0239
0240
0241
0242
0243
0244
0245
0246
0247
0248
0249
0250
0251 SignificantEvent eventRssT1_;
0252 SignificantEvent eventRssT2_;
0253 SignificantEvent eventRssT3_;
0254 SignificantEvent eventDeltaRssT1_;
0255 SignificantEvent eventDeltaRssT2_;
0256 SignificantEvent eventDeltaRssT3_;
0257
0258 void updateEventStats(edm::EventID const& e);
0259 std::string eventStatOutput(std::string title, SignificantEvent const& e) const;
0260 void eventStatOutput(std::string title, SignificantEvent const& e, std::map<std::string, std::string>& m) const;
0261 std::string mallOutput(std::string title, size_t const& n) const;
0262
0263
0264 struct SignificantModule {
0265 int postEarlyCount;
0266 double totalDeltaVsize;
0267 double maxDeltaVsize;
0268 edm::EventID eventMaxDeltaV;
0269 double totalEarlyVsize;
0270 double maxEarlyVsize;
0271 SignificantModule()
0272 : postEarlyCount(0),
0273 totalDeltaVsize(0),
0274 maxDeltaVsize(0),
0275 eventMaxDeltaV(),
0276 totalEarlyVsize(0),
0277 maxEarlyVsize(0) {}
0278 void set(double deltaV, bool early);
0279 };
0280 friend struct SignificantModule;
0281 friend std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantModule const& se);
0282 bool moduleSummaryRequested_;
0283 typedef std::map<std::string, SignificantModule> SignificantModulesMap;
0284 SignificantModulesMap modules_;
0285 double moduleEntryVsize_;
0286 void updateModuleMemoryStats(SignificantModule& m, double dv, edm::EventID const&);
0287
0288
0289 std::atomic<bool> measurementUnderway_;
0290 std::atomic<bool> moduleMeasurementUnderway_;
0291 std::atomic<unsigned int> moduleStreamID_;
0292 std::atomic<unsigned int> moduleID_;
0293
0294 };
0295
0296 std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantEvent const& se);
0297
0298 std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantModule const& se);
0299
0300 }
0301 }
0302
0303 #ifdef __linux__
0304 #define LINUX 1
0305 #endif
0306
0307 #include <fcntl.h>
0308 #include <unistd.h>
0309
0310 namespace edm {
0311 namespace service {
0312
0313 static std::string d2str(double d) {
0314 std::ostringstream t;
0315 t << d;
0316 return t.str();
0317 }
0318
0319 static std::string i2str(int i) {
0320 std::ostringstream t;
0321 t << i;
0322 return t.str();
0323 }
0324
0325 ProcInfo SimpleMemoryCheck::fetch() { return piFetcher_.fetch(); }
0326
0327 smapsInfo SimpleMemoryCheck::fetchSmaps() {
0328 smapsInfo ret;
0329 #ifdef LINUX
0330 fseek(smapsFile_, 0, SEEK_SET);
0331 ssize_t read;
0332 SmapsSection section = SmapsSection::kOther;
0333
0334
0335
0336
0337
0338
0339
0340
0341
0342
0343 while ((read = getline(&smapsLineBuffer(), &smapsLineBufferLen_, smapsFile_)) != -1) {
0344 if (read > 14) {
0345
0346
0347 if (char const* ret = strchr(smapsLineBuffer_, ':'); ret != nullptr and *(ret + 1) != ' ') {
0348 ret = strrchr(smapsLineBuffer_, ' ');
0349 if (ret == nullptr) {
0350
0351 section = SmapsSection::kOther;
0352 } else if (*(ret + 1) == '\n') {
0353
0354 section = SmapsSection::kMmap;
0355 } else if (*(ret + 1) == '/') {
0356
0357
0358 auto len = strlen(ret);
0359 if (0 == strncmp(ret + len - 5, ".pcm", 4)) {
0360 section = SmapsSection::kPcm;
0361 } else if (strstr(ret, ".so") != nullptr) {
0362 section = SmapsSection::kSharedObject;
0363 } else {
0364 section = SmapsSection::kOtherFile;
0365 }
0366 } else if (0 == strncmp("[stack]", ret + 1, 7)) {
0367 section = SmapsSection::kStack;
0368 } else {
0369 section = SmapsSection::kOther;
0370 }
0371 continue;
0372 }
0373
0374
0375 if (0 == strncmp("Private_", smapsLineBuffer_, 8)) {
0376 unsigned int value = atoi(smapsLineBuffer_ + 14);
0377
0378 ret.private_ += static_cast<double>(value) / 1024.;
0379 } else if (0 == strncmp("Pss:", smapsLineBuffer_, 4)) {
0380 unsigned int value = atoi(smapsLineBuffer_ + 4);
0381
0382 ret.pss_ += static_cast<double>(value) / 1024.;
0383 } else if (0 == strncmp("AnonHugePages:", smapsLineBuffer_, 14)) {
0384 unsigned int value = atoi(smapsLineBuffer_ + 14);
0385 ret.anonHugePages_ += static_cast<double>(value) / 1024.;
0386 } else if (0 == strncmp("Rss:", smapsLineBuffer_, 4)) {
0387 unsigned int value = atoi(smapsLineBuffer_ + 4);
0388
0389 ret.sectionRss_[static_cast<unsigned>(section)] += static_cast<double>(value) / 1024.;
0390 } else if (0 == strncmp("Size:", smapsLineBuffer_, 5)) {
0391 unsigned int value = atoi(smapsLineBuffer_ + 5);
0392
0393 ret.sectionVSize_[static_cast<unsigned>(section)] += static_cast<double>(value) / 1024.;
0394 }
0395 }
0396 }
0397 #endif
0398 return ret;
0399 }
0400
0401 JemallocInfo SimpleMemoryCheck::fetchJemalloc() const {
0402 JemallocInfo info;
0403 if (je_mallctl) {
0404
0405 uint64_t epoch = 1;
0406 size_t e_len = sizeof(uint64_t);
0407 if (je_mallctl("epoch", &epoch, &e_len, &epoch, e_len) != 0) {
0408 return info;
0409 }
0410
0411
0412 size_t allocated, active, resident, mapped, metadata;
0413 size_t len = sizeof(size_t);
0414 if (je_mallctl("stats.allocated", &allocated, &len, nullptr, 0) != 0) {
0415 return info;
0416 }
0417 if (je_mallctl("stats.active", &active, &len, nullptr, 0) != 0) {
0418 return info;
0419 }
0420 if (je_mallctl("stats.resident", &resident, &len, nullptr, 0) != 0) {
0421 return info;
0422 }
0423 if (je_mallctl("stats.mapped", &mapped, &len, nullptr, 0) != 0) {
0424 return info;
0425 }
0426 if (je_mallctl("stats.metadata", &metadata, &len, nullptr, 0) != 0) {
0427 return info;
0428 }
0429 info.allocated = allocated / 1024.0 / 1024.0;
0430 info.active = active / 1024.0 / 1024.0;
0431 info.resident = resident / 1024.0 / 1024.0;
0432 info.mapped = mapped / 1024.0 / 1024.0;
0433 info.metadata = metadata / 1024.0 / 1024.0;
0434 }
0435 return info;
0436 }
0437
0438 double SimpleMemoryCheck::averageGrowthRate(double current, double past, int count) {
0439 return (current - past) / (double)std::max(count, 1);
0440 }
0441
0442 SimpleMemoryCheck::SimpleMemoryCheck(ParameterSet const& iPS, ActivityRegistry& iReg)
0443 : a_(),
0444 b_(),
0445 current_(&a_),
0446 previous_(&b_),
0447 pg_size_(sysconf(_SC_PAGESIZE)),
0448 num_to_skip_(iPS.getUntrackedParameter<int>("ignoreTotal")),
0449 showMallocInfo_(iPS.getUntrackedParameter<bool>("showMallocInfo")),
0450 showJemallocInfo_(iPS.getUntrackedParameter<bool>("showJemallocInfo")),
0451 oncePerEventMode_(iPS.getUntrackedParameter<bool>("oncePerEventMode")),
0452 printEachTime_(oncePerEventMode_ or iPS.getUntrackedParameter<bool>("printEachSample")),
0453 jobReportOutputOnly_(iPS.getUntrackedParameter<bool>("jobReportOutputOnly")),
0454 monitorPssAndPrivate_(iPS.getUntrackedParameter<bool>("monitorPssAndPrivate")),
0455 count_(),
0456 sampleEveryNSeconds_(iPS.getUntrackedParameter<unsigned int>("sampleEveryNSeconds")),
0457 smapsFile_(nullptr),
0458 smapsLineBuffer_(nullptr),
0459 smapsLineBufferLen_(0),
0460 growthRateVsize_(),
0461 growthRateRss_(),
0462 moduleSummaryRequested_(iPS.getUntrackedParameter<bool>("moduleMemorySummary")),
0463 measurementUnderway_(false) {
0464
0465
0466 std::ostringstream ost;
0467
0468 if (monitorPssAndPrivate_) {
0469 openFiles();
0470 }
0471
0472 iReg.watchPostEndJob(this, &SimpleMemoryCheck::postEndJob);
0473
0474
0475 iReg.watchPreSourceEarlyTermination([this](TerminationOrigin) { earlyTermination(); });
0476 iReg.watchPreGlobalEarlyTermination([this](GlobalContext const&, TerminationOrigin) { earlyTermination(); });
0477 iReg.watchPreStreamEarlyTermination([this](StreamContext const&, TerminationOrigin) { earlyTermination(); });
0478
0479 if (sampleEveryNSeconds_ > 0) {
0480 if (oncePerEventMode_) {
0481 throw edm::Exception(edm::errors::Configuration)
0482 << "'sampleEventNSeconds' and 'oncePerEventMode' cannot be used together";
0483 }
0484 if (moduleSummaryRequested_) {
0485 throw edm::Exception(edm::errors::Configuration)
0486 << "'sampleEventNSeconds' and 'moduleSummaryRequested' cannot be used together";
0487 }
0488 iReg.watchPostBeginJob(this, &SimpleMemoryCheck::startSamplingThread);
0489 iReg.watchPreEndJob(this, &SimpleMemoryCheck::stopSamplingThread);
0490 iReg.watchPreEvent([this](auto const& iContext) { mostRecentlyStartedEvent_.store(iContext.eventID()); });
0491 return;
0492 }
0493
0494 iReg.watchPostEvent(this, &SimpleMemoryCheck::postEvent);
0495
0496 if (!oncePerEventMode_) {
0497 iReg.watchPreSourceConstruction(this, &SimpleMemoryCheck::preSourceConstruction);
0498 iReg.watchPostSourceConstruction(this, &SimpleMemoryCheck::postSourceConstruction);
0499 iReg.watchPostSourceEvent(this, &SimpleMemoryCheck::postSourceEvent);
0500 iReg.watchPostModuleConstruction(this, &SimpleMemoryCheck::postModuleConstruction);
0501 iReg.watchPostModuleBeginJob(this, &SimpleMemoryCheck::postModuleBeginJob);
0502 iReg.watchPostModuleEvent(this, &SimpleMemoryCheck::postModule);
0503 iReg.watchPostBeginJob(this, &SimpleMemoryCheck::postBeginJob);
0504 }
0505 if (moduleSummaryRequested_) {
0506 iReg.watchPreModuleEvent(this, &SimpleMemoryCheck::preModule);
0507 if (oncePerEventMode_) {
0508 iReg.watchPostModuleEvent(this, &SimpleMemoryCheck::postModule);
0509 }
0510 }
0511
0512
0513
0514
0515
0516
0517
0518
0519
0520
0521 if (showJemallocInfo_) {
0522
0523 je_mallctl = reinterpret_cast<mallctl_t>(::dlsym(RTLD_DEFAULT, "mallctl"));
0524 if (je_mallctl == nullptr) {
0525 showJemallocInfo_ = false;
0526 }
0527 }
0528 }
0529
0530 SimpleMemoryCheck::~SimpleMemoryCheck() {
0531 #ifdef LINUX
0532 if (nullptr != smapsFile_) {
0533 fclose(smapsFile_);
0534 }
0535 #endif
0536 if (smapsLineBuffer_) {
0537
0538 free(smapsLineBuffer_);
0539 }
0540 }
0541
0542 void SimpleMemoryCheck::fillDescriptions(ConfigurationDescriptions& descriptions) {
0543 ParameterSetDescription desc;
0544 desc.addUntracked<int>("ignoreTotal", 1)
0545 ->setComment("Number of events/samples to finish before starting measuring and reporting.");
0546 desc.addUntracked<unsigned int>("sampleEveryNSeconds", 0)
0547 ->setComment(
0548 "Use a special thread to sample memory at the set rate. A value of 0 means no sampling. This option "
0549 "cannot be used with 'oncePerEventMode' or 'moduleMemorySummary'.");
0550 desc.addUntracked<bool>("printEachSample", false)
0551 ->setComment("If sampling on, print each sample taken else will print only when sample is the largest seen.");
0552 desc.addUntracked<bool>("showMallocInfo", false);
0553 desc.addUntracked<bool>("showJemallocInfo", true)
0554 ->setComment(
0555 "If enabled and jemalloc is being used, print high-level jemalloc statistics at the early termination "
0556 "and endJob printouts as well as for the peak VSIZE and RSS -using records.");
0557 desc.addUntracked<bool>("oncePerEventMode", false)
0558 ->setComment(
0559 "Only check memory at the end of each event. Not as useful in multi-threaded job as other running events "
0560 "contribute.");
0561 desc.addUntracked<bool>("jobReportOutputOnly", false);
0562 desc.addUntracked<bool>("monitorPssAndPrivate", false);
0563 desc.addUntracked<bool>("moduleMemorySummary", false)
0564 ->setComment(
0565 "Track significant memory events for each module. This does not work well in multi-threaded jobs.");
0566 descriptions.add("SimpleMemoryCheck", desc);
0567 }
0568
0569 std::optional<std::string> SimpleMemoryCheck::openFilesNoThrow() {
0570 #ifdef LINUX
0571 std::ostringstream smapsNameOst;
0572 smapsNameOst << "/proc/" << getpid() << "/smaps";
0573 auto smapsName = smapsNameOst.str();
0574 if ((smapsFile_ = fopen(smapsName.c_str(), "r")) == nullptr) {
0575 return smapsName;
0576 }
0577 #endif
0578 return {};
0579 }
0580
0581 void SimpleMemoryCheck::openFiles() {
0582 auto smapsFileNameIfFailed = openFilesNoThrow();
0583 if (smapsFileNameIfFailed.has_value()) {
0584 throw Exception(errors::Configuration) << "Failed to open smaps file " << *smapsFileNameIfFailed << std::endl;
0585 }
0586 }
0587
0588 void SimpleMemoryCheck::postBeginJob() {
0589 growthRateVsize_ = current_->vsize;
0590 growthRateRss_ = current_->rss;
0591 }
0592
0593 void SimpleMemoryCheck::preSourceConstruction(ModuleDescription const& md) {
0594 bool expected = false;
0595 if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0596 std::shared_ptr<void> guard(
0597 nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0598 updateAndPrint("pre-ctor", md.moduleLabel(), md.moduleName());
0599 }
0600 }
0601
0602 void SimpleMemoryCheck::postSourceConstruction(ModuleDescription const& md) {
0603 bool expected = false;
0604 if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0605 std::shared_ptr<void> guard(
0606 nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0607 updateAndPrint("ctor", md.moduleLabel(), md.moduleName());
0608 }
0609 }
0610
0611 void SimpleMemoryCheck::postSourceEvent(StreamID sid) {
0612 bool expected = false;
0613 if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0614 std::shared_ptr<void> guard(
0615 nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0616 updateAndPrint("module", "source", "source");
0617 }
0618 }
0619
0620 void SimpleMemoryCheck::postModuleConstruction(ModuleDescription const& md) {
0621 bool expected = false;
0622 if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0623 std::shared_ptr<void> guard(
0624 nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0625 updateAndPrint("ctor", md.moduleLabel(), md.moduleName());
0626 }
0627 }
0628
0629 void SimpleMemoryCheck::postModuleBeginJob(ModuleDescription const& md) {
0630 bool expected = false;
0631 if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0632 std::shared_ptr<void> guard(
0633 nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0634 updateAndPrint("beginJob", md.moduleLabel(), md.moduleName());
0635 }
0636 }
0637
0638 void SimpleMemoryCheck::earlyTermination() {
0639 bool expected = false;
0640 while (not measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0641 expected = false;
0642 }
0643 std::shared_ptr<void> guard(
0644 nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0645 if (not smapsFile_) {
0646 openFilesNoThrow();
0647 }
0648 if (smapsFile_) {
0649 currentSmaps_ = fetchSmaps();
0650 }
0651 update();
0652 andPrintAlways("earlyTermination", "", "", true);
0653 updateMax();
0654 }
0655
0656 void SimpleMemoryCheck::startSamplingThread() {
0657 samplingThread_ = std::thread{[this]() {
0658 while (not stopThread_) {
0659 std::this_thread::sleep_for(std::chrono::duration<unsigned int>(sampleEveryNSeconds_));
0660 ++count_;
0661 update();
0662 if (monitorPssAndPrivate_) {
0663 currentSmaps_ = fetchSmaps();
0664 }
0665 auto e = mostRecentlyStartedEvent_.load();
0666 andPrint("sampling", "", "");
0667 updateEventStats(e);
0668 updateMax();
0669 }
0670 }};
0671 }
0672 void SimpleMemoryCheck::stopSamplingThread() {
0673 stopThread_ = true;
0674 samplingThread_->join();
0675 }
0676
0677 void SimpleMemoryCheck::postEndJob() {
0678 if (not jobReportOutputOnly_) {
0679 LogAbsolute log("MemoryReport");
0680
0681 update();
0682 log << "MemoryReport> EndJob: virtual size " << current_->vsize << " Mbytes, RSS " << current_->rss
0683 << " Mbytes";
0684
0685 if (not smapsFile_) {
0686 openFilesNoThrow();
0687 }
0688 if (smapsFile_) {
0689 currentSmaps_ = fetchSmaps();
0690 auto soRss = currentSmaps_.sectionRss_[static_cast<unsigned>(SmapsSection::kSharedObject)];
0691 auto pcmRss = currentSmaps_.sectionRss_[static_cast<unsigned>(SmapsSection::kPcm)];
0692 auto otherFileRss = currentSmaps_.sectionRss_[static_cast<unsigned>(SmapsSection::kOtherFile)];
0693 auto mmapRss = currentSmaps_.sectionRss_[static_cast<unsigned>(SmapsSection::kMmap)];
0694 auto soVSize = currentSmaps_.sectionVSize_[static_cast<unsigned>(SmapsSection::kSharedObject)];
0695 auto pcmVSize = currentSmaps_.sectionVSize_[static_cast<unsigned>(SmapsSection::kPcm)];
0696 auto otherFileVSize = currentSmaps_.sectionVSize_[static_cast<unsigned>(SmapsSection::kOtherFile)];
0697 auto mmapVSize = currentSmaps_.sectionVSize_[static_cast<unsigned>(SmapsSection::kMmap)];
0698 log << ", PSS " << currentSmaps_.pss_ << " MBytes, Private " << currentSmaps_.private_ << "\n AnonHugePages "
0699 << currentSmaps_.anonHugePages_ << " Mbytes\n"
0700 << " mmapped memory pages " << mmapVSize << " Mbytes (VSize), " << mmapRss << " MBytes (RSS)\n"
0701 << " mmapped file pages " << (soVSize + pcmVSize + otherFileVSize) << " Mbytes (VSize), "
0702 << (soRss + pcmRss + otherFileRss) << " MBytes (RSS)\n"
0703 << " of which .so's " << soVSize << " Mbytes (VSize), " << soRss << " MBytes (RSS)\n"
0704 << " of which PCM's " << pcmVSize << " Mbytes (VSize), " << pcmRss << " MBytes (RSS)\n"
0705 << " of which other " << otherFileVSize << " Mbytes (VSize), " << otherFileRss << " MBytes (RSS)";
0706 }
0707 if (showJemallocInfo_) {
0708 auto info = fetchJemalloc();
0709 log << "\n Jemalloc allocated " << info.allocated << " MBytes, active " << info.active
0710 << " MBytes\n resident " << info.resident << " Mbytes, mapped " << info.mapped << " Mbytes\n metadata "
0711 << info.metadata << " Mbytes";
0712 }
0713 log << "\n";
0714
0715 auto logJemalloc = [&log](std::optional<JemallocInfo> const& info) {
0716 if (info.has_value()) {
0717 log << "\n Jemalloc allocated " << info->allocated << " active " << info->active << " resident "
0718 << info->resident << " mapped " << info->mapped << " metadata " << info->metadata;
0719 }
0720 };
0721
0722 log << "MemoryReport> Peak virtual size " << eventT1_.vsize << " Mbytes (RSS " << eventT1_.rss << ")";
0723 logJemalloc(eventT1_.jemalloc);
0724 log << "\n Key events increasing vsize: \n"
0725 << eventL2_ << "\n"
0726 << eventL1_ << "\n"
0727 << eventM_ << "\n"
0728 << eventR1_ << "\n"
0729 << eventR2_ << "\n"
0730 << eventT3_ << "\n"
0731 << eventT2_ << "\n"
0732 << eventT1_ << "\nMemoryReport> Peak rss size " << eventRssT1_.rss << " Mbytes (VSIZE " << eventRssT1_.vsize
0733 << ")";
0734 ;
0735 logJemalloc(eventRssT1_.jemalloc);
0736 log << "\n Key events increasing rss:\n"
0737 << eventRssT3_ << "\n"
0738 << eventRssT2_ << "\n"
0739 << eventRssT1_ << "\n"
0740 << eventDeltaRssT3_ << "\n"
0741 << eventDeltaRssT2_ << "\n"
0742 << eventDeltaRssT1_;
0743 }
0744 if (moduleSummaryRequested_ and not jobReportOutputOnly_) {
0745 LogAbsolute mmr("ModuleMemoryReport");
0746
0747
0748 mmr << "ModuleMemoryReport> Each line has module label and: \n";
0749 mmr << " (after early ignored events) \n";
0750 mmr << " count of times module executed; average increase in vsize \n";
0751 mmr << " maximum increase in vsize; event on which maximum occurred \n";
0752 mmr << " (during early ignored events) \n";
0753 mmr << " total and maximum vsize increases \n \n";
0754 for (SignificantModulesMap::iterator im = modules_.begin(); im != modules_.end(); ++im) {
0755 SignificantModule const& m = im->second;
0756 if (m.totalDeltaVsize == 0 && m.totalEarlyVsize == 0)
0757 continue;
0758 mmr << im->first << ": ";
0759 mmr << "n = " << m.postEarlyCount;
0760 if (m.postEarlyCount > 0) {
0761 mmr << " avg = " << m.totalDeltaVsize / m.postEarlyCount;
0762 }
0763 mmr << " max = " << m.maxDeltaVsize << " " << m.eventMaxDeltaV;
0764 if (m.totalEarlyVsize > 0) {
0765 mmr << " early total: " << m.totalEarlyVsize;
0766 mmr << " max: " << m.maxEarlyVsize;
0767 }
0768 mmr << "\n";
0769 }
0770 }
0771
0772 Service<JobReport> reportSvc;
0773
0774 #define SIMPLE_MEMORY_CHECK_ORIGINAL_XML_OUTPUT
0775 #ifdef SIMPLE_MEMORY_CHECK_ORIGINAL_XML_OUTPUT
0776
0777 std::map<std::string, std::string> reportData;
0778
0779 if (eventL2_.vsize > 0)
0780 eventStatOutput("LargeVsizeIncreaseEventL2", eventL2_, reportData);
0781 if (eventL1_.vsize > 0)
0782 eventStatOutput("LargeVsizeIncreaseEventL1", eventL1_, reportData);
0783 if (eventM_.vsize > 0)
0784 eventStatOutput("LargestVsizeIncreaseEvent", eventM_, reportData);
0785 if (eventR1_.vsize > 0)
0786 eventStatOutput("LargeVsizeIncreaseEventR1", eventR1_, reportData);
0787 if (eventR2_.vsize > 0)
0788 eventStatOutput("LargeVsizeIncreaseEventR2", eventR2_, reportData);
0789 if (eventT3_.vsize > 0)
0790 eventStatOutput("ThirdLargestVsizeEventT3", eventT3_, reportData);
0791 if (eventT2_.vsize > 0)
0792 eventStatOutput("SecondLargestVsizeEventT2", eventT2_, reportData);
0793 if (eventT1_.vsize > 0)
0794 eventStatOutput("LargestVsizeEventT1", eventT1_, reportData);
0795
0796 if (eventRssT3_.rss > 0)
0797 eventStatOutput("ThirdLargestRssEvent", eventRssT3_, reportData);
0798 if (eventRssT2_.rss > 0)
0799 eventStatOutput("SecondLargestRssEvent", eventRssT2_, reportData);
0800 if (eventRssT1_.rss > 0)
0801 eventStatOutput("LargestRssEvent", eventRssT1_, reportData);
0802 if (eventDeltaRssT3_.deltaRss > 0)
0803 eventStatOutput("ThirdLargestIncreaseRssEvent", eventDeltaRssT3_, reportData);
0804 if (eventDeltaRssT2_.deltaRss > 0)
0805 eventStatOutput("SecondLargestIncreaseRssEvent", eventDeltaRssT2_, reportData);
0806 if (eventDeltaRssT1_.deltaRss > 0)
0807 eventStatOutput("LargestIncreaseRssEvent", eventDeltaRssT1_, reportData);
0808
0809 #ifdef __linux__
0810 #if (__GLIBC__ > 2) || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33)
0811 struct mallinfo2 minfo = mallinfo2();
0812 #else
0813 struct mallinfo minfo = mallinfo();
0814 #endif
0815 reportData.insert(std::make_pair("HEAP_ARENA_SIZE_BYTES", std::to_string(minfo.arena)));
0816 reportData.insert(std::make_pair("HEAP_ARENA_N_UNUSED_CHUNKS", std::to_string(minfo.ordblks)));
0817 reportData.insert(std::make_pair("HEAP_TOP_FREE_BYTES", std::to_string(minfo.keepcost)));
0818 reportData.insert(std::make_pair("HEAP_MAPPED_SIZE_BYTES", std::to_string(minfo.hblkhd)));
0819 reportData.insert(std::make_pair("HEAP_MAPPED_N_CHUNKS", std::to_string(minfo.hblks)));
0820 reportData.insert(std::make_pair("HEAP_USED_BYTES", std::to_string(minfo.uordblks)));
0821 reportData.insert(std::make_pair("HEAP_UNUSED_BYTES", std::to_string(minfo.fordblks)));
0822 #endif
0823
0824
0825 reportData.insert(std::make_pair("AverageGrowthRateVsize",
0826 d2str(averageGrowthRate(current_->vsize, growthRateVsize_, count_))));
0827 reportData.insert(std::make_pair("PeakValueVsize", d2str(eventT1_.vsize)));
0828 reportData.insert(
0829 std::make_pair("AverageGrowthRateRss", d2str(averageGrowthRate(current_->rss, growthRateRss_, count_))));
0830 reportData.insert(std::make_pair("PeakValueRss", d2str(eventRssT1_.rss)));
0831
0832 if (moduleSummaryRequested_) {
0833 for (SignificantModulesMap::iterator im = modules_.begin(); im != modules_.end(); ++im) {
0834 SignificantModule const& m = im->second;
0835 if (m.totalDeltaVsize == 0 && m.totalEarlyVsize == 0)
0836 continue;
0837 std::string label = im->first + ":";
0838 reportData.insert(std::make_pair(label + "PostEarlyCount", i2str(m.postEarlyCount)));
0839 if (m.postEarlyCount > 0) {
0840 reportData.insert(std::make_pair(label + "AverageDeltaVsize", d2str(m.totalDeltaVsize / m.postEarlyCount)));
0841 }
0842 reportData.insert(std::make_pair(label + "MaxDeltaVsize", d2str(m.maxDeltaVsize)));
0843 if (m.totalEarlyVsize > 0) {
0844 reportData.insert(std::make_pair(label + "TotalEarlyVsize", d2str(m.totalEarlyVsize)));
0845 reportData.insert(std::make_pair(label + "MaxEarlyDeltaVsize", d2str(m.maxEarlyVsize)));
0846 }
0847 }
0848 }
0849
0850 std::map<std::string, std::string> reportMemoryProperties;
0851
0852 if (FILE* fmeminfo = fopen("/proc/meminfo", "r")) {
0853 char buf[128];
0854 char space[] = " ";
0855 size_t value;
0856 while (fgets(buf, sizeof(buf), fmeminfo)) {
0857 char* saveptr;
0858 char* token = nullptr;
0859 token = strtok_r(buf, space, &saveptr);
0860 if (token != nullptr) {
0861 value = atol(strtok_r(nullptr, space, &saveptr));
0862 std::string category = token;
0863 reportMemoryProperties.insert(std::make_pair(category.substr(0, strlen(token) - 1), i2str(value)));
0864 }
0865 }
0866
0867 fclose(fmeminfo);
0868 }
0869
0870
0871 reportSvc->reportPerformanceSummary("ApplicationMemory", reportData);
0872 reportSvc->reportPerformanceSummary("SystemMemory", reportMemoryProperties);
0873 #endif
0874
0875 #ifdef SIMPLE_MEMORY_CHECK_DIFFERENT_XML_OUTPUT
0876 std::vector<std::string> reportData;
0877
0878 if (eventL2_.vsize > 0)
0879 reportData.push_back(eventStatOutput("LargeVsizeIncreaseEventL2", eventL2_));
0880 if (eventL1_.vsize > 0)
0881 reportData.push_back(eventStatOutput("LargeVsizeIncreaseEventL1", eventL1_));
0882 if (eventM_.vsize > 0)
0883 reportData.push_back(eventStatOutput("LargestVsizeIncreaseEvent", eventM_));
0884 if (eventR1_.vsize > 0)
0885 reportData.push_back(eventStatOutput("LargeVsizeIncreaseEventR1", eventR1_));
0886 if (eventR2_.vsize > 0)
0887 reportData.push_back(eventStatOutput("LargeVsizeIncreaseEventR2", eventR2_));
0888 if (eventT3_.vsize > 0)
0889 reportData.push_back(eventStatOutput("ThirdLargestVsizeEventT3", eventT3_));
0890 if (eventT2_.vsize > 0)
0891 reportData.push_back(eventStatOutput("SecondLargestVsizeEventT2", eventT2_));
0892 if (eventT1_.vsize > 0)
0893 reportData.push_back(eventStatOutput("LargestVsizeEventT1", eventT1_));
0894
0895 if (eventRssT3_.rss > 0)
0896 eventStatOutput("ThirdLargestRssEvent", eventRssT3_, reportData);
0897 if (eventRssT2_.rss > 0)
0898 eventStatOutput("SecondLargestRssEvent", eventRssT2_, reportData);
0899 if (eventRssT1_.rss > 0)
0900 eventStatOutput("LargestRssEvent", eventRssT1_, reportData);
0901 if (eventDeltaRssT3_.deltaRss > 0)
0902 eventStatOutput("ThirdLargestIncreaseRssEvent", eventDeltaRssT3_, reportData);
0903 if (eventDeltaRssT2_.deltaRss > 0)
0904 eventStatOutput("SecondLargestIncreaseRssEvent", eventDeltaRssT2_, reportData);
0905 if (eventDeltaRssT1_.deltaRss > 0)
0906 eventStatOutput("LargestIncreaseRssEvent", eventDeltaRssT1_, reportData);
0907
0908 #if (__GLIBC__ > 2) || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33)
0909 struct mallinfo2 minfo = mallinfo2();
0910 #else
0911 struct mallinfo minfo = mallinfo();
0912 #endif
0913 reportData.push_back(mallOutput("HEAP_ARENA_SIZE_BYTES", minfo.arena));
0914 reportData.push_back(mallOutput("HEAP_ARENA_N_UNUSED_CHUNKS", minfo.ordblks));
0915 reportData.push_back(mallOutput("HEAP_TOP_FREE_BYTES", minfo.keepcost));
0916 reportData.push_back(mallOutput("HEAP_MAPPED_SIZE_BYTES", minfo.hblkhd));
0917 reportData.push_back(mallOutput("HEAP_MAPPED_N_CHUNKS", minfo.hblks));
0918 reportData.push_back(mallOutput("HEAP_USED_BYTES", minfo.uordblks));
0919 reportData.push_back(mallOutput("HEAP_UNUSED_BYTES", minfo.fordblks));
0920
0921
0922 reportData.insert(std::make_pair("AverageGrowthRateVsize",
0923 d2str(averageGrowthRate(current_->vsize, growthRateVsize_, count_))));
0924 reportData.insert(std::make_pair("PeakValueVsize", d2str(eventT1_.vsize)));
0925 reportData.insert(
0926 std::make_pair("AverageGrowthRateRss", d2str(averageGrowthRate(current_->rss, growthRateRss_, count_))));
0927 reportData.insert(std::make_pair("PeakValueRss", d2str(eventRssT1_.rss)));
0928
0929 reportSvc->reportMemoryInfo(reportData);
0930
0931 #endif
0932 }
0933
0934 void SimpleMemoryCheck::postEvent(StreamContext const& iContext) {
0935 ++count_;
0936 bool expected = false;
0937 if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0938 std::shared_ptr<void> guard(
0939 nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0940 update();
0941 if (monitorPssAndPrivate_) {
0942 currentSmaps_ = fetchSmaps();
0943 }
0944 updateEventStats(iContext.eventID());
0945 if (oncePerEventMode_) {
0946
0947 updateMax();
0948 andPrint("event", "", "");
0949 }
0950 }
0951 }
0952
0953 void SimpleMemoryCheck::preModule(StreamContext const& iStreamContext, ModuleCallingContext const& iModuleContext) {
0954 bool expectedMeasurementUnderway = false;
0955 if (measurementUnderway_.compare_exchange_strong(expectedMeasurementUnderway, true, std::memory_order_acq_rel)) {
0956 std::shared_ptr<void> guard(
0957 nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0958 bool expectedModuleMeasurementUnderway = false;
0959 if (moduleMeasurementUnderway_.compare_exchange_strong(expectedModuleMeasurementUnderway, true)) {
0960 update();
0961
0962 moduleEntryVsize_ = current_->vsize;
0963 moduleStreamID_.store(iStreamContext.streamID().value(), std::memory_order_release);
0964 moduleID_.store(iModuleContext.moduleDescription()->id(), std::memory_order_release);
0965 }
0966 }
0967 }
0968
0969 void SimpleMemoryCheck::postModule(StreamContext const& iStreamContext,
0970 ModuleCallingContext const& iModuleContext) {
0971 if (!oncePerEventMode_) {
0972 bool expected = false;
0973 if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0974 std::shared_ptr<void> guard(
0975 nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0976 auto const md = iModuleContext.moduleDescription();
0977 updateAndPrint("module", md->moduleLabel(), md->moduleName());
0978 }
0979 }
0980
0981 if (moduleSummaryRequested_) {
0982
0983 if (moduleMeasurementUnderway_.load(std::memory_order_acquire) and
0984 (iStreamContext.streamID().value() == moduleStreamID_.load(std::memory_order_acquire)) and
0985 (iModuleContext.moduleDescription()->id() == moduleID_.load(std::memory_order_acquire))) {
0986
0987 std::shared_ptr<void> guardModuleMeasurementUnderway(
0988 nullptr, [this](void const*) { moduleMeasurementUnderway_.store(false, std::memory_order_release); });
0989 bool expected = false;
0990 if (measurementUnderway_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) {
0991 std::shared_ptr<void> guardMeasurementUnderway(
0992 nullptr, [this](void const*) { measurementUnderway_.store(false, std::memory_order_release); });
0993 if (oncePerEventMode_) {
0994 update();
0995 }
0996
0997 double dv = current_->vsize - moduleEntryVsize_;
0998 std::string label = iModuleContext.moduleDescription()->moduleLabel();
0999 updateModuleMemoryStats(modules_[label], dv, iStreamContext.eventID());
1000 }
1001 }
1002 }
1003 }
1004
1005 void SimpleMemoryCheck::update() {
1006 std::swap(current_, previous_);
1007 *current_ = fetch();
1008 }
1009
1010 void SimpleMemoryCheck::updateMax() {
1011 auto v = *current_;
1012 if ((v > max_) || oncePerEventMode_) {
1013 if (max_.vsize < v.vsize) {
1014 max_.vsize = v.vsize;
1015 }
1016 if (max_.rss < v.rss) {
1017 max_.rss = v.rss;
1018 }
1019 }
1020 }
1021
1022 void SimpleMemoryCheck::updateEventStats(EventID const& e) {
1023 if (count_ < num_to_skip_)
1024 return;
1025 if (count_ == num_to_skip_) {
1026 eventT1_.set(0, 0, e, this);
1027 eventM_.set(0, 0, e, this);
1028 eventRssT1_.set(0, 0, e, this);
1029 eventDeltaRssT1_.set(0, 0, e, this);
1030 return;
1031 }
1032 double vsize = current_->vsize;
1033 double deltaVsize = vsize - eventT1_.vsize;
1034
1035
1036 if (vsize > eventT1_.vsize) {
1037 double deltaRss = current_->rss - eventT1_.rss;
1038 eventT3_ = eventT2_;
1039 eventT2_ = eventT1_;
1040 eventT1_.set(deltaVsize, deltaRss, e, this);
1041 } else if (vsize > eventT2_.vsize) {
1042 double deltaRss = current_->rss - eventT1_.rss;
1043 eventT3_ = eventT2_;
1044 eventT2_.set(deltaVsize, deltaRss, e, this);
1045 } else if (vsize > eventT3_.vsize) {
1046 double deltaRss = current_->rss - eventT1_.rss;
1047 eventT3_.set(deltaVsize, deltaRss, e, this);
1048 }
1049
1050 if (deltaVsize > eventM_.deltaVsize) {
1051 double deltaRss = current_->rss - eventM_.rss;
1052 if (eventL1_.deltaVsize >= eventR1_.deltaVsize) {
1053 eventL2_ = eventL1_;
1054 } else {
1055 eventL2_ = eventR1_;
1056 }
1057 eventL1_ = eventM_;
1058 eventM_.set(deltaVsize, deltaRss, e, this);
1059 eventR1_ = SignificantEvent();
1060 eventR2_ = SignificantEvent();
1061 } else if (deltaVsize > eventR1_.deltaVsize) {
1062 double deltaRss = current_->rss - eventM_.rss;
1063 eventR2_ = eventR1_;
1064 eventR1_.set(deltaVsize, deltaRss, e, this);
1065 } else if (deltaVsize > eventR2_.deltaVsize) {
1066 double deltaRss = current_->rss - eventR1_.rss;
1067 eventR2_.set(deltaVsize, deltaRss, e, this);
1068 }
1069
1070
1071 double rss = current_->rss;
1072 double deltaRss = rss - eventRssT1_.rss;
1073
1074 if (rss > eventRssT1_.rss) {
1075 eventRssT3_ = eventRssT2_;
1076 eventRssT2_ = eventRssT1_;
1077 eventRssT1_.set(deltaVsize, deltaRss, e, this);
1078 } else if (rss > eventRssT2_.rss) {
1079 eventRssT3_ = eventRssT2_;
1080 eventRssT2_.set(deltaVsize, deltaRss, e, this);
1081 } else if (rss > eventRssT3_.rss) {
1082 eventRssT3_.set(deltaVsize, deltaRss, e, this);
1083 }
1084 if (deltaRss > eventDeltaRssT1_.deltaRss) {
1085 eventDeltaRssT3_ = eventDeltaRssT2_;
1086 eventDeltaRssT2_ = eventDeltaRssT1_;
1087 eventDeltaRssT1_.set(deltaVsize, deltaRss, e, this);
1088 } else if (deltaRss > eventDeltaRssT2_.deltaRss) {
1089 eventDeltaRssT3_ = eventDeltaRssT2_;
1090 eventDeltaRssT2_.set(deltaVsize, deltaRss, e, this);
1091 } else if (deltaRss > eventDeltaRssT3_.deltaRss) {
1092 eventDeltaRssT3_.set(deltaVsize, deltaRss, e, this);
1093 }
1094 }
1095
1096 void SimpleMemoryCheck::andPrintAlways(std::string const& type,
1097 std::string const& mdlabel,
1098 std::string const& mdname,
1099 bool includeSmapsAndJe) const {
1100 double deltaVSIZE = current_->vsize - max_.vsize;
1101 double deltaRSS = current_->rss - max_.rss;
1102
1103 LogWarning log("MemoryCheck");
1104
1105 log << "MemoryCheck: " << type << " " << mdname << ":" << mdlabel << " VSIZE " << current_->vsize << " "
1106 << deltaVSIZE << " RSS " << current_->rss << " " << deltaRSS;
1107 if (showMallocInfo_) {
1108 #ifdef __linux__
1109 #if (__GLIBC__ > 2) || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33)
1110 struct mallinfo2 minfo = mallinfo2();
1111 #else
1112 struct mallinfo minfo = mallinfo();
1113 #endif
1114 log << " HEAP-ARENA [ SIZE-BYTES " << minfo.arena << " N-UNUSED-CHUNKS " << minfo.ordblks << " TOP-FREE-BYTES "
1115 << minfo.keepcost << " ]"
1116 << " HEAP-MAPPED [ SIZE-BYTES " << minfo.hblkhd << " N-CHUNKS " << minfo.hblks << " ]"
1117 << " HEAP-USED-BYTES " << minfo.uordblks << " HEAP-UNUSED-BYTES " << minfo.fordblks;
1118 #endif
1119 }
1120 if (includeSmapsAndJe) {
1121 if (smapsFile_) {
1122 log << " PSS " << currentSmaps_.pss_ << " PRIVATE " << currentSmaps_.private_ << " ANONHUGEPAGES "
1123 << currentSmaps_.anonHugePages_;
1124 }
1125 if (je_mallctl) {
1126 auto info = fetchJemalloc();
1127 log << " JeMalloc allocated " << info.allocated << " active " << info.active << " resident " << info.resident
1128 << " mapped " << info.mapped << " metadata " << info.metadata;
1129 }
1130 }
1131 }
1132
1133 void SimpleMemoryCheck::andPrint(std::string const& type,
1134 std::string const& mdlabel,
1135 std::string const& mdname) const {
1136 if (not jobReportOutputOnly_ && ((*current_ > max_) || printEachTime_)) {
1137 if (count_ >= num_to_skip_) {
1138 andPrintAlways(type, mdlabel, mdname);
1139 }
1140 }
1141 }
1142
1143 void SimpleMemoryCheck::updateAndPrint(std::string const& type,
1144 std::string const& mdlabel,
1145 std::string const& mdname) {
1146 update();
1147 andPrint(type, mdlabel, mdname);
1148 updateMax();
1149 }
1150
1151 #ifdef SIMPLE_MEMORY_CHECK_ORIGINAL_XML_OUTPUT
1152 void SimpleMemoryCheck::eventStatOutput(std::string title,
1153 SignificantEvent const& e,
1154 std::map<std::string, std::string>& m) const {
1155 {
1156 std::ostringstream os;
1157 os << title << "-a-COUNT";
1158 m.insert(std::make_pair(os.str(), i2str(e.count)));
1159 }
1160 {
1161 std::ostringstream os;
1162 os << title << "-b-RUN";
1163 m.insert(std::make_pair(os.str(), d2str(static_cast<double>(e.event.run()))));
1164 }
1165 {
1166 std::ostringstream os;
1167 os << title << "-c-EVENT";
1168 m.insert(std::make_pair(os.str(), d2str(static_cast<double>(e.event.event()))));
1169 }
1170 {
1171 std::ostringstream os;
1172 os << title << "-d-VSIZE";
1173 m.insert(std::make_pair(os.str(), d2str(e.vsize)));
1174 }
1175 {
1176 std::ostringstream os;
1177 os << title << "-e-DELTV";
1178 m.insert(std::make_pair(os.str(), d2str(e.deltaVsize)));
1179 }
1180 {
1181 std::ostringstream os;
1182 os << title << "-f-RSS";
1183 m.insert(std::make_pair(os.str(), d2str(e.rss)));
1184 }
1185 if (monitorPssAndPrivate_) {
1186 {
1187 std::ostringstream os;
1188 os << title << "-g-PRIVATE";
1189 m.insert(std::make_pair(os.str(), d2str(e.privateSize)));
1190 }
1191 {
1192 std::ostringstream os;
1193 os << title << "-h-PSS";
1194 m.insert(std::make_pair(os.str(), d2str(e.pss)));
1195 }
1196 }
1197 }
1198 #endif
1199
1200 #ifdef SIMPLE_MEMORY_CHECK_DIFFERENT_XML_OUTPUT
1201 std::string SimpleMemoryCheck::eventStatOutput(std::string title, SignificantEvent const& e) const {
1202 std::ostringstream os;
1203 os << " <" << title << ">\n";
1204 os << " " << e.count << ": " << e.event;
1205 os << " vsize " << e.vsize - e.deltaVsize << " + " << e.deltaVsize << " = " << e.vsize;
1206 os << " rss: " << e.rss << "\n";
1207 os << " </" << title << ">\n";
1208 return os.str();
1209 }
1210
1211 std::string SimpleMemoryCheck::mallOutput(std::string title, size_t const& n) const {
1212 std::ostringstream os;
1213 os << " <" << title << ">\n";
1214 os << " " << n << "\n";
1215 os << " </" << title << ">\n";
1216 return os.str();
1217 }
1218 #endif
1219
1220 void SimpleMemoryCheck::updateModuleMemoryStats(SignificantModule& m,
1221 double dv,
1222 edm::EventID const& currentEventID) {
1223 if (count_ < num_to_skip_) {
1224 m.totalEarlyVsize += dv;
1225 if (dv > m.maxEarlyVsize)
1226 m.maxEarlyVsize = dv;
1227 } else {
1228 ++m.postEarlyCount;
1229 m.totalDeltaVsize += dv;
1230 if (dv > m.maxDeltaVsize) {
1231 m.maxDeltaVsize = dv;
1232 m.eventMaxDeltaV = currentEventID;
1233 }
1234 }
1235 }
1236
1237 std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantEvent const& se) {
1238 os << "[" << se.count << "] " << se.event << " vsize = " << se.vsize << " deltaVsize = " << se.deltaVsize
1239 << " rss = " << se.rss << " delta = " << se.deltaRss;
1240
1241 if (se.monitorPssAndPrivate) {
1242 os << " private = " << se.privateSize << " pss = " << se.pss;
1243 }
1244 if (se.jemalloc.has_value()) {
1245 os << " allocated = " << se.jemalloc->allocated << " active = " << se.jemalloc->active;
1246 }
1247 return os;
1248 }
1249
1250 std::ostream& operator<<(std::ostream& os, SimpleMemoryCheck::SignificantModule const& sm) {
1251 if (sm.postEarlyCount > 0) {
1252 os << "\nPost Early Events: TotalDeltaVsize: " << sm.totalDeltaVsize
1253 << " (avg: " << sm.totalDeltaVsize / sm.postEarlyCount << "; max: " << sm.maxDeltaVsize << " during "
1254 << sm.eventMaxDeltaV << ")";
1255 }
1256 if (sm.totalEarlyVsize > 0) {
1257 os << "\n Early Events: TotalDeltaVsize: " << sm.totalEarlyVsize << " (max: " << sm.maxEarlyVsize << ")";
1258 }
1259
1260 return os;
1261 }
1262
1263 }
1264 }
1265
1266 #if defined(__linux__)
1267 using edm::service::SimpleMemoryCheck;
1268 DEFINE_FWK_SERVICE(SimpleMemoryCheck);
1269 #endif