File indexing completed on 2023-06-26 23:21:21
0001 #include "FWCore/Utilities/interface/RootHandlers.h"
0002
0003 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0004 #include "FWCore/ServiceRegistry/interface/SystemBounds.h"
0005 #include "DataFormats/Common/interface/RefCoreStreamer.h"
0006 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0007 #include "FWCore/MessageLogger/interface/ELseverityLevel.h"
0008 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0009 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0010 #include "FWCore/PluginManager/interface/PluginCapabilities.h"
0011 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0012 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0013 #include "FWCore/Utilities/interface/EDMException.h"
0014 #include "FWCore/Reflection/interface/TypeWithDict.h"
0015 #include "FWCore/Utilities/interface/UnixSignalHandlers.h"
0016 #include "FWCore/ServiceRegistry/interface/CurrentModuleOnThread.h"
0017 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0018
0019 #include "oneapi/tbb/concurrent_unordered_set.h"
0020 #include "oneapi/tbb/task.h"
0021 #include "oneapi/tbb/task_scheduler_observer.h"
0022 #include "oneapi/tbb/global_control.h"
0023 #include <memory>
0024
0025 #include <thread>
0026 #include <sys/wait.h>
0027 #include <sstream>
0028 #include <cstring>
0029 #include <poll.h>
0030 #include <atomic>
0031 #include <algorithm>
0032 #include <vector>
0033 #include <string>
0034 #include <array>
0035
0036
0037
0038
0039 #ifdef __linux__
0040 #include <syscall.h>
0041 #endif
0042
0043 #include "TROOT.h"
0044 #include "TError.h"
0045 #include "TFile.h"
0046 #include "TInterpreter.h"
0047 #include "TH1.h"
0048 #include "TSystem.h"
0049 #include "TUnixSystem.h"
0050 #include "TTree.h"
0051 #include "TVirtualStreamerInfo.h"
0052
0053 #include "TClassTable.h"
0054
0055 #include <memory>
0056
0057 namespace {
0058
0059
0060 constexpr std::size_t moduleBufferSize = 128;
0061 }
0062
0063 namespace edm {
0064 class ConfigurationDescriptions;
0065 class ParameterSet;
0066 class ActivityRegistry;
0067
0068 namespace service {
0069 class InitRootHandlers : public RootHandlers {
0070 friend int cmssw_stacktrace(void*);
0071
0072 public:
0073 class ThreadTracker : public oneapi::tbb::task_scheduler_observer {
0074 public:
0075 typedef oneapi::tbb::concurrent_unordered_set<pthread_t> Container_type;
0076
0077 ThreadTracker() : oneapi::tbb::task_scheduler_observer() { observe(); }
0078 ~ThreadTracker() override = default;
0079
0080 void on_scheduler_entry(bool) override {
0081
0082
0083
0084
0085
0086 edm::CurrentModuleOnThread::getCurrentModuleOnThread();
0087 threadIDs_.insert(pthread_self());
0088 }
0089 void on_scheduler_exit(bool) override {}
0090 const Container_type& IDs() { return threadIDs_; }
0091
0092 private:
0093 Container_type threadIDs_;
0094 };
0095
0096 explicit InitRootHandlers(ParameterSet const& pset, ActivityRegistry& iReg);
0097 ~InitRootHandlers() override;
0098
0099 static void fillDescriptions(ConfigurationDescriptions& descriptions);
0100 static void stacktraceFromThread();
0101 static const ThreadTracker::Container_type& threadIDs() {
0102 static const ThreadTracker::Container_type empty;
0103 if (threadTracker_) {
0104 return threadTracker_->IDs();
0105 }
0106 return empty;
0107 }
0108 static int stackTracePause() { return stackTracePause_; }
0109
0110 static std::vector<std::array<char, moduleBufferSize>> moduleListBuffers_;
0111 static std::atomic<std::size_t> nextModule_, doneModules_;
0112
0113 private:
0114 static char const* const* getPstackArgv();
0115 void enableWarnings_() override;
0116 void ignoreWarnings_(edm::RootHandlers::SeverityLevel level) override;
0117 void willBeUsingThreads() override;
0118
0119 void cachePidInfo();
0120 static void stacktraceHelperThread();
0121
0122 static constexpr int pidStringLength_ = 200;
0123 static char pidString_[pidStringLength_];
0124 static char const* const pstackArgv_[];
0125 static int parentToChild_[2];
0126 static int childToParent_[2];
0127 static std::unique_ptr<std::thread> helperThread_;
0128 static std::unique_ptr<ThreadTracker> threadTracker_;
0129 static int stackTracePause_;
0130
0131 bool unloadSigHandler_;
0132 bool resetErrHandler_;
0133 bool loadAllDictionaries_;
0134 bool autoLibraryLoader_;
0135 bool autoClassParser_;
0136 bool interactiveDebug_;
0137 std::shared_ptr<const void> sigBusHandler_;
0138 std::shared_ptr<const void> sigSegvHandler_;
0139 std::shared_ptr<const void> sigIllHandler_;
0140 std::shared_ptr<const void> sigTermHandler_;
0141 std::shared_ptr<const void> sigAbrtHandler_;
0142 std::shared_ptr<const void> sigFpeHandler_;
0143 };
0144
0145 inline bool isProcessWideService(InitRootHandlers const*) { return true; }
0146
0147 }
0148 }
0149
0150 namespace edm {
0151 namespace service {
0152 int cmssw_stacktrace(void*);
0153 }
0154 }
0155
0156 namespace {
0157 thread_local edm::RootHandlers::SeverityLevel s_ignoreWarnings = edm::RootHandlers::SeverityLevel::kInfo;
0158
0159 constexpr bool s_ignoreEverything = false;
0160
0161 template <std::size_t SIZE>
0162 bool find_if_string(const std::string& search, const std::array<const char* const, SIZE>& substrs) {
0163 return (std::find_if(substrs.begin(), substrs.end(), [&search](const char* const s) -> bool {
0164 return (search.find(s) != std::string::npos);
0165 }) != substrs.end());
0166 }
0167
0168
0169 constexpr std::array<const char* const, 9> in_message{
0170 {"no dictionary for class",
0171 "already in TClassTable",
0172 "matrix not positive definite",
0173 "not a TStreamerInfo object",
0174 "Problems declaring payload",
0175 "Announced number of args different from the real number of argument passed",
0176 "nbins is <=0 - set to nbins = 1",
0177 "nbinsy is <=0 - set to nbinsy = 1",
0178 "oneapi::tbb::global_control is limiting"}};
0179
0180
0181 constexpr std::array<const char* const, 7> in_location{{"Fit",
0182 "TDecompChol::Solve",
0183 "THistPainter::PaintInit",
0184 "TUnixSystem::SetDisplay",
0185 "TGClient::GetFontByName",
0186 "Inverter::Dinv",
0187 "RTaskArenaWrapper"}};
0188
0189 constexpr std::array<const char* const, 3> in_message_print_error{{"number of iterations was insufficient",
0190 "bad integrand behavior",
0191 "integral is divergent, or slowly convergent"}};
0192
0193 void RootErrorHandlerImpl(int level, char const* location, char const* message) {
0194 bool die = false;
0195
0196
0197
0198 edm::RootHandlers::SeverityLevel el_severity = edm::RootHandlers::SeverityLevel::kInfo;
0199
0200 if (level >= kFatal) {
0201 el_severity = edm::RootHandlers::SeverityLevel::kFatal;
0202 } else if (level >= kSysError) {
0203 el_severity = edm::RootHandlers::SeverityLevel::kSysError;
0204 } else if (level >= kError) {
0205 el_severity = edm::RootHandlers::SeverityLevel::kError;
0206 } else if (level >= kWarning) {
0207 el_severity = edm::RootHandlers::SeverityLevel::kWarning;
0208 }
0209
0210 if (s_ignoreEverything || el_severity <= s_ignoreWarnings) {
0211 el_severity = edm::RootHandlers::SeverityLevel::kInfo;
0212 }
0213
0214
0215
0216
0217 std::string el_location = "@SUB=?";
0218 if (location != nullptr)
0219 el_location = std::string("@SUB=") + std::string(location);
0220
0221 std::string el_message = "?";
0222 if (message != nullptr)
0223 el_message = message;
0224
0225
0226
0227
0228
0229
0230
0231 std::string el_identifier = "ROOT";
0232
0233 std::string precursor("class ");
0234 size_t index1 = el_message.find(precursor);
0235 if (index1 != std::string::npos) {
0236 size_t index2 = index1 + precursor.length();
0237 size_t index3 = el_message.find_first_of(" :", index2);
0238 if (index3 != std::string::npos) {
0239 size_t substrlen = index3 - index2;
0240 el_identifier += "-";
0241 el_identifier += el_message.substr(index2, substrlen);
0242 }
0243 } else {
0244 index1 = el_location.find("::");
0245 if (index1 != std::string::npos) {
0246 el_identifier += "/";
0247 el_identifier += el_location.substr(0, index1);
0248 }
0249 }
0250
0251
0252
0253 if ((el_location.find("TBranchElement::Fill") != std::string::npos) &&
0254 (el_message.find("fill branch") != std::string::npos) && (el_message.find("address") != std::string::npos) &&
0255 (el_message.find("not set") != std::string::npos)) {
0256 el_severity = edm::RootHandlers::SeverityLevel::kFatal;
0257 }
0258
0259 if ((el_message.find("Tree branches") != std::string::npos) &&
0260 (el_message.find("different numbers of entries") != std::string::npos)) {
0261 el_severity = edm::RootHandlers::SeverityLevel::kFatal;
0262 }
0263
0264
0265
0266 if (find_if_string(el_message, in_message) || find_if_string(el_location, in_location) ||
0267 (level < kError and (el_location.find("CINTTypedefBuilder::Setup") != std::string::npos) and
0268 (el_message.find("possible entries are in use!") != std::string::npos))) {
0269 el_severity = edm::RootHandlers::SeverityLevel::kInfo;
0270 }
0271
0272
0273
0274 bool alreadyPrinted = false;
0275 if (find_if_string(el_message, in_message_print_error)) {
0276 el_severity = edm::RootHandlers::SeverityLevel::kInfo;
0277 edm::LogError("Root_Error") << el_location << el_message;
0278 alreadyPrinted = true;
0279 }
0280
0281 if (el_severity == edm::RootHandlers::SeverityLevel::kInfo) {
0282
0283 die = false;
0284 } else {
0285 die = true;
0286 }
0287
0288
0289
0290
0291
0292
0293 if (die && (el_location != std::string("@SUB=TUnixSystem::DispatchSignals"))) {
0294 std::ostringstream sstr;
0295 sstr << "Fatal Root Error: " << el_location << "\n" << el_message << '\n';
0296 edm::Exception except(edm::errors::FatalRootError, sstr.str());
0297 except.addAdditionalInfo(except.message());
0298 except.clearMessage();
0299 throw except;
0300 }
0301
0302
0303
0304
0305 if (!alreadyPrinted) {
0306 if (el_severity == edm::RootHandlers::SeverityLevel::kFatal) {
0307 edm::LogError("Root_Fatal") << el_location << el_message;
0308 } else if (el_severity == edm::RootHandlers::SeverityLevel::kSysError) {
0309 edm::LogError("Root_Severe") << el_location << el_message;
0310 } else if (el_severity == edm::RootHandlers::SeverityLevel::kError) {
0311 edm::LogError("Root_Error") << el_location << el_message;
0312 } else if (el_severity == edm::RootHandlers::SeverityLevel::kWarning) {
0313 edm::LogWarning("Root_Warning") << el_location << el_message;
0314 } else if (el_severity == edm::RootHandlers::SeverityLevel::kInfo) {
0315 edm::LogInfo("Root_Information") << el_location << el_message;
0316 }
0317 }
0318 }
0319
0320 void RootErrorHandler(int level, bool, char const* location, char const* message) {
0321 RootErrorHandlerImpl(level, location, message);
0322 }
0323
0324 extern "C" {
0325 void set_default_signals() {
0326 signal(SIGILL, SIG_DFL);
0327 signal(SIGSEGV, SIG_DFL);
0328 signal(SIGBUS, SIG_DFL);
0329 signal(SIGTERM, SIG_DFL);
0330 signal(SIGFPE, SIG_DFL);
0331 signal(SIGABRT, SIG_DFL);
0332 }
0333
0334 static int full_write(int fd, const char* text) {
0335 const char* buffer = text;
0336 size_t count = strlen(text);
0337 ssize_t written = 0;
0338 while (count) {
0339 written = write(fd, buffer, count);
0340 if (written == -1) {
0341 if (errno == EINTR) {
0342 continue;
0343 } else {
0344 return -errno;
0345 }
0346 }
0347 count -= written;
0348 buffer += written;
0349 }
0350 return 0;
0351 }
0352
0353 static int full_read(int fd, char* inbuf, size_t len, int timeout_s = -1) {
0354 char* buf = inbuf;
0355 size_t count = len;
0356 ssize_t complete = 0;
0357 std::chrono::time_point<std::chrono::steady_clock> end_time =
0358 std::chrono::steady_clock::now() + std::chrono::seconds(timeout_s);
0359 int flags;
0360 if (timeout_s < 0) {
0361 flags = O_NONBLOCK;
0362 } else if ((-1 == (flags = fcntl(fd, F_GETFL)))) {
0363 return -errno;
0364 }
0365 if ((flags & O_NONBLOCK) != O_NONBLOCK) {
0366 if (-1 == fcntl(fd, F_SETFL, flags | O_NONBLOCK)) {
0367 return -errno;
0368 }
0369 }
0370 while (count) {
0371 if (timeout_s >= 0) {
0372 struct pollfd poll_info {
0373 fd, POLLIN, 0
0374 };
0375 int ms_remaining =
0376 std::chrono::duration_cast<std::chrono::milliseconds>(end_time - std::chrono::steady_clock::now()).count();
0377 if (ms_remaining > 0) {
0378 int rc = poll(&poll_info, 1, ms_remaining);
0379 if (rc <= 0) {
0380 if (rc < 0) {
0381 if (errno == EINTR || errno == EAGAIN) {
0382 continue;
0383 }
0384 rc = -errno;
0385 } else {
0386 rc = -ETIMEDOUT;
0387 }
0388 if ((flags & O_NONBLOCK) != O_NONBLOCK) {
0389 fcntl(fd, F_SETFL, flags);
0390 }
0391 return rc;
0392 }
0393 } else if (ms_remaining < 0) {
0394 if ((flags & O_NONBLOCK) != O_NONBLOCK) {
0395 fcntl(fd, F_SETFL, flags);
0396 }
0397 return -ETIMEDOUT;
0398 }
0399 }
0400 complete = read(fd, buf, count);
0401 if (complete == -1) {
0402 if (errno == EINTR) {
0403 continue;
0404 } else if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) {
0405 continue;
0406 } else {
0407 int orig_errno = errno;
0408 if ((flags & O_NONBLOCK) != O_NONBLOCK) {
0409 fcntl(fd, F_SETFL, flags);
0410 }
0411 return -orig_errno;
0412 }
0413 }
0414 count -= complete;
0415 buf += complete;
0416 }
0417 if ((flags & O_NONBLOCK) != O_NONBLOCK) {
0418 fcntl(fd, F_SETFL, flags);
0419 }
0420 return 0;
0421 }
0422
0423 static int full_cerr_write(const char* text) { return full_write(2, text); }
0424
0425
0426
0427
0428
0429 #if defined(SIGRTMAX)
0430 #define PAUSE_SIGNAL SIGRTMAX
0431 #define RESUME_SIGNAL SIGRTMAX - 1
0432 #elif defined(SIGINFO)
0433 #define PAUSE_SIGNAL SIGINFO
0434 #define RESUME_SIGNAL SIGALRM
0435 #endif
0436
0437
0438 void sig_resume_handler(int sig, siginfo_t*, void*) {}
0439
0440
0441 void sig_pause_for_stacktrace(int sig, siginfo_t*, void*) {
0442 using namespace edm::service;
0443
0444 #ifdef RESUME_SIGNAL
0445 sigset_t sigset;
0446 sigemptyset(&sigset);
0447 sigaddset(&sigset, RESUME_SIGNAL);
0448 pthread_sigmask(SIG_UNBLOCK, &sigset, nullptr);
0449 #endif
0450
0451 sleep(InitRootHandlers::stackTracePause());
0452
0453 if (InitRootHandlers::doneModules_.is_lock_free() && InitRootHandlers::nextModule_.is_lock_free()) {
0454 auto i = InitRootHandlers::nextModule_++;
0455 if (i < InitRootHandlers::moduleListBuffers_.size()) {
0456 char* buff = InitRootHandlers::moduleListBuffers_[i].data();
0457
0458 strlcpy(buff, "\nModule: ", moduleBufferSize);
0459 if (edm::CurrentModuleOnThread::getCurrentModuleOnThread() != nullptr) {
0460 strlcat(buff,
0461 edm::CurrentModuleOnThread::getCurrentModuleOnThread()->moduleDescription()->moduleName().c_str(),
0462 moduleBufferSize);
0463 strlcat(buff, ":", moduleBufferSize);
0464 strlcat(buff,
0465 edm::CurrentModuleOnThread::getCurrentModuleOnThread()->moduleDescription()->moduleLabel().c_str(),
0466 moduleBufferSize);
0467 } else {
0468 strlcat(buff, "none", moduleBufferSize);
0469 }
0470 ++edm::service::InitRootHandlers::doneModules_;
0471 }
0472 }
0473 }
0474
0475 void sig_dostack_then_abort(int sig, siginfo_t*, void*) {
0476 using namespace edm::service;
0477
0478 const auto& tids = InitRootHandlers::threadIDs();
0479
0480 const auto self = pthread_self();
0481 #ifdef PAUSE_SIGNAL
0482 if (InitRootHandlers::stackTracePause() > 0 && tids.size() > 1) {
0483
0484 struct sigaction act;
0485 act.sa_sigaction = sig_pause_for_stacktrace;
0486 act.sa_flags = 0;
0487 sigemptyset(&act.sa_mask);
0488 sigaction(PAUSE_SIGNAL, &act, nullptr);
0489
0490
0491 sigset_t pausesigset;
0492 sigemptyset(&pausesigset);
0493 sigaddset(&pausesigset, PAUSE_SIGNAL);
0494 sigprocmask(SIG_UNBLOCK, &pausesigset, nullptr);
0495
0496
0497 for (auto id : tids) {
0498 if (self != id) {
0499 pthread_kill(id, PAUSE_SIGNAL);
0500 }
0501 }
0502
0503 #ifdef RESUME_SIGNAL
0504
0505 act.sa_sigaction = sig_resume_handler;
0506 sigaction(RESUME_SIGNAL, &act, nullptr);
0507 #endif
0508 }
0509 #endif
0510
0511 const char* signalname = "unknown";
0512 switch (sig) {
0513 case SIGBUS: {
0514 signalname = "bus error";
0515 break;
0516 }
0517 case SIGSEGV: {
0518 signalname = "segmentation violation";
0519 break;
0520 }
0521 case SIGILL: {
0522 signalname = "illegal instruction";
0523 break;
0524 }
0525 case SIGFPE: {
0526 signalname = "floating point exception";
0527 break;
0528 }
0529 case SIGTERM: {
0530 signalname = "external termination request";
0531 break;
0532 }
0533 case SIGABRT: {
0534 signalname = "abort signal";
0535 break;
0536 }
0537 default:
0538 break;
0539 }
0540 full_cerr_write("\n\nA fatal system signal has occurred: ");
0541 full_cerr_write(signalname);
0542 full_cerr_write("\nThe following is the call stack containing the origin of the signal.\n\n");
0543
0544 edm::service::InitRootHandlers::stacktraceFromThread();
0545
0546
0547
0548
0549
0550 #ifdef RESUME_SIGNAL
0551 std::size_t notified = 0;
0552 if (InitRootHandlers::stackTracePause() > 0 && tids.size() > 1) {
0553 for (auto id : tids) {
0554 if (self != id) {
0555 if (pthread_kill(id, RESUME_SIGNAL) == 0)
0556 ++notified;
0557 }
0558 }
0559 }
0560 #endif
0561
0562 full_cerr_write("\nCurrent Modules:\n");
0563
0564
0565
0566
0567
0568
0569 if (tids.count(self) > 0) {
0570 char buff[moduleBufferSize] = "\nModule: ";
0571 if (edm::CurrentModuleOnThread::getCurrentModuleOnThread() != nullptr) {
0572 strlcat(buff,
0573 edm::CurrentModuleOnThread::getCurrentModuleOnThread()->moduleDescription()->moduleName().c_str(),
0574 moduleBufferSize);
0575 strlcat(buff, ":", moduleBufferSize);
0576 strlcat(buff,
0577 edm::CurrentModuleOnThread::getCurrentModuleOnThread()->moduleDescription()->moduleLabel().c_str(),
0578 moduleBufferSize);
0579 } else {
0580 strlcat(buff, "none", moduleBufferSize);
0581 }
0582 strlcat(buff, " (crashed)", moduleBufferSize);
0583 full_cerr_write(buff);
0584 } else {
0585 full_cerr_write("\nModule: non-CMSSW (crashed)");
0586 }
0587
0588 #ifdef PAUSE_SIGNAL
0589
0590
0591 if (InitRootHandlers::doneModules_.is_lock_free()) {
0592 int spincount = 0;
0593 timespec t = {0, 1000};
0594 while (++spincount < 1000 && InitRootHandlers::doneModules_ < notified) {
0595 nanosleep(&t, nullptr);
0596 }
0597 for (std::size_t i = 0; i < InitRootHandlers::doneModules_; ++i) {
0598 full_cerr_write(InitRootHandlers::moduleListBuffers_[i].data());
0599 }
0600 }
0601 #endif
0602
0603 full_cerr_write("\n\nA fatal system signal has occurred: ");
0604 full_cerr_write(signalname);
0605 full_cerr_write("\n");
0606
0607
0608
0609 if ((sig == SIGILL) || (sig == SIGSEGV) || (sig == SIGBUS) || (sig == SIGTERM) || (sig == SIGFPE) ||
0610 (sig == SIGABRT)) {
0611 signal(sig, SIG_DFL);
0612 raise(sig);
0613 } else {
0614 set_default_signals();
0615 ::abort();
0616 }
0617 }
0618
0619 void sig_abort(int sig, siginfo_t*, void*) {
0620 full_cerr_write("\n\nFatal system signal has occurred during exit\n");
0621
0622
0623 signal(sig, SIG_DFL);
0624 raise(sig);
0625
0626
0627 set_default_signals();
0628 ::sleep(10);
0629 ::abort();
0630 }
0631 }
0632 }
0633
0634 namespace edm {
0635 namespace service {
0636
0637
0638
0639
0640
0641
0642
0643 static void cmssw_stacktrace_fork();
0644
0645 void InitRootHandlers::stacktraceHelperThread() {
0646 int toParent = childToParent_[1];
0647 int fromParent = parentToChild_[0];
0648 char buf[2];
0649 buf[1] = '\0';
0650
0651 while (true) {
0652 int result = full_read(fromParent, buf, 1);
0653 if (result < 0) {
0654
0655
0656
0657 set_default_signals();
0658 close(toParent);
0659 full_cerr_write("\n\nTraceback helper thread failed to read from parent: ");
0660 full_cerr_write(strerror(-result));
0661 full_cerr_write("\n");
0662 ::abort();
0663 }
0664 if (buf[0] == '1') {
0665 set_default_signals();
0666 cmssw_stacktrace_fork();
0667 full_write(toParent, buf);
0668 } else if (buf[0] == '2') {
0669
0670
0671 close(toParent);
0672 close(fromParent);
0673 toParent = childToParent_[1];
0674 fromParent = parentToChild_[0];
0675 } else if (buf[0] == '3') {
0676 break;
0677 } else {
0678 set_default_signals();
0679 close(toParent);
0680 full_cerr_write("\n\nTraceback helper thread got unknown command from parent: ");
0681 full_cerr_write(buf);
0682 full_cerr_write("\n");
0683 ::abort();
0684 }
0685 }
0686 }
0687
0688 void InitRootHandlers::stacktraceFromThread() {
0689 int result = full_write(parentToChild_[1], "1");
0690 if (result < 0) {
0691 full_cerr_write("\n\nAttempt to request stacktrace failed: ");
0692 full_cerr_write(strerror(-result));
0693 full_cerr_write("\n");
0694 return;
0695 }
0696 char buf[2];
0697 buf[1] = '\0';
0698 if ((result = full_read(childToParent_[0], buf, 1, 5 * 60)) < 0) {
0699 full_cerr_write("\n\nWaiting for stacktrace completion failed: ");
0700 if (result == -ETIMEDOUT) {
0701 full_cerr_write("timed out waiting for GDB to complete.");
0702 } else {
0703 full_cerr_write(strerror(-result));
0704 }
0705 full_cerr_write("\n");
0706 return;
0707 }
0708 }
0709
0710 void cmssw_stacktrace_fork() {
0711 char child_stack[4 * 1024];
0712 char* child_stack_ptr = child_stack + 4 * 1024;
0713
0714
0715
0716
0717 int pid =
0718 #ifdef __linux__
0719 clone(edm::service::cmssw_stacktrace, child_stack_ptr, CLONE_VM | CLONE_FS | SIGCHLD, nullptr);
0720 #else
0721 fork();
0722 if (child_stack_ptr) {
0723 }
0724 if (pid == 0) {
0725 edm::service::cmssw_stacktrace(nullptr);
0726 }
0727 #endif
0728 if (pid == -1) {
0729 full_cerr_write("(Attempt to perform stack dump failed.)\n");
0730 } else {
0731 int status;
0732 if (waitpid(pid, &status, 0) == -1) {
0733 full_cerr_write("(Failed to wait on stack dump output.)\n");
0734 }
0735 if (status) {
0736 full_cerr_write("(GDB stack trace failed unexpectedly)\n");
0737 }
0738 }
0739 }
0740
0741 int cmssw_stacktrace(void* ) {
0742 set_default_signals();
0743
0744 char const* const* argv = edm::service::InitRootHandlers::getPstackArgv();
0745
0746
0747
0748 #ifdef __linux__
0749 syscall(SYS_execve, "/bin/sh", argv, __environ);
0750 #else
0751 execv("/bin/sh", argv);
0752 #endif
0753 ::abort();
0754 return 1;
0755 }
0756
0757 static constexpr char pstackName[] = "(CMSSW stack trace helper)";
0758 static constexpr char dashC[] = "-c";
0759 char InitRootHandlers::pidString_[InitRootHandlers::pidStringLength_] = {};
0760 char const* const InitRootHandlers::pstackArgv_[] = {pstackName, dashC, InitRootHandlers::pidString_, nullptr};
0761 int InitRootHandlers::parentToChild_[2] = {-1, -1};
0762 int InitRootHandlers::childToParent_[2] = {-1, -1};
0763 std::unique_ptr<std::thread> InitRootHandlers::helperThread_;
0764 std::unique_ptr<InitRootHandlers::ThreadTracker> InitRootHandlers::threadTracker_;
0765 int InitRootHandlers::stackTracePause_ = 300;
0766 std::vector<std::array<char, moduleBufferSize>> InitRootHandlers::moduleListBuffers_;
0767 std::atomic<std::size_t> InitRootHandlers::nextModule_(0), InitRootHandlers::doneModules_(0);
0768
0769 InitRootHandlers::InitRootHandlers(ParameterSet const& pset, ActivityRegistry& iReg)
0770 : RootHandlers(),
0771 unloadSigHandler_(pset.getUntrackedParameter<bool>("UnloadRootSigHandler")),
0772 resetErrHandler_(pset.getUntrackedParameter<bool>("ResetRootErrHandler")),
0773 loadAllDictionaries_(pset.getUntrackedParameter<bool>("LoadAllDictionaries")),
0774 autoLibraryLoader_(loadAllDictionaries_ or pset.getUntrackedParameter<bool>("AutoLibraryLoader")),
0775 autoClassParser_(pset.getUntrackedParameter<bool>("AutoClassParser")),
0776 interactiveDebug_(pset.getUntrackedParameter<bool>("InteractiveDebug")) {
0777 stackTracePause_ = pset.getUntrackedParameter<int>("StackTracePauseTime");
0778
0779 if (not threadTracker_) {
0780 threadTracker_ = std::make_unique<ThreadTracker>();
0781 iReg.watchPostEndJob([]() {
0782 if (threadTracker_) {
0783 threadTracker_->observe(false);
0784 }
0785 });
0786 }
0787
0788 if (unloadSigHandler_) {
0789
0790 gSystem->ResetSignal(kSigChild);
0791 gSystem->ResetSignal(kSigBus);
0792 gSystem->ResetSignal(kSigSegmentationViolation);
0793 gSystem->ResetSignal(kSigIllegalInstruction);
0794 gSystem->ResetSignal(kSigSystem);
0795 gSystem->ResetSignal(kSigPipe);
0796 gSystem->ResetSignal(kSigAlarm);
0797 gSystem->ResetSignal(kSigUrgent);
0798 gSystem->ResetSignal(kSigFloatingException);
0799 gSystem->ResetSignal(kSigWindowChanged);
0800 } else if (pset.getUntrackedParameter<bool>("AbortOnSignal")) {
0801 cachePidInfo();
0802
0803
0804
0805 gSystem->ResetSignal(kSigBus);
0806 gSystem->ResetSignal(kSigSegmentationViolation);
0807 gSystem->ResetSignal(kSigIllegalInstruction);
0808 gSystem->ResetSignal(kSigFloatingException);
0809 installCustomHandler(SIGBUS, sig_dostack_then_abort);
0810 sigBusHandler_ = std::shared_ptr<const void>(nullptr, [](void*) { installCustomHandler(SIGBUS, sig_abort); });
0811 installCustomHandler(SIGSEGV, sig_dostack_then_abort);
0812 sigSegvHandler_ = std::shared_ptr<const void>(nullptr, [](void*) { installCustomHandler(SIGSEGV, sig_abort); });
0813 installCustomHandler(SIGILL, sig_dostack_then_abort);
0814 sigIllHandler_ = std::shared_ptr<const void>(nullptr, [](void*) { installCustomHandler(SIGILL, sig_abort); });
0815 installCustomHandler(SIGTERM, sig_dostack_then_abort);
0816 sigTermHandler_ = std::shared_ptr<const void>(nullptr, [](void*) { installCustomHandler(SIGTERM, sig_abort); });
0817 installCustomHandler(SIGFPE, sig_dostack_then_abort);
0818 sigFpeHandler_ = std::shared_ptr<const void>(nullptr, [](void*) { installCustomHandler(SIGFPE, sig_abort); });
0819 installCustomHandler(SIGABRT, sig_dostack_then_abort);
0820 sigAbrtHandler_ = std::shared_ptr<const void>(nullptr, [](void*) {
0821 signal(SIGABRT, SIG_DFL);
0822 });
0823 }
0824
0825 iReg.watchPreallocate([](edm::service::SystemBounds const& iBounds) {
0826 if (iBounds.maxNumberOfThreads() > moduleListBuffers_.size()) {
0827 moduleListBuffers_.resize(iBounds.maxNumberOfThreads());
0828 }
0829 });
0830
0831 if (resetErrHandler_) {
0832
0833 SetErrorHandler(RootErrorHandler);
0834 }
0835
0836
0837 if (autoLibraryLoader_) {
0838 gInterpreter->SetClassAutoloading(1);
0839 }
0840
0841
0842 if (not autoClassParser_) {
0843
0844 iReg.watchPreModuleConstruction(
0845 [](edm::ModuleDescription const&) { gInterpreter->SetClassAutoparsing(false); });
0846 iReg.watchPostModuleConstruction(
0847 [](edm::ModuleDescription const&) { gInterpreter->SetClassAutoparsing(true); });
0848 }
0849
0850
0851 TTree::SetMaxTreeSize(kMaxLong64);
0852 TH1::AddDirectory(kFALSE);
0853
0854
0855
0856 setRefCoreStreamerInTClass();
0857
0858
0859 if (!hasDictionary(typeid(std::vector<std::vector<unsigned int>>))) {
0860 TypeWithDict::byName("std::vector<std::vector<unsigned int> >");
0861 }
0862
0863 int debugLevel = pset.getUntrackedParameter<int>("DebugLevel");
0864 if (debugLevel > 0) {
0865 gDebug = debugLevel;
0866 }
0867
0868
0869 bool imt = pset.getUntrackedParameter<bool>("EnableIMT");
0870 if (imt && not ROOT::IsImplicitMTEnabled()) {
0871
0872
0873 ROOT::EnableImplicitMT(
0874 oneapi::tbb::global_control::active_value(oneapi::tbb::global_control::max_allowed_parallelism));
0875 }
0876 }
0877
0878 InitRootHandlers::~InitRootHandlers() {
0879
0880 TIter iter(gROOT->GetListOfFiles());
0881 TObject* obj = nullptr;
0882 while (nullptr != (obj = iter.Next())) {
0883 TFile* f = dynamic_cast<TFile*>(obj);
0884 if (f) {
0885
0886
0887 f->Close();
0888 iter = TIter(gROOT->GetListOfFiles());
0889 }
0890 }
0891
0892 threadTracker_.reset();
0893 }
0894
0895 void InitRootHandlers::willBeUsingThreads() {
0896
0897 ROOT::EnableThreadSafety();
0898
0899
0900 TObject::SetObjectStat(false);
0901
0902
0903 TVirtualStreamerInfo::Optimize(false);
0904 }
0905
0906 void InitRootHandlers::fillDescriptions(ConfigurationDescriptions& descriptions) {
0907 ParameterSetDescription desc;
0908 desc.setComment("Centralized interface to ROOT.");
0909 desc.addUntracked<bool>("UnloadRootSigHandler", false)
0910 ->setComment("If True, signals are handled by this service, rather than by ROOT.");
0911 desc.addUntracked<bool>("ResetRootErrHandler", true)
0912 ->setComment(
0913 "If True, ROOT messages (e.g. errors, warnings) are handled by this service, rather than by ROOT.");
0914 desc.addUntracked<bool>("AutoLibraryLoader", true)
0915 ->setComment("If True, enables automatic loading of data dictionaries.");
0916 desc.addUntracked<bool>("AutoClassParser", true)
0917 ->setComment(
0918 "If False, the automatic parsing of class headers for dictionaries when pre-built dictionaries are "
0919 "missing is disable during module construction. The current implementation of disabling the parsing is "
0920 "fragile, and may work only in a single-thread job that does not use reco::parser::cutParser() or "
0921 "reco::parser::expressionParser() (and it certainly does not work on multiple threads).");
0922 desc.addUntracked<bool>("LoadAllDictionaries", false)->setComment("If True, loads all ROOT dictionaries.");
0923 desc.addUntracked<bool>("EnableIMT", true)->setComment("If True, calls ROOT::EnableImplicitMT().");
0924 desc.addUntracked<bool>("AbortOnSignal", true)
0925 ->setComment(
0926 "If True, do an abort when a signal occurs that causes a crash. If False, ROOT will do an exit which "
0927 "attempts to do a clean shutdown.");
0928 desc.addUntracked<bool>("InteractiveDebug", false)
0929 ->setComment(
0930 "If True, leave gdb attached to cmsRun after a crash; "
0931 "if False, attach gdb, print a stack trace, and quit gdb");
0932 desc.addUntracked<int>("DebugLevel", 0)->setComment("Sets ROOT's gDebug value.");
0933 desc.addUntracked<int>("StackTracePauseTime", 300)
0934 ->setComment("Seconds to pause other threads during stack trace.");
0935 descriptions.add("InitRootHandlers", desc);
0936 }
0937
0938 char const* const* InitRootHandlers::getPstackArgv() { return pstackArgv_; }
0939
0940 void InitRootHandlers::enableWarnings_() { s_ignoreWarnings = edm::RootHandlers::SeverityLevel::kInfo; }
0941
0942 void InitRootHandlers::ignoreWarnings_(edm::RootHandlers::SeverityLevel level) { s_ignoreWarnings = level; }
0943
0944 void InitRootHandlers::cachePidInfo() {
0945 if (helperThread_) {
0946
0947
0948
0949 return;
0950 }
0951 std::string gdbcmd{"date; gdb -quiet -p %d"};
0952 if (!interactiveDebug_) {
0953 gdbcmd +=
0954 " 2>&1 <<EOF |\n"
0955 "set width 0\n"
0956 "set height 0\n"
0957 "set pagination no\n"
0958 "thread apply all bt\n"
0959 "EOF\n"
0960 "/bin/sed -n -e 's/^\\((gdb) \\)*//' -e '/^#/p' -e '/^Thread/p'";
0961 }
0962 if (snprintf(pidString_, pidStringLength_ - 1, gdbcmd.c_str(), getpid()) >= pidStringLength_) {
0963 std::ostringstream sstr;
0964 sstr << "Unable to pre-allocate stacktrace handler information";
0965 edm::Exception except(edm::errors::OtherCMS, sstr.str());
0966 throw except;
0967 }
0968
0969
0970
0971
0972 close(childToParent_[0]);
0973 close(childToParent_[1]);
0974 childToParent_[0] = -1;
0975 childToParent_[1] = -1;
0976 close(parentToChild_[0]);
0977 close(parentToChild_[1]);
0978 parentToChild_[0] = -1;
0979 parentToChild_[1] = -1;
0980
0981 if (-1 == pipe2(childToParent_, O_CLOEXEC)) {
0982 std::ostringstream sstr;
0983 sstr << "Failed to create child-to-parent pipes (errno=" << errno << "): " << strerror(errno);
0984 edm::Exception except(edm::errors::OtherCMS, sstr.str());
0985 throw except;
0986 }
0987
0988 if (-1 == pipe2(parentToChild_, O_CLOEXEC)) {
0989 close(childToParent_[0]);
0990 close(childToParent_[1]);
0991 childToParent_[0] = -1;
0992 childToParent_[1] = -1;
0993 std::ostringstream sstr;
0994 sstr << "Failed to create child-to-parent pipes (errno=" << errno << "): " << strerror(errno);
0995 edm::Exception except(edm::errors::OtherCMS, sstr.str());
0996 throw except;
0997 }
0998
0999 helperThread_ = std::make_unique<std::thread>(stacktraceHelperThread);
1000 helperThread_->detach();
1001 }
1002
1003 }
1004 }
1005
1006 #include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
1007
1008 using edm::service::InitRootHandlers;
1009 typedef edm::serviceregistry::AllArgsMaker<edm::RootHandlers, InitRootHandlers> RootHandlersMaker;
1010 DEFINE_FWK_SERVICE_MAKER(InitRootHandlers, RootHandlersMaker);