MessageService/src/ThreadSafeLogMessageLoggerScribe.cc

0001 // ----------------------------------------------------------------------
0002 //
0003 // ThreadSafeLogMessageLoggerScribe.cc
0004 //
0005 // NOTE: This was originally copied from MessageLoggerScribe but removed
0006 //  the potential use of the ThreadQueue. MessageLoggerScribe was not
0007 //  modified since it was decided we might have to keep the old behaviour
0008 //  around for 'legacy' reasons.
0009 //
0010 // ----------------------------------------------------------------------
0011
0012 #include "FWCore/MessageService/src/ThreadSafeLogMessageLoggerScribe.h"
0013 #include "FWCore/MessageService/src/ELadministrator.h"
0014 #include "FWCore/MessageService/src/ELoutput.h"
0015 #include "FWCore/MessageService/src/ELstatistics.h"
0016
0017 #include "FWCore/MessageLogger/interface/ErrorObj.h"
0018 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0019 #include "FWCore/MessageService/src/ConfigurationHandshake.h"
0020 #include "FWCore/MessageLogger/interface/MessageDrop.h"      // change log 37
0021 #include "FWCore/MessageLogger/interface/ELseverityLevel.h"  // change log 37
0022
0023 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0024 #include "FWCore/ParameterSet/interface/ParameterWildcardWithSpecifics.h"
0025
0026 #include "FWCore/Utilities/interface/EDMException.h"
0027 #include "FWCore/Utilities/interface/Algorithms.h"
0028
0029 #include <algorithm>
0030 #include <cassert>
0031 #include <fstream>
0032 #include <string>
0033 #include <csignal>
0034
0035 using std::cerr;
0036 using namespace edm::messagelogger;
0037
0038 namespace edm {
0039   namespace service {
0040
0041     ThreadSafeLogMessageLoggerScribe::ThreadSafeLogMessageLoggerScribe()
0042         : m_admin_p(std::make_shared<ELadministrator>()),
0043           m_early_dest(m_admin_p->attach(std::make_shared<ELoutput>(std::cerr, false))),
0044           m_clean_slate_configuration(true),
0045           m_active(true),
0046           m_purge_mode(false),
0047           m_count(0),
0048           m_messageBeingSent(false),
0049           m_waitingThreshold(100),
0050           m_tooManyWaitingMessagesCount(0) {}
0051
0052     ThreadSafeLogMessageLoggerScribe::~ThreadSafeLogMessageLoggerScribe() {
0053       //if there are any waiting message, finish them off
0054       ErrorObj* errorobj_p = nullptr;
0055       std::vector<std::string> categories;
0056       while (m_waitingMessages.try_pop(errorobj_p)) {
0057         if (not m_purge_mode) {
0058           categories.clear();
0059           parseCategories(errorobj_p->xid().id, categories);
0060           for (unsigned int icat = 0; icat < categories.size(); ++icat) {
0061             errorobj_p->setID(categories[icat]);
0062             m_admin_p->log(*errorobj_p);  // route the message text
0063           }
0064         }
0065         delete errorobj_p;
0066       }
0067
0068       m_admin_p->finish();
0069     }
0070
0071     void ThreadSafeLogMessageLoggerScribe::runCommand(  // changeLog 32
0072         MessageLoggerQ::OpCode opcode,
0073         void* operand) {
0074       switch (opcode) {  // interpret the work item
0075         default: {
0076           assert(false);  // can't happen (we certainly hope!)
0077           break;
0078         }
0079         case MessageLoggerQ::END_THREAD: {
0080           break;
0081         }
0082         case MessageLoggerQ::LOG_A_MESSAGE: {
0083           ErrorObj* errorobj_p = static_cast<ErrorObj*>(operand);
0084           try {
0085             if (m_active && !m_purge_mode) {
0086               log(errorobj_p);
0087             }
0088           } catch (cms::Exception& e) {
0089             ++m_count;
0090             std::cerr << "ThreadSafeLogMessageLoggerScribe caught " << m_count << " cms::Exceptions, text = \n"
0091                       << e.what() << "\n";
0092
0093             if (m_count > 25) {
0094               cerr << "MessageLogger will no longer be processing "
0095                    << "messages due to errors (entering purge mode).\n";
0096               m_purge_mode = true;
0097             }
0098           } catch (...) {
0099             std::cerr << "ThreadSafeLogMessageLoggerScribe caught an unknown exception and "
0100                       << "will no longer be processing "
0101                       << "messages. (entering purge mode)\n";
0102             m_purge_mode = true;
0103           }
0104           break;
0105         }
0106         case MessageLoggerQ::CONFIGURE: {  // changelog 17
0107           auto job_pset_p = std::unique_ptr<edm::ParameterSet>(
0108               static_cast<edm::ParameterSet*>(operand));  // propagate_const<T> has no reset() function
0109           validate(*job_pset_p);
0110           configure_errorlog(*job_pset_p);
0111           break;
0112         }
0113         case MessageLoggerQ::SUMMARIZE: {
0114           assert(operand == nullptr);
0115           try {
0116             triggerStatisticsSummaries();
0117           } catch (cms::Exception& e) {
0118             std::cerr << "ThreadSafeLogMessageLoggerScribe caught exception "
0119                       << "during summarize:\n"
0120                       << e.what() << "\n";
0121           } catch (...) {
0122             std::cerr << "ThreadSafeLogMessageLoggerScribe caught unkonwn exception type "
0123                       << "during summarize. (Ignored)\n";
0124           }
0125           break;
0126         }
0127         case MessageLoggerQ::JOBMODE: {  // change log 24
0128           std::string* jobMode_p = static_cast<std::string*>(operand);
0129           JobMode jm = MessageLoggerDefaults::mode(*jobMode_p);
0130           m_messageLoggerDefaults = value_ptr<MessageLoggerDefaults>(new MessageLoggerDefaults(jm));
0131           // Note - since m_messageLoggerDefaults is a value_ptr,
0132           //        there is no concern about deleting here.
0133           delete jobMode_p;  // dispose of the message text
0134                              // which will have been new-ed
0135                              // in MessageLogger.cc (service version)
0136           break;
0137         }
0138         case MessageLoggerQ::SHUT_UP: {
0139           assert(operand == nullptr);
0140           m_active = false;
0141           break;
0142         }
0143         case MessageLoggerQ::FLUSH_LOG_Q: {  // changelog 26
0144           break;
0145         }
0146         case MessageLoggerQ::GROUP_STATS: {  // change log 27
0147           std::string* cat_p = static_cast<std::string*>(operand);
0148           ELstatistics::noteGroupedCategory(*cat_p);
0149           delete cat_p;  // dispose of the message text
0150           break;
0151         }
0152         case MessageLoggerQ::FJR_SUMMARY: {  // changelog 29
0153           std::map<std::string, double>* smp = static_cast<std::map<std::string, double>*>(operand);
0154           triggerFJRmessageSummary(*smp);
0155           break;
0156         }
0157       }  // switch
0158
0159     }  // ThreadSafeLogMessageLoggerScribe::runCommand(opcode, operand)
0160
0161     void ThreadSafeLogMessageLoggerScribe::log(ErrorObj* errorobj_p) {
0162       bool expected = false;
0163       std::unique_ptr<ErrorObj> obj(errorobj_p);
0164       if (m_messageBeingSent.compare_exchange_strong(expected, true)) {
0165         std::vector<std::string> categories;
0166         parseCategories(errorobj_p->xid().id, categories);
0167         for (unsigned int icat = 0; icat < categories.size(); ++icat) {
0168           errorobj_p->setID(categories[icat]);
0169           m_admin_p->log(*errorobj_p);  // route the message text
0170         }
0171         //process any waiting messages
0172         errorobj_p = nullptr;
0173         while (not m_purge_mode and m_waitingMessages.try_pop(errorobj_p)) {
0174           obj.reset(errorobj_p);
0175           categories.clear();
0176           parseCategories(errorobj_p->xid().id, categories);
0177           for (unsigned int icat = 0; icat < categories.size(); ++icat) {
0178             errorobj_p->setID(categories[icat]);
0179             m_admin_p->log(*errorobj_p);  // route the message text
0180           }
0181         }
0182         m_messageBeingSent.store(false);
0183       } else {
0184         if (m_waitingMessages.unsafe_size() < m_waitingThreshold) {
0185           obj.release();
0186           m_waitingMessages.push(errorobj_p);
0187         } else {
0188           ++m_tooManyWaitingMessagesCount;
0189         }
0190       }
0191     }
0192
0193     namespace {
0194       bool usingOldConfig(edm::ParameterSet const& pset) {
0195         if (not pset.exists("files") and
0196             ((pset.exists("destinations") or pset.existsAs<std::vector<std::string>>("statistics", true) or
0197               pset.existsAs<std::vector<std::string>>("statistics", false) or pset.exists("categories")))) {
0198           return true;
0199         }
0200         return false;
0201       }
0202
0203       std::set<std::string> findCategoriesInDestination(edm::ParameterSet const& pset) {
0204         auto psets = pset.getParameterNamesForType<edm::ParameterSet>(false);
0205         auto itFound = std::find(psets.begin(), psets.end(), "default");
0206         if (itFound != psets.end()) {
0207           psets.erase(itFound);
0208         }
0209
0210         return std::set<std::string>(psets.begin(), psets.end());
0211       }
0212       std::vector<std::string> findAllCategories(edm::ParameterSet const& pset) {
0213         std::set<std::string> categories;
0214
0215         auto psets = pset.getParameterNamesForType<edm::ParameterSet>(false);
0216         auto itFound = std::find(psets.begin(), psets.end(), "default");
0217         if (itFound != psets.end()) {
0218           categories = findCategoriesInDestination(pset.getUntrackedParameter<edm::ParameterSet>("default"));
0219           psets.erase(itFound);
0220         }
0221
0222         itFound = std::find(psets.begin(), psets.end(), "cout");
0223         if (itFound != psets.end()) {
0224           categories.merge(findCategoriesInDestination(pset.getUntrackedParameter<edm::ParameterSet>("cout")));
0225           psets.erase(itFound);
0226         }
0227
0228         itFound = std::find(psets.begin(), psets.end(), "cerr");
0229         if (itFound != psets.end()) {
0230           categories.merge(findCategoriesInDestination(pset.getUntrackedParameter<edm::ParameterSet>("cerr")));
0231           psets.erase(itFound);
0232         }
0233
0234         auto const& files = pset.getUntrackedParameter<edm::ParameterSet>("files");
0235         for (auto const& name : files.getParameterNamesForType<edm::ParameterSet>(false)) {
0236           categories.merge(findCategoriesInDestination(files.getUntrackedParameter<edm::ParameterSet>(name)));
0237         }
0238         categories.insert(psets.begin(), psets.end());
0239
0240         return std::vector<std::string>(categories.begin(), categories.end());
0241       }
0242
0243     }  // namespace
0244
0245     std::string ThreadSafeLogMessageLoggerScribe::destinationFileName(edm::ParameterSet const& dest_pset,
0246                                                                       std::string const& psetname) const {
0247       // Determine the destination file name to use if no explicit filename is
0248       // supplied in the cfg.
0249       std::string const empty_String;
0250       std::string filename = psetname;
0251       std::string filename_default = getAparameter<std::string>(dest_pset, "output", empty_String);
0252       if (filename_default == empty_String) {
0253         filename_default = m_messageLoggerDefaults->output(psetname);  // change log 31
0254         if (filename_default == empty_String) {
0255           filename_default = filename;
0256         }
0257       }
0258
0259       std::string explicit_filename = getAparameter<std::string>(dest_pset, "filename", filename_default);
0260       if (explicit_filename != empty_String)
0261         filename = explicit_filename;
0262       std::string explicit_extension = getAparameter<std::string>(dest_pset, "extension", empty_String);
0263       if (explicit_extension != empty_String) {
0264         if (explicit_extension[0] == '.') {
0265           filename += explicit_extension;
0266         } else {
0267           filename = filename + "." + explicit_extension;
0268         }
0269       }
0270
0271       // Attach a default extension of .log if there is no extension on a file
0272       if ((filename != "cout") && (filename != "cerr")) {
0273         if (filename.find('.') == std::string::npos) {
0274           filename += ".log";
0275         }
0276       }
0277       return filename;
0278     }
0279
0280     void ThreadSafeLogMessageLoggerScribe::configure_errorlog_new(edm::ParameterSet& job_pset) {
0281       {
0282         auto preconfiguration_message =
0283             job_pset.getUntrackedParameter<std::string>("generate_preconfiguration_message");
0284         if (not preconfiguration_message.empty()) {
0285           // To test a preconfiguration message without first going thru the
0286           // configuration we are about to do, we issue the message (so it sits
0287           // on the queue), then copy the processing that the LOG_A_MESSAGE case
0288           // does.  We suppress the timestamp to allow for automated unit testing.
0289           m_early_dest->suppressTime();
0290           LogError("preconfiguration") << preconfiguration_message;
0291         }
0292       }
0293       if (!m_stream_ps.empty()) {
0294         LogWarning("multiLogConfig") << "The message logger has been configured multiple times";
0295         m_clean_slate_configuration = false;  // Change Log 22
0296       }
0297       m_waitingThreshold = job_pset.getUntrackedParameter<unsigned int>("waiting_threshold");
0298
0299       auto defaults = parseDefaults(job_pset);
0300       auto categories = findAllCategories(job_pset);
0301
0302       // Initialize unversal suppression variables
0303       MessageDrop::debugAlwaysSuppressed = true;
0304       MessageDrop::infoAlwaysSuppressed = true;
0305       MessageDrop::fwkInfoAlwaysSuppressed = true;
0306       MessageDrop::warningAlwaysSuppressed = true;
0307
0308       m_early_dest->setThreshold(ELhighestSeverity);
0309
0310       auto cout_dest = job_pset.getUntrackedParameter<edm::ParameterSet>("cout");
0311       if (cout_dest.getUntrackedParameter<bool>("enable")) {
0312         auto dest_ctrl = makeDestinationCtrl("cout");
0313         configure_dest(job_pset, defaults, categories, dest_ctrl, cout_dest, "cout");
0314       }
0315
0316       auto cerr_dest = job_pset.getUntrackedParameter<edm::ParameterSet>("cerr");
0317       if (cerr_dest.getUntrackedParameter<bool>("enable")) {
0318         auto dest_ctrl = makeDestinationCtrl("cerr");
0319         configure_dest(job_pset, defaults, categories, dest_ctrl, cerr_dest, "cerr");
0320       }
0321
0322       auto const& files = job_pset.getUntrackedParameter<edm::ParameterSet>("files");
0323       for (auto const& name : files.getParameterNamesForType<edm::ParameterSet>(false)) {
0324         auto const& dest_pset = files.getUntrackedParameter<edm::ParameterSet>(name);
0325         auto const actual_filename = destinationFileName(dest_pset, name);
0326
0327         // Check that this is not a duplicate name
0328         if (m_stream_ps.find(actual_filename) != m_stream_ps.end()) {
0329           if (m_clean_slate_configuration) {
0330             throw cms::Exception("DuplicateDestination")
0331                 << "Duplicate name for a MessageLogger Destination: " << actual_filename << "\n"
0332                 << "Please modify the configuration to use unique file names.";
0333           } else {
0334             LogWarning("duplicateDestination")
0335                 << "Duplicate name for a MessageLogger Destination: " << actual_filename << "\n"
0336                 << "Only original configuration instructions are used";
0337             continue;
0338           }
0339         }
0340
0341         auto dest_ctrl = makeDestinationCtrl(actual_filename);
0342         configure_dest(job_pset, defaults, categories, dest_ctrl, dest_pset, name);
0343       }
0344       //NOTE: statistics destinations MUST BE last in the list else they can be fooled into
0345       // thinking a message has been ignored just because a later destination which uses it
0346       // falls later in the list.
0347       for (auto const& name : files.getParameterNamesForType<edm::ParameterSet>(false)) {
0348         auto const& dest_pset = files.getUntrackedParameter<edm::ParameterSet>(name);
0349         auto const actual_filename = destinationFileName(dest_pset, name);
0350         if (getAparameter<bool>(dest_pset, "enableStatistics", false)) {
0351           configure_statistics_dest(job_pset, defaults, categories, dest_pset, name, actual_filename);
0352         }
0353       }
0354       if (cout_dest.getUntrackedParameter<bool>("enable") and
0355           getAparameter<bool>(cout_dest, "enableStatistics", true)) {
0356         configure_statistics_dest(job_pset, defaults, categories, cout_dest, "cout", "cout");
0357       }
0358       if (cerr_dest.getUntrackedParameter<bool>("enable") and
0359           getAparameter<bool>(cerr_dest, "enableStatistics", true)) {
0360         configure_statistics_dest(job_pset, defaults, categories, cerr_dest, "cerr", "cerr");
0361       }
0362     }
0363
0364     void ThreadSafeLogMessageLoggerScribe::configure_errorlog(edm::ParameterSet& job_pset) {
0365       if (not usingOldConfig(job_pset)) {
0366         configure_errorlog_new(job_pset);
0367         return;
0368       }
0369       const vString empty_vString;
0370       const std::string empty_String;
0371       const edm::ParameterSet empty_PSet;
0372
0373       // The following is present to test pre-configuration message handling:
0374       std::string preconfiguration_message =
0375           getAparameter<std::string>(job_pset, "generate_preconfiguration_message", empty_String);
0376       if (preconfiguration_message != empty_String) {
0377         // To test a preconfiguration message without first going thru the
0378         // configuration we are about to do, we issue the message (so it sits
0379         // on the queue), then copy the processing that the LOG_A_MESSAGE case
0380         // does.  We suppress the timestamp to allow for automated unit testing.
0381         m_early_dest->suppressTime();
0382         LogError("preconfiguration") << preconfiguration_message;
0383       }
0384
0385       if (!m_stream_ps.empty()) {
0386         LogWarning("multiLogConfig") << "The message logger has been configured multiple times";
0387         m_clean_slate_configuration = false;  // Change Log 22
0388       }
0389       m_waitingThreshold = getAparameter<unsigned int>(job_pset, "waiting_threshold", 100);
0390       auto defaults = parseDefaults(job_pset);
0391       // grab list of categories
0392       vString categories = getAparameter<vString>(job_pset, "categories", empty_vString);
0393       // grab list of hardwired categories (hardcats) -- these are to be added
0394       // to the list of categories -- change log 24
0395       {
0396         std::vector<std::string> hardcats = m_messageLoggerDefaults->categories;
0397         // combine the lists, not caring about possible duplicates (for now)
0398         copy_all(hardcats, std::back_inserter(categories));
0399       }  // no longer need hardcats
0400
0401       auto destination_names = configure_ordinary_destinations(job_pset, defaults, categories);
0402       configure_statistics(job_pset, defaults, categories, destination_names);
0403     }  // ThreadSafeLogMessageLoggerScribe::configure_errorlog()
0404
0405     std::shared_ptr<ELdestination> ThreadSafeLogMessageLoggerScribe::makeDestinationCtrl(std::string const& filename) {
0406       std::shared_ptr<ELdestination> dest_ctrl;
0407       if (filename == "cout") {
0408         dest_ctrl = m_admin_p->attach(std::make_shared<ELoutput>(std::cout));
0409         m_stream_ps["cout"] = &std::cout;
0410       } else if (filename == "cerr") {
0411         m_early_dest->setThreshold(ELzeroSeverity);
0412         dest_ctrl = m_early_dest;
0413         m_stream_ps["cerr"] = &std::cerr;
0414       } else {
0415         auto os_sp = std::make_shared<std::ofstream>(filename.c_str());
0416         m_file_ps.push_back(os_sp);
0417         dest_ctrl = m_admin_p->attach(std::make_shared<ELoutput>(*os_sp));
0418         m_stream_ps[filename] = os_sp.get();
0419       }
0420       return dest_ctrl;
0421     }
0422
0423     namespace {
0424       void setGlobalThresholds(ELseverityLevel threshold_sev) {
0425         if (threshold_sev <= ELseverityLevel::ELsev_success) {
0426           edm::MessageDrop::debugAlwaysSuppressed = false;
0427         }
0428         if (threshold_sev <= ELseverityLevel::ELsev_info) {
0429           edm::MessageDrop::infoAlwaysSuppressed = false;
0430         }
0431         if (threshold_sev <= ELseverityLevel::ELsev_fwkInfo) {
0432           edm::MessageDrop::fwkInfoAlwaysSuppressed = false;
0433         }
0434         if (threshold_sev <= ELseverityLevel::ELsev_warning) {
0435           edm::MessageDrop::warningAlwaysSuppressed = false;
0436         }
0437       }
0438     }  // namespace
0439
0440     ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults ThreadSafeLogMessageLoggerScribe::parseDefaults(
0441         edm::ParameterSet const& job_pset) {
0442       const edm::ParameterSet empty_PSet;
0443       ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults returnValue;
0444       // grab default limit/interval/timespan common to all destinations/categories:
0445       edm::ParameterSet default_pset = getAparameter<edm::ParameterSet>(job_pset, "default", empty_PSet);
0446       returnValue.limit_ = getAparameter<int>(
0447           default_pset, "limit", ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults::COMMON_DEFAULT_LIMIT);
0448       returnValue.reportEvery_ = getAparameter<int>(
0449           default_pset, "reportEvery", ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults::COMMON_DEFAULT_INTERVAL);
0450       returnValue.timespan_ = getAparameter<int>(
0451           default_pset, "timespan", ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults::COMMON_DEFAULT_TIMESPAN);
0452       std::string default_threshold = getAparameter<std::string>(job_pset, "threshold", std::string());
0453       returnValue.threshold_ = getAparameter<std::string>(default_pset, "threshold", default_threshold);
0454       returnValue.noLineBreaks_ = getAparameter<bool>(default_pset, "noLineBreaks", false);
0455       returnValue.lineLength_ = getAparameter<int>(default_pset, "lineLength", 80);
0456       returnValue.noTimeStamps_ = getAparameter<bool>(default_pset, "noTimeStamps", false);
0457
0458       return returnValue;
0459     }
0460
0461     void ThreadSafeLogMessageLoggerScribe::configure_dest(edm::ParameterSet const& job_pset,
0462                                                           ConfigurableDefaults const& defaults,
0463                                                           vString const& categories,
0464                                                           std::shared_ptr<ELdestination> dest_ctrl,
0465                                                           edm::ParameterSet const& dest_pset,
0466                                                           std::string const& filename) {
0467       vString const empty_vString;
0468       edm::ParameterSet const empty_PSet;
0469       std::string const empty_String;
0470
0471       // Defaults:                          // change log 3a
0472       const std::string COMMON_DEFAULT_THRESHOLD = "INFO";
0473
0474       vString const severities = {{"WARNING", "INFO", "FWKINFO", "ERROR", "DEBUG"}};
0475
0476       // grab default threshold common to all destinations
0477       std::string const default_threshold = getAparameter<std::string>(job_pset, "threshold", empty_String);
0478       // change log 3a
0479       // change log 24
0480
0481       // grab default limit/interval/timespan common to all destinations/categories:
0482       edm::ParameterSet const default_pset = getAparameter<edm::ParameterSet>(job_pset, "default", empty_PSet);
0483
0484       // See if this is just a placeholder          // change log 9
0485       bool is_placeholder = getAparameter<bool>(dest_pset, "placeholder", false);
0486       if (is_placeholder)
0487         return;
0488
0489       // grab this destination's default limit/interval/timespan:
0490       edm::ParameterSet dest_default_pset = getAparameter<edm::ParameterSet>(dest_pset, "default", empty_PSet);
0491       int dest_default_limit = getAparameter<int>(dest_default_pset, "limit", defaults.limit_);
0492       int dest_default_interval = getAparameter<int>(dest_default_pset, "reportEvery", defaults.reportEvery_);
0493       // change log 6
0494       int dest_default_timespan = getAparameter<int>(dest_default_pset, "timespan", defaults.timespan_);
0495       // change log 1a
0496       if (dest_default_limit != defaults.NO_VALUE_SET) {
0497         if (dest_default_limit < 0)
0498           dest_default_limit = 2000000000;
0499         dest_ctrl->setLimit("*", dest_default_limit);
0500       }  // change log 1b, 2a, 2b
0501       if (dest_default_interval != defaults.NO_VALUE_SET) {  // change log 6
0502         dest_ctrl->setInterval("*", dest_default_interval);
0503       }
0504       if (dest_default_timespan != defaults.NO_VALUE_SET) {
0505         if (dest_default_timespan < 0)
0506           dest_default_timespan = 2000000000;
0507         dest_ctrl->setTimespan("*", dest_default_timespan);
0508       }  // change log 1b, 2a, 2b
0509
0510       // establish this destination's threshold:
0511       std::string dest_threshold = getAparameter<std::string>(dest_pset, "threshold", default_threshold);
0512       if (dest_threshold == empty_String) {
0513         dest_threshold = default_threshold;
0514       }
0515       if (dest_threshold == empty_String) {  // change log 34
0516         dest_threshold = defaults.threshold_;
0517       }
0518       if (dest_threshold == empty_String) {
0519         dest_threshold = m_messageLoggerDefaults->threshold(filename);
0520       }
0521       if (dest_threshold == empty_String)
0522         dest_threshold = COMMON_DEFAULT_THRESHOLD;
0523       ELseverityLevel threshold_sev(dest_threshold);
0524       dest_ctrl->setThreshold(threshold_sev);
0525       // change log 37
0526       setGlobalThresholds(threshold_sev);
0527
0528       // establish this destination's limit/interval/timespan for each category:
0529       for (vString::const_iterator id_it = categories.begin(); id_it != categories.end(); ++id_it) {
0530         const std::string& msgID = *id_it;
0531         edm::ParameterSet default_category_pset =
0532             getAparameter<edm::ParameterSet>(default_pset, msgID, empty_PSet);  // change log 5
0533         edm::ParameterSet category_pset = getAparameter<edm::ParameterSet>(dest_pset, msgID, default_category_pset);
0534
0535         int category_default_limit = getAparameter<int>(default_category_pset, "limit", defaults.NO_VALUE_SET);
0536         int limit = getAparameter<int>(category_pset, "limit", category_default_limit);
0537         if (limit == defaults.NO_VALUE_SET)
0538           limit = dest_default_limit;
0539         // change log 7
0540         int category_default_interval = getAparameter<int>(default_category_pset, "reportEvery", defaults.NO_VALUE_SET);
0541         int interval = getAparameter<int>(category_pset, "reportEvery", category_default_interval);
0542         if (interval == defaults.NO_VALUE_SET)
0543           interval = dest_default_interval;
0544         // change log 6  and then 7
0545         int category_default_timespan = getAparameter<int>(default_category_pset, "timespan", defaults.NO_VALUE_SET);
0546         int timespan = getAparameter<int>(category_pset, "timespan", category_default_timespan);
0547         if (timespan == defaults.NO_VALUE_SET)
0548           timespan = dest_default_timespan;
0549         // change log 7
0550
0551         const std::string& category = msgID;
0552         if (limit == defaults.NO_VALUE_SET) {  // change log 24
0553           limit = m_messageLoggerDefaults->limit(filename, category);
0554         }
0555         if (interval == defaults.NO_VALUE_SET) {  // change log 24
0556           interval = m_messageLoggerDefaults->reportEvery(filename, category);
0557         }
0558         if (timespan == defaults.NO_VALUE_SET) {  // change log 24
0559           timespan = m_messageLoggerDefaults->timespan(filename, category);
0560         }
0561
0562         if (limit != defaults.NO_VALUE_SET) {
0563           if (limit < 0)
0564             limit = 2000000000;
0565           dest_ctrl->setLimit(msgID, limit);
0566         }  // change log 2a, 2b
0567         if (interval != defaults.NO_VALUE_SET) {
0568           dest_ctrl->setInterval(msgID, interval);
0569         }  // change log 6
0570         if (timespan != defaults.NO_VALUE_SET) {
0571           if (timespan < 0)
0572             timespan = 2000000000;
0573           dest_ctrl->setTimespan(msgID, timespan);
0574         }  // change log 2a, 2b
0575
0576       }  // for
0577
0578       // establish this destination's limit for each severity:
0579       for (vString::const_iterator sev_it = severities.begin(); sev_it != severities.end(); ++sev_it) {
0580         const std::string& sevID = *sev_it;
0581         ELseverityLevel severity(sevID);
0582         edm::ParameterSet default_sev_pset = getAparameter<edm::ParameterSet>(default_pset, sevID, empty_PSet);
0583         edm::ParameterSet sev_pset = getAparameter<edm::ParameterSet>(dest_pset, sevID, default_sev_pset);
0584         // change log 5
0585         int limit = getAparameter<int>(sev_pset, "limit", defaults.NO_VALUE_SET);
0586         if (limit == defaults.NO_VALUE_SET) {  // change log 24
0587           limit = m_messageLoggerDefaults->sev_limit(filename, sevID);
0588         }
0589         if (limit != defaults.NO_VALUE_SET) {
0590           if (limit < 0)
0591             limit = 2000000000;  // change log 38
0592           dest_ctrl->setLimit(severity, limit);
0593         }
0594         int interval = getAparameter<int>(sev_pset, "reportEvery", defaults.NO_VALUE_SET);
0595         if (interval == defaults.NO_VALUE_SET) {  // change log 24
0596           interval = m_messageLoggerDefaults->sev_reportEvery(filename, sevID);
0597         }
0598         if (interval != defaults.NO_VALUE_SET)
0599           dest_ctrl->setInterval(severity, interval);
0600         // change log 2
0601         int timespan = getAparameter<int>(sev_pset, "timespan", defaults.NO_VALUE_SET);
0602         if (timespan == defaults.NO_VALUE_SET) {  // change log 24
0603           timespan = m_messageLoggerDefaults->sev_timespan(filename, sevID);
0604         }
0605         if (timespan != defaults.NO_VALUE_SET) {
0606           if (timespan < 0)
0607             timespan = 2000000000;  // change log 38
0608           dest_ctrl->setTimespan(severity, timespan);
0609         }
0610       }  // for
0611
0612       // establish this destination's linebreak policy:
0613       // change log 5
0614       bool noLineBreaks = getAparameter<bool>(dest_pset, "noLineBreaks", defaults.noLineBreaks_);
0615       if (noLineBreaks) {
0616         dest_ctrl->setLineLength(32000);
0617       } else {
0618         // change log 5
0619         int lineLen = getAparameter<int>(dest_pset, "lineLength", defaults.lineLength_);
0620         dest_ctrl->setLineLength(lineLen);
0621       }
0622
0623       // if indicated, suppress time stamps in this destination's output
0624       bool suppressTime = getAparameter<bool>(dest_pset, "noTimeStamps", defaults.noTimeStamps_);
0625       if (suppressTime) {
0626         dest_ctrl->suppressTime();
0627       }
0628
0629     }  // ThreadSafeLogMessageLoggerScribe::configure_dest()
0630
0631     std::vector<std::string> ThreadSafeLogMessageLoggerScribe::configure_ordinary_destinations(
0632         edm::ParameterSet const& job_pset, ConfigurableDefaults const& defaults, vString const& categories) {
0633       vString const empty_vString;
0634       std::string const empty_String;
0635       edm::ParameterSet const empty_PSet;
0636
0637       // Initialize unversal suppression variables
0638       MessageDrop::debugAlwaysSuppressed = true;
0639       MessageDrop::infoAlwaysSuppressed = true;
0640       MessageDrop::fwkInfoAlwaysSuppressed = true;
0641       MessageDrop::warningAlwaysSuppressed = true;
0642
0643       // grab list of destinations:
0644       vString destinations = getAparameter<vString>(job_pset, "destinations", empty_vString);
0645
0646       // Use the default list of destinations if and only if the grabbed list is
0647       // empty                          // change log 24
0648       if (destinations.empty()) {
0649         destinations = m_messageLoggerDefaults->destinations;
0650       }
0651
0652       // dial down the early destination if other dest's are supplied:
0653       if (!destinations.empty())
0654         m_early_dest->setThreshold(ELhighestSeverity);
0655
0656       // establish each destination:
0657       std::vector<std::string> ordinary_destination_filenames;
0658       for (vString::const_iterator it = destinations.begin(); it != destinations.end(); ++it) {
0659         const std::string& filename = *it;
0660         const std::string& psetname = filename;
0661
0662         // check that this destination is not just a placeholder // change log 11
0663         edm::ParameterSet dest_pset = getAparameter<edm::ParameterSet>(job_pset, psetname, empty_PSet);
0664         bool is_placeholder = getAparameter<bool>(dest_pset, "placeholder", false);
0665         if (is_placeholder)
0666           continue;
0667
0668         // Modify the file name if extension or name is explicitly specified
0669         // change log 14
0670
0671         // Although for an ordinary destination there is no output attribute
0672         // for the cfg (you can use filename instead) we provide output() for
0673         // uniformity with the statistics destinations.  The "right way" to
0674         // work this would have been to provide a filename() method, along with
0675         // an extension() method.  We recognize the potential name confusion here
0676         // (filename(filename))!
0677
0678         auto const actual_filename = destinationFileName(dest_pset, psetname);
0679
0680         // Check that this is not a duplicate name          // change log 18
0681         if (m_stream_ps.find(actual_filename) != m_stream_ps.end()) {
0682           if (m_clean_slate_configuration) {  // change log 22
0683                                               //        throw edm::Exception ( edm::errors::Configuration )
0684             LogError("duplicateDestination")  // change log 35
0685                 << "Duplicate name for a MessageLogger Destination: " << actual_filename << "\n"
0686                 << "Only the first configuration instructions are used";
0687             continue;
0688           } else {
0689             LogWarning("duplicateDestination")
0690                 << "Duplicate name for a MessageLogger Destination: " << actual_filename << "\n"
0691                 << "Only original configuration instructions are used";
0692             continue;
0693           }
0694         }
0695
0696         ordinary_destination_filenames.push_back(actual_filename);
0697
0698         // attach the current destination, keeping a control handle to it:
0699         std::shared_ptr<ELdestination> dest_ctrl = makeDestinationCtrl(actual_filename);
0700         // now configure this destination:
0701         configure_dest(job_pset, defaults, categories, dest_ctrl, dest_pset, psetname);
0702
0703       }  // for [it = destinations.begin() to end()]
0704
0705       return ordinary_destination_filenames;
0706     }  // configure_ordinary_destinations
0707
0708     void ThreadSafeLogMessageLoggerScribe::configure_statistics_dest(edm::ParameterSet const& job_pset,
0709                                                                      ConfigurableDefaults const& defaults,
0710                                                                      vString const& categories,
0711                                                                      edm::ParameterSet const& stat_pset,
0712                                                                      std::string const& psetname,
0713                                                                      std::string const& filename) {
0714       auto os_p = m_stream_ps[filename];
0715
0716       auto stat = std::make_shared<ELstatistics>(*os_p);
0717       m_admin_p->attach(stat);
0718       m_statisticsDestControls.push_back(stat);
0719       bool reset = getAparameter<bool>(stat_pset, "resetStatistics", false);
0720       if (not reset) {
0721         //check for old syntax
0722         reset = getAparameter<bool>(stat_pset, "reset", false);
0723       }
0724       m_statisticsResets.push_back(reset);
0725
0726       // now configure this destination:
0727       configure_dest(job_pset, defaults, categories, stat, stat_pset, psetname);
0728
0729       std::string dest_threshold = getAparameter<std::string>(stat_pset, "statisticsThreshold", std::string());
0730       if (not dest_threshold.empty()) {
0731         ELseverityLevel threshold_sev(dest_threshold);
0732         stat->setThreshold(threshold_sev);
0733
0734         setGlobalThresholds(threshold_sev);
0735       }
0736
0737       // and suppress the desire to do an extra termination summary just because
0738       // of end-of-job info messages
0739       stat->noTerminationSummary();
0740     }
0741
0742     void ThreadSafeLogMessageLoggerScribe::configure_statistics(edm::ParameterSet const& job_pset,
0743                                                                 ConfigurableDefaults const& defaults,
0744                                                                 vString const& categories,
0745                                                                 vString const& ordinary_destination_filenames) {
0746       vString const empty_vString;
0747       std::string const empty_String;
0748       edm::ParameterSet const empty_PSet;
0749
0750       // grab list of statistics destinations:
0751       vString statistics = getAparameter<vString>(job_pset, "statistics", empty_vString);
0752
0753       bool no_statistics_configured = statistics.empty();  // change log 24
0754
0755       if (no_statistics_configured) {
0756         // Read the list of staistics destinations from hardwired defaults,
0757         // but only if there is also no list of ordinary destinations.
0758         // (If a cfg specifies destinations, and no statistics, assume that
0759         // is what the user wants.)
0760         vString destinations = getAparameter<vString>(job_pset, "destinations", empty_vString);
0761         if (destinations.empty()) {
0762           statistics = m_messageLoggerDefaults->statistics;
0763           no_statistics_configured = statistics.empty();
0764         } else {
0765           for (auto const& dest : destinations) {
0766             edm::ParameterSet stat_pset = getAparameter<edm::ParameterSet>(job_pset, dest, empty_PSet);
0767             if (getAparameter<bool>(stat_pset, "enableStatistics", false)) {
0768               statistics.push_back(dest);
0769             }
0770           }
0771         }
0772       }
0773
0774       // establish each statistics destination:
0775       for (auto const& psetname : statistics) {
0776         // check that this destination is not just a placeholder // change log 20
0777         edm::ParameterSet stat_pset = getAparameter<edm::ParameterSet>(job_pset, psetname, empty_PSet);
0778         bool is_placeholder = getAparameter<bool>(stat_pset, "placeholder", false);
0779         if (is_placeholder)
0780           continue;
0781
0782         // Determine the destination file name
0783         std::string filename = getAparameter<std::string>(stat_pset, "output", empty_String);
0784         if (filename == empty_String) {
0785           filename = m_messageLoggerDefaults->output(psetname);  // change log 31
0786           if (filename == empty_String) {
0787             filename = psetname;
0788           }
0789         }
0790
0791         // Modify the file name if extension or name is explicitly specified
0792         // change log 14 -- probably suspenders and a belt, because ouput option
0793         // is present, but uniformity is nice.
0794
0795         std::string explicit_filename = getAparameter<std::string>(stat_pset, "filename", filename);
0796         if (explicit_filename != empty_String)
0797           filename = explicit_filename;
0798         std::string explicit_extension = getAparameter<std::string>(stat_pset, "extension", empty_String);
0799         if (explicit_extension != empty_String) {
0800           if (explicit_extension[0] == '.') {
0801             filename += explicit_extension;
0802           } else {
0803             filename = filename + "." + explicit_extension;
0804           }
0805         }
0806
0807         // Attach a default extension of .log if there is no extension on a file
0808         // change log 18 - this had been done in concert with attaching destination
0809
0810         std::string actual_filename = filename;              // change log 4
0811         if ((filename != "cout") && (filename != "cerr")) {  // change log 23
0812           const std::string::size_type npos = std::string::npos;
0813           if (filename.find('.') == npos) {
0814             actual_filename += ".log";
0815           }
0816         }
0817
0818         // Check that this is not a duplicate name -
0819         // unless it is an ordinary destination (which stats can share)
0820         if (!search_all(ordinary_destination_filenames, actual_filename)) {
0821           if (m_stream_ps.find(actual_filename) != m_stream_ps.end()) {
0822             if (m_clean_slate_configuration) {  // change log 22
0823               throw edm::Exception(edm::errors::Configuration)
0824                   << "Duplicate name for a MessageLogger Statistics Destination: " << actual_filename << "\n";
0825             } else {
0826               LogWarning("duplicateDestination")
0827                   << "Duplicate name for a MessageLogger Statistics Destination: " << actual_filename << "\n"
0828                   << "Only original configuration instructions are used";
0829               continue;
0830             }
0831           }
0832         }
0833
0834         // create (if statistics file does not match any destination file name)
0835         // or note (if statistics file matches a destination file name) the ostream.
0836         // But if no statistics destinations were provided in the config, do not
0837         // create a new destination for this hardwired statistics - only act if
0838         // it is matches a destination.  (shange log 24)
0839         bool statistics_destination_is_real = !no_statistics_configured;
0840         std::ostream* os_p;
0841         if (m_stream_ps.find(actual_filename) == m_stream_ps.end()) {
0842           if (actual_filename == "cout") {
0843             os_p = &std::cout;
0844           } else if (actual_filename == "cerr") {
0845             os_p = &std::cerr;
0846           } else {
0847             auto os_sp = std::make_shared<std::ofstream>(actual_filename.c_str());
0848             m_file_ps.push_back(os_sp);
0849             os_p = os_sp.get();
0850           }
0851           m_stream_ps[actual_filename] = os_p;
0852         } else {
0853           statistics_destination_is_real = true;  // change log 24
0854         }
0855
0856         if (statistics_destination_is_real) {  // change log 24
0857                                                // attach the statistics destination, keeping a control handle to it:
0858
0859           configure_statistics_dest(job_pset, defaults, categories, stat_pset, psetname, actual_filename);
0860         }
0861
0862       }  // for [it = statistics.begin() to end()]
0863
0864     }  // configure_statistics
0865
0866     void ThreadSafeLogMessageLoggerScribe::parseCategories(std::string const& s, std::vector<std::string>& cats) {
0867       const std::string::size_type npos = std::string::npos;
0868       std::string::size_type i = 0;
0869       while (i != npos) {
0870         std::string::size_type j = s.find('|', i);
0871         cats.push_back(s.substr(i, j - i));
0872         i = j;
0873         while ((i != npos) && (s[i] == '|'))
0874           ++i;
0875         // the above handles cases of || and also | at end of string
0876       }
0877       // Note:  This algorithm assigns, as desired, one null category if it
0878       //        encounters an empty categories string
0879     }
0880
0881     void ThreadSafeLogMessageLoggerScribe::triggerStatisticsSummaries() {
0882       assert(m_statisticsDestControls.size() == m_statisticsResets.size());
0883       for (unsigned int i = 0; i != m_statisticsDestControls.size(); ++i) {
0884         m_statisticsDestControls[i]->summary(m_tooManyWaitingMessagesCount.load());
0885         if (m_statisticsResets[i])
0886           m_statisticsDestControls[i]->wipe();
0887       }
0888       auto dropped = m_tooManyWaitingMessagesCount.load();
0889       if (m_statisticsDestControls.empty() and dropped != 0) {
0890         if (m_stream_ps.find("cerr") != m_stream_ps.end()) {
0891           std::cerr << "MessageLogger: dropped waiting message count " << dropped << "\n";
0892         }
0893         if (m_stream_ps.find("cout") != m_stream_ps.end()) {
0894           std::cout << "MessageLogger: dropped waiting message count " << dropped << "\n";
0895         }
0896       }
0897     }
0898
0899     void ThreadSafeLogMessageLoggerScribe::triggerFJRmessageSummary(std::map<std::string, double>& sm)  // ChangeLog 29
0900     {
0901       if (m_statisticsDestControls.empty()) {
0902         sm["NoStatisticsDestinationsConfigured"] = 0.0;
0903       } else {
0904         m_statisticsDestControls[0]->summaryForJobReport(sm);
0905       }
0906     }
0907
0908     namespace {
0909       void fillDescriptions(edm::ConfigurationDescriptions& config) {
0910         edm::ParameterSetDescription topDesc;
0911
0912         topDesc.addUntracked<bool>("messageSummaryToJobReport", false);
0913         topDesc.addUntracked<std::string>("generate_preconfiguration_message", "");
0914         topDesc.addUntracked<unsigned int>("waiting_threshold", 100);
0915         topDesc.addUntracked<std::vector<std::string>>("suppressDebug", {});
0916         topDesc.addUntracked<std::vector<std::string>>("suppressInfo", {});
0917         topDesc.addUntracked<std::vector<std::string>>("suppressFwkInfo", {});
0918         topDesc.addUntracked<std::vector<std::string>>("suppressWarning", {});
0919         topDesc.addUntracked<std::vector<std::string>>("suppressError", {});
0920         topDesc.addUntracked<std::vector<std::string>>("debugModules", {});
0921
0922         edm::ParameterSetDescription category;
0923         category.addUntracked<int>("reportEvery", 1);
0924         category.addUntracked<int>("limit", ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults::NO_VALUE_SET)
0925             ->setComment(
0926                 "Set a limit on the number of messages of this category. The default value is used to denote no "
0927                 "limit.");
0928         category.addOptionalUntracked<int>("timespan");
0929
0930         edm::ParameterSetDescription destination_base;
0931         destination_base.addOptionalUntracked<bool>("noLineBreaks");
0932         destination_base.addOptionalUntracked<bool>("noTimeStamps");
0933         destination_base.addOptionalUntracked<int>("lineLength");
0934         destination_base.addOptionalUntracked<std::string>("threshold");
0935         destination_base.addOptionalUntracked<std::string>("statisticsThreshold");
0936
0937         edm::ParameterWildcard<edm::ParameterSetDescription> catnode("*", edm::RequireZeroOrMore, false, category);
0938         catnode.setComment("Specialize either a category or any of 'DEBUG', 'INFO', 'FWKINFO', 'WARNING' or 'ERROR'");
0939         destination_base.addNode(catnode);
0940
0941         edm::ParameterSetDescription destination_noStats(destination_base);
0942         destination_noStats.addUntracked<bool>("enableStatistics", false);
0943         destination_noStats.addUntracked<bool>("resetStatistics", false);
0944
0945         {
0946           edm::ParameterSetDescription default_pset;
0947           default_pset.addUntracked<int>("reportEvery", 1);
0948           default_pset.addUntracked<int>("limit", ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults::NO_VALUE_SET)
0949               ->setComment(
0950                   "Set a limit on the number of messages of this category. The default value is used to denote no "
0951                   "limit.");
0952           default_pset.addOptionalUntracked<int>("timespan");
0953           default_pset.addUntracked<bool>("noLineBreaks", false);
0954           default_pset.addUntracked<bool>("noTimeStamps", false);
0955           default_pset.addUntracked<int>("lineLength", 80);
0956           default_pset.addUntracked<std::string>("threshold", "INFO");
0957           default_pset.addUntracked<std::string>("statisticsThreshold", "INFO");
0958           default_pset.addNode(catnode);
0959
0960           edm::ParameterSetDescription cerr_destination(destination_base);
0961           cerr_destination.addUntracked<bool>("enableStatistics", true);
0962           cerr_destination.addUntracked<bool>("resetStatistics", false);
0963           cerr_destination.addUntracked<bool>("enable", true);
0964
0965           edm::ParameterSetDescription cout_destination(destination_noStats);
0966           cout_destination.addUntracked<bool>("enable", false);
0967
0968           edm::ParameterSetDescription fileDestination(destination_noStats);
0969
0970           fileDestination.addOptionalUntracked<std::string>("output");
0971           fileDestination.addOptionalUntracked<std::string>("filename");
0972           fileDestination.addOptionalUntracked<std::string>("extension");
0973           edm::ParameterSetDescription files;
0974           edm::ParameterWildcard<edm::ParameterSetDescription> fileWildcard(
0975               "*", edm::RequireZeroOrMore, false, fileDestination);
0976           files.addNode(fileWildcard);
0977
0978           std::map<std::string, edm::ParameterSetDescription> standards = {
0979               {"cerr", cerr_destination}, {"cout", cout_destination}, {"default", default_pset}, {"files", files}};
0980
0981           edm::ParameterWildcardWithSpecifics psets("*", edm::RequireZeroOrMore, false, category, std::move(standards));
0982           topDesc.addNode(psets);
0983         }
0984
0985         config.addDefault(topDesc);
0986       }
0987     }  // namespace
0988
0989     void ThreadSafeLogMessageLoggerScribe::validate(edm::ParameterSet& pset) const {
0990       // See if old config API is being used
0991       if (usingOldConfig(pset))
0992         return;
0993       if (not pset.exists("files") and
0994           ((pset.exists("destinations") or pset.existsAs<std::vector<std::string>>("statistics", true) or
0995             pset.existsAs<std::vector<std::string>>("statistics", false) or pset.exists("categories")))) {
0996         return;
0997       }
0998
0999       ConfigurationDescriptions config{"MessageLogger", "MessageLogger"};
1000       fillDescriptions(config);
1001
1002       config.validate(pset, "MessageLogger");
1003     }
1004
1005   }  // end of namespace service
1006 }  // end of namespace edm