MessageService/src/ThreadSafeLogMessageLoggerScribe.cc

0001 // ----------------------------------------------------------------------
0002 //
0003 // ThreadSafeLogMessageLoggerScribe.cc
0004 //
0005 // NOTE: This was originally copied from MessageLoggerScribe but removed
0006 //  the potential use of the ThreadQueue. MessageLoggerScribe was not
0007 //  modified since it was decided we might have to keep the old behaviour
0008 //  around for 'legacy' reasons.
0009 //
0010 // ----------------------------------------------------------------------
0011
0012 #include "FWCore/MessageService/src/ThreadSafeLogMessageLoggerScribe.h"
0013 #include "FWCore/MessageService/src/ELadministrator.h"
0014 #include "FWCore/MessageService/src/ELoutput.h"
0015 #include "FWCore/MessageService/src/ELstatistics.h"
0016
0017 #include "FWCore/MessageLogger/interface/ErrorObj.h"
0018 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0019 #include "FWCore/MessageService/src/ConfigurationHandshake.h"
0020 #include "FWCore/MessageLogger/interface/MessageDrop.h"      // change log 37
0021 #include "FWCore/MessageLogger/interface/ELseverityLevel.h"  // change log 37
0022
0023 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0024 #include "FWCore/ParameterSet/interface/ParameterWildcardWithSpecifics.h"
0025
0026 #include "FWCore/Utilities/interface/EDMException.h"
0027 #include "FWCore/Utilities/interface/Algorithms.h"
0028
0029 #include <algorithm>
0030 #include <cassert>
0031 #include <fstream>
0032 #include <string>
0033 #include <csignal>
0034
0035 using std::cerr;
0036 using namespace edm::messagelogger;
0037
0038 namespace edm {
0039   namespace service {
0040
0041     ThreadSafeLogMessageLoggerScribe::ThreadSafeLogMessageLoggerScribe()
0042         : m_admin_p(std::make_shared<ELadministrator>()),
0043           m_early_dest(m_admin_p->attach(std::make_shared<ELoutput>(std::cerr, false))),
0044           m_clean_slate_configuration(true),
0045           m_active(true),
0046           m_purge_mode(false),
0047           m_count(0),
0048           m_messageBeingSent(false),
0049           m_waitingThreshold(100),
0050           m_tooManyWaitingMessagesCount(0) {}
0051
0052     ThreadSafeLogMessageLoggerScribe::~ThreadSafeLogMessageLoggerScribe() {
0053       //if there are any waiting message, finish them off
0054       ErrorObj* errorobj_p = nullptr;
0055       std::vector<std::string> categories;
0056       while (m_waitingMessages.try_pop(errorobj_p)) {
0057         if (not m_purge_mode) {
0058           categories.clear();
0059           parseCategories(errorobj_p->xid().id, categories);
0060           for (unsigned int icat = 0; icat < categories.size(); ++icat) {
0061             errorobj_p->setID(categories[icat]);
0062             m_admin_p->log(*errorobj_p);  // route the message text
0063           }
0064         }
0065         delete errorobj_p;
0066       }
0067
0068       m_admin_p->finish();
0069     }
0070
0071     void ThreadSafeLogMessageLoggerScribe::runCommand(  // changeLog 32
0072         MessageLoggerQ::OpCode opcode,
0073         void* operand) {
0074       switch (opcode) {  // interpret the work item
0075         default: {
0076           assert(false);  // can't happen (we certainly hope!)
0077           break;
0078         }
0079         case MessageLoggerQ::END_THREAD: {
0080           break;
0081         }
0082         case MessageLoggerQ::LOG_A_MESSAGE: {
0083           ErrorObj* errorobj_p = static_cast<ErrorObj*>(operand);
0084           try {
0085             if (m_active && !m_purge_mode) {
0086               log(errorobj_p);
0087             }
0088           } catch (cms::Exception& e) {
0089             ++m_count;
0090             std::cerr << "ThreadSafeLogMessageLoggerScribe caught " << m_count << " cms::Exceptions, text = \n"
0091                       << e.what() << "\n";
0092
0093             if (m_count > 25) {
0094               cerr << "MessageLogger will no longer be processing "
0095                    << "messages due to errors (entering purge mode).\n";
0096               m_purge_mode = true;
0097             }
0098           } catch (...) {
0099             std::cerr << "ThreadSafeLogMessageLoggerScribe caught an unknown exception and "
0100                       << "will no longer be processing "
0101                       << "messages. (entering purge mode)\n";
0102             m_purge_mode = true;
0103           }
0104           break;
0105         }
0106         case MessageLoggerQ::CONFIGURE: {  // changelog 17
0107           auto job_pset_p = std::unique_ptr<edm::ParameterSet>(
0108               static_cast<edm::ParameterSet*>(operand));  // propagate_const<T> has no reset() function
0109           validate(*job_pset_p);
0110           configure_errorlog(*job_pset_p);
0111           break;
0112         }
0113         case MessageLoggerQ::SUMMARIZE: {
0114           assert(operand == nullptr);
0115           CMS_SA_ALLOW try { triggerStatisticsSummaries(); } catch (cms::Exception& e) {
0116             std::cerr << "ThreadSafeLogMessageLoggerScribe caught exception "
0117                       << "during summarize:\n"
0118                       << e.what() << "\n";
0119           } catch (...) {
0120             std::cerr << "ThreadSafeLogMessageLoggerScribe caught unkonwn exception type "
0121                       << "during summarize. (Ignored)\n";
0122           }
0123           break;
0124         }
0125         case MessageLoggerQ::JOBMODE: {  // change log 24
0126           std::string* jobMode_p = static_cast<std::string*>(operand);
0127           JobMode jm = MessageLoggerDefaults::mode(*jobMode_p);
0128           m_messageLoggerDefaults = value_ptr<MessageLoggerDefaults>(new MessageLoggerDefaults(jm));
0129           // Note - since m_messageLoggerDefaults is a value_ptr,
0130           //        there is no concern about deleting here.
0131           delete jobMode_p;  // dispose of the message text
0132                              // which will have been new-ed
0133                              // in MessageLogger.cc (service version)
0134           break;
0135         }
0136         case MessageLoggerQ::SHUT_UP: {
0137           assert(operand == nullptr);
0138           m_active = false;
0139           break;
0140         }
0141         case MessageLoggerQ::FLUSH_LOG_Q: {  // changelog 26
0142           break;
0143         }
0144         case MessageLoggerQ::GROUP_STATS: {  // change log 27
0145           std::string* cat_p = static_cast<std::string*>(operand);
0146           ELstatistics::noteGroupedCategory(*cat_p);
0147           delete cat_p;  // dispose of the message text
0148           break;
0149         }
0150         case MessageLoggerQ::FJR_SUMMARY: {  // changelog 29
0151           std::map<std::string, double>* smp = static_cast<std::map<std::string, double>*>(operand);
0152           triggerFJRmessageSummary(*smp);
0153           break;
0154         }
0155       }  // switch
0156
0157     }  // ThreadSafeLogMessageLoggerScribe::runCommand(opcode, operand)
0158
0159     void ThreadSafeLogMessageLoggerScribe::log(ErrorObj* errorobj_p) {
0160       bool expected = false;
0161       std::unique_ptr<ErrorObj> obj(errorobj_p);
0162       if (m_messageBeingSent.compare_exchange_strong(expected, true)) {
0163         std::vector<std::string> categories;
0164         parseCategories(errorobj_p->xid().id, categories);
0165         for (unsigned int icat = 0; icat < categories.size(); ++icat) {
0166           errorobj_p->setID(categories[icat]);
0167           m_admin_p->log(*errorobj_p);  // route the message text
0168         }
0169         //process any waiting messages
0170         errorobj_p = nullptr;
0171         while (not m_purge_mode and m_waitingMessages.try_pop(errorobj_p)) {
0172           obj.reset(errorobj_p);
0173           categories.clear();
0174           parseCategories(errorobj_p->xid().id, categories);
0175           for (unsigned int icat = 0; icat < categories.size(); ++icat) {
0176             errorobj_p->setID(categories[icat]);
0177             m_admin_p->log(*errorobj_p);  // route the message text
0178           }
0179         }
0180         m_messageBeingSent.store(false);
0181       } else {
0182         if (m_waitingMessages.unsafe_size() < m_waitingThreshold) {
0183           obj.release();
0184           m_waitingMessages.push(errorobj_p);
0185         } else {
0186           ++m_tooManyWaitingMessagesCount;
0187         }
0188       }
0189     }
0190
0191     namespace {
0192       bool usingOldConfig(edm::ParameterSet const& pset) {
0193         if (not pset.exists("files") and
0194             ((pset.exists("destinations") or pset.existsAs<std::vector<std::string>>("statistics", true) or
0195               pset.existsAs<std::vector<std::string>>("statistics", false) or pset.exists("categories")))) {
0196           return true;
0197         }
0198         return false;
0199       }
0200
0201       std::set<std::string> findCategoriesInDestination(edm::ParameterSet const& pset) {
0202         auto psets = pset.getParameterNamesForType<edm::ParameterSet>(false);
0203         auto itFound = std::find(psets.begin(), psets.end(), "default");
0204         if (itFound != psets.end()) {
0205           psets.erase(itFound);
0206         }
0207
0208         return std::set<std::string>(psets.begin(), psets.end());
0209       }
0210       std::vector<std::string> findAllCategories(edm::ParameterSet const& pset) {
0211         std::set<std::string> categories;
0212
0213         auto psets = pset.getParameterNamesForType<edm::ParameterSet>(false);
0214         auto itFound = std::find(psets.begin(), psets.end(), "default");
0215         if (itFound != psets.end()) {
0216           categories = findCategoriesInDestination(pset.getUntrackedParameter<edm::ParameterSet>("default"));
0217           psets.erase(itFound);
0218         }
0219
0220         itFound = std::find(psets.begin(), psets.end(), "cout");
0221         if (itFound != psets.end()) {
0222           categories.merge(findCategoriesInDestination(pset.getUntrackedParameter<edm::ParameterSet>("cout")));
0223           psets.erase(itFound);
0224         }
0225
0226         itFound = std::find(psets.begin(), psets.end(), "cerr");
0227         if (itFound != psets.end()) {
0228           categories.merge(findCategoriesInDestination(pset.getUntrackedParameter<edm::ParameterSet>("cerr")));
0229           psets.erase(itFound);
0230         }
0231
0232         auto const& files = pset.getUntrackedParameter<edm::ParameterSet>("files");
0233         for (auto const& name : files.getParameterNamesForType<edm::ParameterSet>(false)) {
0234           categories.merge(findCategoriesInDestination(files.getUntrackedParameter<edm::ParameterSet>(name)));
0235         }
0236         categories.insert(psets.begin(), psets.end());
0237
0238         return std::vector<std::string>(categories.begin(), categories.end());
0239       }
0240
0241     }  // namespace
0242
0243     std::string ThreadSafeLogMessageLoggerScribe::destinationFileName(edm::ParameterSet const& dest_pset,
0244                                                                       std::string const& psetname) const {
0245       // Determine the destination file name to use if no explicit filename is
0246       // supplied in the cfg.
0247       std::string const empty_String;
0248       std::string filename = psetname;
0249       std::string filename_default = getAparameter<std::string>(dest_pset, "output", empty_String);
0250       if (filename_default == empty_String) {
0251         filename_default = m_messageLoggerDefaults->output(psetname);  // change log 31
0252         if (filename_default == empty_String) {
0253           filename_default = filename;
0254         }
0255       }
0256
0257       std::string explicit_filename = getAparameter<std::string>(dest_pset, "filename", filename_default);
0258       if (explicit_filename != empty_String)
0259         filename = explicit_filename;
0260       std::string explicit_extension = getAparameter<std::string>(dest_pset, "extension", empty_String);
0261       if (explicit_extension != empty_String) {
0262         if (explicit_extension[0] == '.') {
0263           filename += explicit_extension;
0264         } else {
0265           filename = filename + "." + explicit_extension;
0266         }
0267       }
0268
0269       // Attach a default extension of .log if there is no extension on a file
0270       if ((filename != "cout") && (filename != "cerr")) {
0271         if (filename.find('.') == std::string::npos) {
0272           filename += ".log";
0273         }
0274       }
0275       return filename;
0276     }
0277
0278     void ThreadSafeLogMessageLoggerScribe::configure_errorlog_new(edm::ParameterSet& job_pset) {
0279       {
0280         auto preconfiguration_message =
0281             job_pset.getUntrackedParameter<std::string>("generate_preconfiguration_message");
0282         if (not preconfiguration_message.empty()) {
0283           // To test a preconfiguration message without first going thru the
0284           // configuration we are about to do, we issue the message (so it sits
0285           // on the queue), then copy the processing that the LOG_A_MESSAGE case
0286           // does.  We suppress the timestamp to allow for automated unit testing.
0287           m_early_dest->suppressTime();
0288           LogError("preconfiguration") << preconfiguration_message;
0289         }
0290       }
0291       if (!m_stream_ps.empty()) {
0292         LogWarning("multiLogConfig") << "The message logger has been configured multiple times";
0293         m_clean_slate_configuration = false;  // Change Log 22
0294       }
0295       m_waitingThreshold = job_pset.getUntrackedParameter<unsigned int>("waiting_threshold");
0296
0297       auto defaults = parseDefaults(job_pset);
0298       auto categories = findAllCategories(job_pset);
0299
0300       // Initialize unversal suppression variables
0301       MessageDrop::debugAlwaysSuppressed = true;
0302       MessageDrop::infoAlwaysSuppressed = true;
0303       MessageDrop::fwkInfoAlwaysSuppressed = true;
0304       MessageDrop::warningAlwaysSuppressed = true;
0305
0306       m_early_dest->setThreshold(ELhighestSeverity);
0307
0308       auto cout_dest = job_pset.getUntrackedParameter<edm::ParameterSet>("cout");
0309       if (cout_dest.getUntrackedParameter<bool>("enable")) {
0310         auto dest_ctrl = makeDestinationCtrl("cout");
0311         configure_dest(job_pset, defaults, categories, dest_ctrl, cout_dest, "cout");
0312       }
0313
0314       auto cerr_dest = job_pset.getUntrackedParameter<edm::ParameterSet>("cerr");
0315       if (cerr_dest.getUntrackedParameter<bool>("enable")) {
0316         auto dest_ctrl = makeDestinationCtrl("cerr");
0317         configure_dest(job_pset, defaults, categories, dest_ctrl, cerr_dest, "cerr");
0318       }
0319
0320       auto const& files = job_pset.getUntrackedParameter<edm::ParameterSet>("files");
0321       for (auto const& name : files.getParameterNamesForType<edm::ParameterSet>(false)) {
0322         auto const& dest_pset = files.getUntrackedParameter<edm::ParameterSet>(name);
0323         auto const actual_filename = destinationFileName(dest_pset, name);
0324
0325         // Check that this is not a duplicate name
0326         if (m_stream_ps.find(actual_filename) != m_stream_ps.end()) {
0327           if (m_clean_slate_configuration) {
0328             throw cms::Exception("DuplicateDestination")
0329                 << "Duplicate name for a MessageLogger Destination: " << actual_filename << "\n"
0330                 << "Please modify the configuration to use unique file names.";
0331           } else {
0332             LogWarning("duplicateDestination")
0333                 << "Duplicate name for a MessageLogger Destination: " << actual_filename << "\n"
0334                 << "Only original configuration instructions are used";
0335             continue;
0336           }
0337         }
0338
0339         auto dest_ctrl = makeDestinationCtrl(actual_filename);
0340         configure_dest(job_pset, defaults, categories, dest_ctrl, dest_pset, name);
0341       }
0342       //NOTE: statistics destinations MUST BE last in the list else they can be fooled into
0343       // thinking a message has been ignored just because a later destination which uses it
0344       // falls later in the list.
0345       for (auto const& name : files.getParameterNamesForType<edm::ParameterSet>(false)) {
0346         auto const& dest_pset = files.getUntrackedParameter<edm::ParameterSet>(name);
0347         auto const actual_filename = destinationFileName(dest_pset, name);
0348         if (getAparameter<bool>(dest_pset, "enableStatistics", false)) {
0349           configure_statistics_dest(job_pset, defaults, categories, dest_pset, name, actual_filename);
0350         }
0351       }
0352       if (cout_dest.getUntrackedParameter<bool>("enable") and
0353           getAparameter<bool>(cout_dest, "enableStatistics", true)) {
0354         configure_statistics_dest(job_pset, defaults, categories, cout_dest, "cout", "cout");
0355       }
0356       if (cerr_dest.getUntrackedParameter<bool>("enable") and
0357           getAparameter<bool>(cerr_dest, "enableStatistics", true)) {
0358         configure_statistics_dest(job_pset, defaults, categories, cerr_dest, "cerr", "cerr");
0359       }
0360     }
0361
0362     void ThreadSafeLogMessageLoggerScribe::configure_errorlog(edm::ParameterSet& job_pset) {
0363       if (not usingOldConfig(job_pset)) {
0364         configure_errorlog_new(job_pset);
0365         return;
0366       }
0367       const vString empty_vString;
0368       const std::string empty_String;
0369       const edm::ParameterSet empty_PSet;
0370
0371       // The following is present to test pre-configuration message handling:
0372       std::string preconfiguration_message =
0373           getAparameter<std::string>(job_pset, "generate_preconfiguration_message", empty_String);
0374       if (preconfiguration_message != empty_String) {
0375         // To test a preconfiguration message without first going thru the
0376         // configuration we are about to do, we issue the message (so it sits
0377         // on the queue), then copy the processing that the LOG_A_MESSAGE case
0378         // does.  We suppress the timestamp to allow for automated unit testing.
0379         m_early_dest->suppressTime();
0380         LogError("preconfiguration") << preconfiguration_message;
0381       }
0382
0383       if (!m_stream_ps.empty()) {
0384         LogWarning("multiLogConfig") << "The message logger has been configured multiple times";
0385         m_clean_slate_configuration = false;  // Change Log 22
0386       }
0387       m_waitingThreshold = getAparameter<unsigned int>(job_pset, "waiting_threshold", 100);
0388       auto defaults = parseDefaults(job_pset);
0389       // grab list of categories
0390       vString categories = getAparameter<vString>(job_pset, "categories", empty_vString);
0391       // grab list of hardwired categories (hardcats) -- these are to be added
0392       // to the list of categories -- change log 24
0393       {
0394         std::vector<std::string> hardcats = m_messageLoggerDefaults->categories;
0395         // combine the lists, not caring about possible duplicates (for now)
0396         copy_all(hardcats, std::back_inserter(categories));
0397       }  // no longer need hardcats
0398
0399       auto destination_names = configure_ordinary_destinations(job_pset, defaults, categories);
0400       configure_statistics(job_pset, defaults, categories, destination_names);
0401     }  // ThreadSafeLogMessageLoggerScribe::configure_errorlog()
0402
0403     std::shared_ptr<ELdestination> ThreadSafeLogMessageLoggerScribe::makeDestinationCtrl(std::string const& filename) {
0404       std::shared_ptr<ELdestination> dest_ctrl;
0405       if (filename == "cout") {
0406         dest_ctrl = m_admin_p->attach(std::make_shared<ELoutput>(std::cout));
0407         m_stream_ps["cout"] = &std::cout;
0408       } else if (filename == "cerr") {
0409         m_early_dest->setThreshold(ELzeroSeverity);
0410         dest_ctrl = m_early_dest;
0411         m_stream_ps["cerr"] = &std::cerr;
0412       } else {
0413         auto os_sp = std::make_shared<std::ofstream>(filename.c_str());
0414         m_file_ps.push_back(os_sp);
0415         dest_ctrl = m_admin_p->attach(std::make_shared<ELoutput>(*os_sp));
0416         m_stream_ps[filename] = os_sp.get();
0417       }
0418       return dest_ctrl;
0419     }
0420
0421     namespace {
0422       void setGlobalThresholds(ELseverityLevel threshold_sev) {
0423         if (threshold_sev <= ELseverityLevel::ELsev_success) {
0424           edm::MessageDrop::debugAlwaysSuppressed = false;
0425         }
0426         if (threshold_sev <= ELseverityLevel::ELsev_info) {
0427           edm::MessageDrop::infoAlwaysSuppressed = false;
0428         }
0429         if (threshold_sev <= ELseverityLevel::ELsev_fwkInfo) {
0430           edm::MessageDrop::fwkInfoAlwaysSuppressed = false;
0431         }
0432         if (threshold_sev <= ELseverityLevel::ELsev_warning) {
0433           edm::MessageDrop::warningAlwaysSuppressed = false;
0434         }
0435       }
0436     }  // namespace
0437
0438     ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults ThreadSafeLogMessageLoggerScribe::parseDefaults(
0439         edm::ParameterSet const& job_pset) {
0440       const edm::ParameterSet empty_PSet;
0441       ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults returnValue;
0442       // grab default limit/interval/timespan common to all destinations/categories:
0443       edm::ParameterSet default_pset = getAparameter<edm::ParameterSet>(job_pset, "default", empty_PSet);
0444       returnValue.limit_ = getAparameter<int>(
0445           default_pset, "limit", ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults::COMMON_DEFAULT_LIMIT);
0446       returnValue.reportEvery_ = getAparameter<int>(
0447           default_pset, "reportEvery", ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults::COMMON_DEFAULT_INTERVAL);
0448       returnValue.timespan_ = getAparameter<int>(
0449           default_pset, "timespan", ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults::COMMON_DEFAULT_TIMESPAN);
0450       std::string default_threshold = getAparameter<std::string>(job_pset, "threshold", std::string());
0451       returnValue.threshold_ = getAparameter<std::string>(default_pset, "threshold", default_threshold);
0452       returnValue.noLineBreaks_ = getAparameter<bool>(default_pset, "noLineBreaks", false);
0453       returnValue.lineLength_ = getAparameter<int>(default_pset, "lineLength", 80);
0454       returnValue.noTimeStamps_ = getAparameter<bool>(default_pset, "noTimeStamps", false);
0455
0456       return returnValue;
0457     }
0458
0459     void ThreadSafeLogMessageLoggerScribe::configure_dest(edm::ParameterSet const& job_pset,
0460                                                           ConfigurableDefaults const& defaults,
0461                                                           vString const& categories,
0462                                                           std::shared_ptr<ELdestination> dest_ctrl,
0463                                                           edm::ParameterSet const& dest_pset,
0464                                                           std::string const& filename) {
0465       vString const empty_vString;
0466       edm::ParameterSet const empty_PSet;
0467       std::string const empty_String;
0468
0469       // Defaults:                          // change log 3a
0470       const std::string COMMON_DEFAULT_THRESHOLD = "INFO";
0471
0472       vString const severities = {{"WARNING", "INFO", "FWKINFO", "ERROR", "DEBUG"}};
0473
0474       // grab default threshold common to all destinations
0475       std::string const default_threshold = getAparameter<std::string>(job_pset, "threshold", empty_String);
0476       // change log 3a
0477       // change log 24
0478
0479       // grab default limit/interval/timespan common to all destinations/categories:
0480       edm::ParameterSet const default_pset = getAparameter<edm::ParameterSet>(job_pset, "default", empty_PSet);
0481
0482       // See if this is just a placeholder          // change log 9
0483       bool is_placeholder = getAparameter<bool>(dest_pset, "placeholder", false);
0484       if (is_placeholder)
0485         return;
0486
0487       // grab this destination's default limit/interval/timespan:
0488       edm::ParameterSet dest_default_pset = getAparameter<edm::ParameterSet>(dest_pset, "default", empty_PSet);
0489       int dest_default_limit = getAparameter<int>(dest_default_pset, "limit", defaults.limit_);
0490       int dest_default_interval = getAparameter<int>(dest_default_pset, "reportEvery", defaults.reportEvery_);
0491       // change log 6
0492       int dest_default_timespan = getAparameter<int>(dest_default_pset, "timespan", defaults.timespan_);
0493       // change log 1a
0494       if (dest_default_limit != defaults.NO_VALUE_SET) {
0495         if (dest_default_limit < 0)
0496           dest_default_limit = 2000000000;
0497         dest_ctrl->setLimit("*", dest_default_limit);
0498       }  // change log 1b, 2a, 2b
0499       if (dest_default_interval != defaults.NO_VALUE_SET) {  // change log 6
0500         dest_ctrl->setInterval("*", dest_default_interval);
0501       }
0502       if (dest_default_timespan != defaults.NO_VALUE_SET) {
0503         if (dest_default_timespan < 0)
0504           dest_default_timespan = 2000000000;
0505         dest_ctrl->setTimespan("*", dest_default_timespan);
0506       }  // change log 1b, 2a, 2b
0507
0508       // establish this destination's threshold:
0509       std::string dest_threshold = getAparameter<std::string>(dest_pset, "threshold", default_threshold);
0510       if (dest_threshold == empty_String) {
0511         dest_threshold = default_threshold;
0512       }
0513       if (dest_threshold == empty_String) {  // change log 34
0514         dest_threshold = defaults.threshold_;
0515       }
0516       if (dest_threshold == empty_String) {
0517         dest_threshold = m_messageLoggerDefaults->threshold(filename);
0518       }
0519       if (dest_threshold == empty_String)
0520         dest_threshold = COMMON_DEFAULT_THRESHOLD;
0521       ELseverityLevel threshold_sev(dest_threshold);
0522       dest_ctrl->setThreshold(threshold_sev);
0523       // change log 37
0524       setGlobalThresholds(threshold_sev);
0525
0526       // establish this destination's limit/interval/timespan for each category:
0527       for (vString::const_iterator id_it = categories.begin(); id_it != categories.end(); ++id_it) {
0528         const std::string& msgID = *id_it;
0529         edm::ParameterSet default_category_pset =
0530             getAparameter<edm::ParameterSet>(default_pset, msgID, empty_PSet);  // change log 5
0531         edm::ParameterSet category_pset = getAparameter<edm::ParameterSet>(dest_pset, msgID, default_category_pset);
0532
0533         int category_default_limit = getAparameter<int>(default_category_pset, "limit", defaults.NO_VALUE_SET);
0534         int limit = getAparameter<int>(category_pset, "limit", category_default_limit);
0535         if (limit == defaults.NO_VALUE_SET)
0536           limit = dest_default_limit;
0537         // change log 7
0538         int category_default_interval = getAparameter<int>(default_category_pset, "reportEvery", defaults.NO_VALUE_SET);
0539         int interval = getAparameter<int>(category_pset, "reportEvery", category_default_interval);
0540         if (interval == defaults.NO_VALUE_SET)
0541           interval = dest_default_interval;
0542         // change log 6  and then 7
0543         int category_default_timespan = getAparameter<int>(default_category_pset, "timespan", defaults.NO_VALUE_SET);
0544         int timespan = getAparameter<int>(category_pset, "timespan", category_default_timespan);
0545         if (timespan == defaults.NO_VALUE_SET)
0546           timespan = dest_default_timespan;
0547         // change log 7
0548
0549         const std::string& category = msgID;
0550         if (limit == defaults.NO_VALUE_SET) {  // change log 24
0551           limit = m_messageLoggerDefaults->limit(filename, category);
0552         }
0553         if (interval == defaults.NO_VALUE_SET) {  // change log 24
0554           interval = m_messageLoggerDefaults->reportEvery(filename, category);
0555         }
0556         if (timespan == defaults.NO_VALUE_SET) {  // change log 24
0557           timespan = m_messageLoggerDefaults->timespan(filename, category);
0558         }
0559
0560         if (limit != defaults.NO_VALUE_SET) {
0561           if (limit < 0)
0562             limit = 2000000000;
0563           dest_ctrl->setLimit(msgID, limit);
0564         }  // change log 2a, 2b
0565         if (interval != defaults.NO_VALUE_SET) {
0566           dest_ctrl->setInterval(msgID, interval);
0567         }  // change log 6
0568         if (timespan != defaults.NO_VALUE_SET) {
0569           if (timespan < 0)
0570             timespan = 2000000000;
0571           dest_ctrl->setTimespan(msgID, timespan);
0572         }  // change log 2a, 2b
0573
0574       }  // for
0575
0576       // establish this destination's limit for each severity:
0577       for (vString::const_iterator sev_it = severities.begin(); sev_it != severities.end(); ++sev_it) {
0578         const std::string& sevID = *sev_it;
0579         ELseverityLevel severity(sevID);
0580         edm::ParameterSet default_sev_pset = getAparameter<edm::ParameterSet>(default_pset, sevID, empty_PSet);
0581         edm::ParameterSet sev_pset = getAparameter<edm::ParameterSet>(dest_pset, sevID, default_sev_pset);
0582         // change log 5
0583         int limit = getAparameter<int>(sev_pset, "limit", defaults.NO_VALUE_SET);
0584         if (limit == defaults.NO_VALUE_SET) {  // change log 24
0585           limit = m_messageLoggerDefaults->sev_limit(filename, sevID);
0586         }
0587         if (limit != defaults.NO_VALUE_SET) {
0588           if (limit < 0)
0589             limit = 2000000000;  // change log 38
0590           dest_ctrl->setLimit(severity, limit);
0591         }
0592         int interval = getAparameter<int>(sev_pset, "reportEvery", defaults.NO_VALUE_SET);
0593         if (interval == defaults.NO_VALUE_SET) {  // change log 24
0594           interval = m_messageLoggerDefaults->sev_reportEvery(filename, sevID);
0595         }
0596         if (interval != defaults.NO_VALUE_SET)
0597           dest_ctrl->setInterval(severity, interval);
0598         // change log 2
0599         int timespan = getAparameter<int>(sev_pset, "timespan", defaults.NO_VALUE_SET);
0600         if (timespan == defaults.NO_VALUE_SET) {  // change log 24
0601           timespan = m_messageLoggerDefaults->sev_timespan(filename, sevID);
0602         }
0603         if (timespan != defaults.NO_VALUE_SET) {
0604           if (timespan < 0)
0605             timespan = 2000000000;  // change log 38
0606           dest_ctrl->setTimespan(severity, timespan);
0607         }
0608       }  // for
0609
0610       // establish this destination's linebreak policy:
0611       // change log 5
0612       bool noLineBreaks = getAparameter<bool>(dest_pset, "noLineBreaks", defaults.noLineBreaks_);
0613       if (noLineBreaks) {
0614         dest_ctrl->setLineLength(32000);
0615       } else {
0616         // change log 5
0617         int lineLen = getAparameter<int>(dest_pset, "lineLength", defaults.lineLength_);
0618         dest_ctrl->setLineLength(lineLen);
0619       }
0620
0621       // if indicated, suppress time stamps in this destination's output
0622       bool suppressTime = getAparameter<bool>(dest_pset, "noTimeStamps", defaults.noTimeStamps_);
0623       if (suppressTime) {
0624         dest_ctrl->suppressTime();
0625       }
0626
0627     }  // ThreadSafeLogMessageLoggerScribe::configure_dest()
0628
0629     std::vector<std::string> ThreadSafeLogMessageLoggerScribe::configure_ordinary_destinations(
0630         edm::ParameterSet const& job_pset, ConfigurableDefaults const& defaults, vString const& categories) {
0631       vString const empty_vString;
0632       std::string const empty_String;
0633       edm::ParameterSet const empty_PSet;
0634
0635       // Initialize unversal suppression variables
0636       MessageDrop::debugAlwaysSuppressed = true;
0637       MessageDrop::infoAlwaysSuppressed = true;
0638       MessageDrop::fwkInfoAlwaysSuppressed = true;
0639       MessageDrop::warningAlwaysSuppressed = true;
0640
0641       // grab list of destinations:
0642       vString destinations = getAparameter<vString>(job_pset, "destinations", empty_vString);
0643
0644       // Use the default list of destinations if and only if the grabbed list is
0645       // empty                          // change log 24
0646       if (destinations.empty()) {
0647         destinations = m_messageLoggerDefaults->destinations;
0648       }
0649
0650       // dial down the early destination if other dest's are supplied:
0651       if (!destinations.empty())
0652         m_early_dest->setThreshold(ELhighestSeverity);
0653
0654       // establish each destination:
0655       std::vector<std::string> ordinary_destination_filenames;
0656       for (vString::const_iterator it = destinations.begin(); it != destinations.end(); ++it) {
0657         const std::string& filename = *it;
0658         const std::string& psetname = filename;
0659
0660         // check that this destination is not just a placeholder // change log 11
0661         edm::ParameterSet dest_pset = getAparameter<edm::ParameterSet>(job_pset, psetname, empty_PSet);
0662         bool is_placeholder = getAparameter<bool>(dest_pset, "placeholder", false);
0663         if (is_placeholder)
0664           continue;
0665
0666         // Modify the file name if extension or name is explicitly specified
0667         // change log 14
0668
0669         // Although for an ordinary destination there is no output attribute
0670         // for the cfg (you can use filename instead) we provide output() for
0671         // uniformity with the statistics destinations.  The "right way" to
0672         // work this would have been to provide a filename() method, along with
0673         // an extension() method.  We recognize the potential name confusion here
0674         // (filename(filename))!
0675
0676         auto const actual_filename = destinationFileName(dest_pset, psetname);
0677
0678         // Check that this is not a duplicate name          // change log 18
0679         if (m_stream_ps.find(actual_filename) != m_stream_ps.end()) {
0680           if (m_clean_slate_configuration) {  // change log 22
0681                                               //        throw edm::Exception ( edm::errors::Configuration )
0682             LogError("duplicateDestination")  // change log 35
0683                 << "Duplicate name for a MessageLogger Destination: " << actual_filename << "\n"
0684                 << "Only the first configuration instructions are used";
0685             continue;
0686           } else {
0687             LogWarning("duplicateDestination")
0688                 << "Duplicate name for a MessageLogger Destination: " << actual_filename << "\n"
0689                 << "Only original configuration instructions are used";
0690             continue;
0691           }
0692         }
0693
0694         ordinary_destination_filenames.push_back(actual_filename);
0695
0696         // attach the current destination, keeping a control handle to it:
0697         std::shared_ptr<ELdestination> dest_ctrl = makeDestinationCtrl(actual_filename);
0698         // now configure this destination:
0699         configure_dest(job_pset, defaults, categories, dest_ctrl, dest_pset, psetname);
0700
0701       }  // for [it = destinations.begin() to end()]
0702
0703       return ordinary_destination_filenames;
0704     }  // configure_ordinary_destinations
0705
0706     void ThreadSafeLogMessageLoggerScribe::configure_statistics_dest(edm::ParameterSet const& job_pset,
0707                                                                      ConfigurableDefaults const& defaults,
0708                                                                      vString const& categories,
0709                                                                      edm::ParameterSet const& stat_pset,
0710                                                                      std::string const& psetname,
0711                                                                      std::string const& filename) {
0712       auto os_p = m_stream_ps[filename];
0713
0714       auto stat = std::make_shared<ELstatistics>(*os_p);
0715       m_admin_p->attach(stat);
0716       m_statisticsDestControls.push_back(stat);
0717       bool reset = getAparameter<bool>(stat_pset, "resetStatistics", false);
0718       if (not reset) {
0719         //check for old syntax
0720         reset = getAparameter<bool>(stat_pset, "reset", false);
0721       }
0722       m_statisticsResets.push_back(reset);
0723
0724       // now configure this destination:
0725       configure_dest(job_pset, defaults, categories, stat, stat_pset, psetname);
0726
0727       std::string dest_threshold = getAparameter<std::string>(stat_pset, "statisticsThreshold", std::string());
0728       if (not dest_threshold.empty()) {
0729         ELseverityLevel threshold_sev(dest_threshold);
0730         stat->setThreshold(threshold_sev);
0731
0732         setGlobalThresholds(threshold_sev);
0733       }
0734
0735       // and suppress the desire to do an extra termination summary just because
0736       // of end-of-job info messages
0737       stat->noTerminationSummary();
0738     }
0739
0740     void ThreadSafeLogMessageLoggerScribe::configure_statistics(edm::ParameterSet const& job_pset,
0741                                                                 ConfigurableDefaults const& defaults,
0742                                                                 vString const& categories,
0743                                                                 vString const& ordinary_destination_filenames) {
0744       vString const empty_vString;
0745       std::string const empty_String;
0746       edm::ParameterSet const empty_PSet;
0747
0748       // grab list of statistics destinations:
0749       vString statistics = getAparameter<vString>(job_pset, "statistics", empty_vString);
0750
0751       bool no_statistics_configured = statistics.empty();  // change log 24
0752
0753       if (no_statistics_configured) {
0754         // Read the list of staistics destinations from hardwired defaults,
0755         // but only if there is also no list of ordinary destinations.
0756         // (If a cfg specifies destinations, and no statistics, assume that
0757         // is what the user wants.)
0758         vString destinations = getAparameter<vString>(job_pset, "destinations", empty_vString);
0759         if (destinations.empty()) {
0760           statistics = m_messageLoggerDefaults->statistics;
0761           no_statistics_configured = statistics.empty();
0762         } else {
0763           for (auto const& dest : destinations) {
0764             edm::ParameterSet stat_pset = getAparameter<edm::ParameterSet>(job_pset, dest, empty_PSet);
0765             if (getAparameter<bool>(stat_pset, "enableStatistics", false)) {
0766               statistics.push_back(dest);
0767             }
0768           }
0769         }
0770       }
0771
0772       // establish each statistics destination:
0773       for (auto const& psetname : statistics) {
0774         // check that this destination is not just a placeholder // change log 20
0775         edm::ParameterSet stat_pset = getAparameter<edm::ParameterSet>(job_pset, psetname, empty_PSet);
0776         bool is_placeholder = getAparameter<bool>(stat_pset, "placeholder", false);
0777         if (is_placeholder)
0778           continue;
0779
0780         // Determine the destination file name
0781         std::string filename = getAparameter<std::string>(stat_pset, "output", empty_String);
0782         if (filename == empty_String) {
0783           filename = m_messageLoggerDefaults->output(psetname);  // change log 31
0784           if (filename == empty_String) {
0785             filename = psetname;
0786           }
0787         }
0788
0789         // Modify the file name if extension or name is explicitly specified
0790         // change log 14 -- probably suspenders and a belt, because ouput option
0791         // is present, but uniformity is nice.
0792
0793         std::string explicit_filename = getAparameter<std::string>(stat_pset, "filename", filename);
0794         if (explicit_filename != empty_String)
0795           filename = explicit_filename;
0796         std::string explicit_extension = getAparameter<std::string>(stat_pset, "extension", empty_String);
0797         if (explicit_extension != empty_String) {
0798           if (explicit_extension[0] == '.') {
0799             filename += explicit_extension;
0800           } else {
0801             filename = filename + "." + explicit_extension;
0802           }
0803         }
0804
0805         // Attach a default extension of .log if there is no extension on a file
0806         // change log 18 - this had been done in concert with attaching destination
0807
0808         std::string actual_filename = filename;              // change log 4
0809         if ((filename != "cout") && (filename != "cerr")) {  // change log 23
0810           const std::string::size_type npos = std::string::npos;
0811           if (filename.find('.') == npos) {
0812             actual_filename += ".log";
0813           }
0814         }
0815
0816         // Check that this is not a duplicate name -
0817         // unless it is an ordinary destination (which stats can share)
0818         if (!search_all(ordinary_destination_filenames, actual_filename)) {
0819           if (m_stream_ps.find(actual_filename) != m_stream_ps.end()) {
0820             if (m_clean_slate_configuration) {  // change log 22
0821               throw edm::Exception(edm::errors::Configuration)
0822                   << "Duplicate name for a MessageLogger Statistics Destination: " << actual_filename << "\n";
0823             } else {
0824               LogWarning("duplicateDestination")
0825                   << "Duplicate name for a MessageLogger Statistics Destination: " << actual_filename << "\n"
0826                   << "Only original configuration instructions are used";
0827               continue;
0828             }
0829           }
0830         }
0831
0832         // create (if statistics file does not match any destination file name)
0833         // or note (if statistics file matches a destination file name) the ostream.
0834         // But if no statistics destinations were provided in the config, do not
0835         // create a new destination for this hardwired statistics - only act if
0836         // it is matches a destination.  (shange log 24)
0837         bool statistics_destination_is_real = !no_statistics_configured;
0838         std::ostream* os_p;
0839         if (m_stream_ps.find(actual_filename) == m_stream_ps.end()) {
0840           if (actual_filename == "cout") {
0841             os_p = &std::cout;
0842           } else if (actual_filename == "cerr") {
0843             os_p = &std::cerr;
0844           } else {
0845             auto os_sp = std::make_shared<std::ofstream>(actual_filename.c_str());
0846             m_file_ps.push_back(os_sp);
0847             os_p = os_sp.get();
0848           }
0849           m_stream_ps[actual_filename] = os_p;
0850         } else {
0851           statistics_destination_is_real = true;  // change log 24
0852         }
0853
0854         if (statistics_destination_is_real) {  // change log 24
0855                                                // attach the statistics destination, keeping a control handle to it:
0856
0857           configure_statistics_dest(job_pset, defaults, categories, stat_pset, psetname, actual_filename);
0858         }
0859
0860       }  // for [it = statistics.begin() to end()]
0861
0862     }  // configure_statistics
0863
0864     void ThreadSafeLogMessageLoggerScribe::parseCategories(std::string const& s, std::vector<std::string>& cats) {
0865       const std::string::size_type npos = std::string::npos;
0866       std::string::size_type i = 0;
0867       while (i != npos) {
0868         std::string::size_type j = s.find('|', i);
0869         cats.push_back(s.substr(i, j - i));
0870         i = j;
0871         while ((i != npos) && (s[i] == '|'))
0872           ++i;
0873         // the above handles cases of || and also | at end of string
0874       }
0875       // Note:  This algorithm assigns, as desired, one null category if it
0876       //        encounters an empty categories string
0877     }
0878
0879     void ThreadSafeLogMessageLoggerScribe::triggerStatisticsSummaries() {
0880       assert(m_statisticsDestControls.size() == m_statisticsResets.size());
0881       for (unsigned int i = 0; i != m_statisticsDestControls.size(); ++i) {
0882         m_statisticsDestControls[i]->summary(m_tooManyWaitingMessagesCount.load());
0883         if (m_statisticsResets[i])
0884           m_statisticsDestControls[i]->wipe();
0885       }
0886       auto dropped = m_tooManyWaitingMessagesCount.load();
0887       if (m_statisticsDestControls.empty() and dropped != 0) {
0888         if (m_stream_ps.find("cerr") != m_stream_ps.end()) {
0889           std::cerr << "MessageLogger: dropped waiting message count " << dropped << "\n";
0890         }
0891         if (m_stream_ps.find("cout") != m_stream_ps.end()) {
0892           std::cout << "MessageLogger: dropped waiting message count " << dropped << "\n";
0893         }
0894       }
0895     }
0896
0897     void ThreadSafeLogMessageLoggerScribe::triggerFJRmessageSummary(std::map<std::string, double>& sm)  // ChangeLog 29
0898     {
0899       if (m_statisticsDestControls.empty()) {
0900         sm["NoStatisticsDestinationsConfigured"] = 0.0;
0901       } else {
0902         m_statisticsDestControls[0]->summaryForJobReport(sm);
0903       }
0904     }
0905
0906     namespace {
0907       void fillDescriptions(edm::ConfigurationDescriptions& config) {
0908         edm::ParameterSetDescription topDesc;
0909
0910         topDesc.addUntracked<bool>("messageSummaryToJobReport", false);
0911         topDesc.addUntracked<std::string>("generate_preconfiguration_message", "");
0912         topDesc.addUntracked<unsigned int>("waiting_threshold", 100);
0913         topDesc.addUntracked<std::vector<std::string>>("suppressDebug", {});
0914         topDesc.addUntracked<std::vector<std::string>>("suppressInfo", {});
0915         topDesc.addUntracked<std::vector<std::string>>("suppressFwkInfo", {});
0916         topDesc.addUntracked<std::vector<std::string>>("suppressWarning", {});
0917         topDesc.addUntracked<std::vector<std::string>>("suppressError", {});
0918         topDesc.addUntracked<std::vector<std::string>>("debugModules", {})
0919             ->setComment(
0920                 "Set to limit the DEBUG-level messages to modules with these labels. If empty or contains '*', all "
0921                 "DEBUG messages (also those outside modules) will be issued (if allowed by the threshold parameter).");
0922
0923         edm::ParameterSetDescription category;
0924         category.addUntracked<int>("reportEvery", 1);
0925         category.addUntracked<int>("limit", ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults::NO_VALUE_SET)
0926             ->setComment(
0927                 "Set a limit on the number of messages of this category. The default value is used to denote no "
0928                 "limit.");
0929         category.addOptionalUntracked<int>("timespan");
0930
0931         edm::ParameterSetDescription destination_base;
0932         destination_base.addOptionalUntracked<bool>("noLineBreaks");
0933         destination_base.addOptionalUntracked<bool>("noTimeStamps");
0934         destination_base.addOptionalUntracked<int>("lineLength");
0935         destination_base.addOptionalUntracked<std::string>("threshold");
0936         destination_base.addOptionalUntracked<std::string>("statisticsThreshold");
0937
0938         edm::ParameterWildcard<edm::ParameterSetDescription> catnode("*", edm::RequireZeroOrMore, false, category);
0939         catnode.setComment("Specialize either a category or any of 'DEBUG', 'INFO', 'FWKINFO', 'WARNING' or 'ERROR'");
0940         destination_base.addNode(catnode);
0941
0942         edm::ParameterSetDescription destination_noStats(destination_base);
0943         destination_noStats.addUntracked<bool>("enableStatistics", false);
0944         destination_noStats.addUntracked<bool>("resetStatistics", false);
0945
0946         {
0947           edm::ParameterSetDescription default_pset;
0948           default_pset.addUntracked<int>("reportEvery", 1);
0949           default_pset.addUntracked<int>("limit", ThreadSafeLogMessageLoggerScribe::ConfigurableDefaults::NO_VALUE_SET)
0950               ->setComment(
0951                   "Set a limit on the number of messages of this category. The default value is used to denote no "
0952                   "limit.");
0953           default_pset.addOptionalUntracked<int>("timespan");
0954           default_pset.addUntracked<bool>("noLineBreaks", false);
0955           default_pset.addUntracked<bool>("noTimeStamps", false);
0956           default_pset.addUntracked<int>("lineLength", 80);
0957           default_pset.addUntracked<std::string>("threshold", "INFO");
0958           default_pset.addUntracked<std::string>("statisticsThreshold", "INFO");
0959           default_pset.addNode(catnode);
0960
0961           edm::ParameterSetDescription cerr_destination(destination_base);
0962           cerr_destination.addUntracked<bool>("enableStatistics", true);
0963           cerr_destination.addUntracked<bool>("resetStatistics", false);
0964           cerr_destination.addUntracked<bool>("enable", true);
0965
0966           edm::ParameterSetDescription cout_destination(destination_noStats);
0967           cout_destination.addUntracked<bool>("enable", false);
0968
0969           edm::ParameterSetDescription fileDestination(destination_noStats);
0970
0971           fileDestination.addOptionalUntracked<std::string>("output");
0972           fileDestination.addOptionalUntracked<std::string>("filename");
0973           fileDestination.addOptionalUntracked<std::string>("extension");
0974           edm::ParameterSetDescription files;
0975           edm::ParameterWildcard<edm::ParameterSetDescription> fileWildcard(
0976               "*", edm::RequireZeroOrMore, false, fileDestination);
0977           files.addNode(fileWildcard);
0978
0979           std::map<std::string, edm::ParameterSetDescription> standards = {
0980               {"cerr", cerr_destination}, {"cout", cout_destination}, {"default", default_pset}, {"files", files}};
0981
0982           edm::ParameterWildcardWithSpecifics psets("*", edm::RequireZeroOrMore, false, category, std::move(standards));
0983           topDesc.addNode(psets);
0984         }
0985
0986         config.addDefault(topDesc);
0987       }
0988     }  // namespace
0989
0990     void ThreadSafeLogMessageLoggerScribe::validate(edm::ParameterSet& pset) const {
0991       // See if old config API is being used
0992       if (usingOldConfig(pset))
0993         return;
0994       if (not pset.exists("files") and
0995           ((pset.exists("destinations") or pset.existsAs<std::vector<std::string>>("statistics", true) or
0996             pset.existsAs<std::vector<std::string>>("statistics", false) or pset.exists("categories")))) {
0997         return;
0998       }
0999
1000       ConfigurationDescriptions config{"MessageLogger", "MessageLogger"};
1001       fillDescriptions(config);
1002
1003       config.validate(pset, "MessageLogger");
1004     }
1005
1006   }  // end of namespace service
1007 }  // end of namespace edm