special/plugins/HLTLogMonitorFilter.cc

0001 // -*- C++ -*-
0002 //
0003 // Package:    HLTLogMonitorFilter
0004 // Class:      HLTLogMonitorFilter
0005 //
0006 /**\class HLTLogMonitorFilter HLTLogMonitorFilter.cc Work/HLTLogMonitorFilter/src/HLTLogMonitorFilter.cc
0007
0008  Description: Accept events if any LogError or LogWarning was raised
0009
0010  Implementation:
0011      <Notes on implementation>
0012 */
0013 //
0014 // Original Author:  Andrea Bocci
0015 //         Created:  Thu Nov  5 15:16:46 CET 2009
0016 //
0017
0018 // system include files
0019 #include <cstdint>
0020 #include <oneapi/tbb/concurrent_unordered_map.h>
0021 #include <set>
0022
0023 // user include files
0024 #include "FWCore/Framework/interface/Frameworkfwd.h"
0025 #include "FWCore/Framework/interface/Event.h"
0026 #include "FWCore/Framework/interface/global/EDFilter.h"
0027 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0028 #include "FWCore/Utilities/interface/InputTag.h"
0029 #include "DataFormats/Provenance/interface/EventID.h"
0030 #include "DataFormats/Common/interface/ErrorSummaryEntry.h"
0031 #include "FWCore/MessageLogger/interface/LoggedErrorsSummary.h"
0032
0033 //
0034 // class declaration
0035 //
0036
0037 class HLTLogMonitorFilter : public edm::global::EDFilter<> {
0038 public:
0039   explicit HLTLogMonitorFilter(const edm::ParameterSet&);
0040   ~HLTLogMonitorFilter() override;
0041
0042   struct CategoryEntry {
0043     uint32_t
0044         threshold;  // configurable threshold, after which messages in this Category start to be logarithmically prescaled
0045     std::atomic<uint32_t> maxPrescale;  // maximum prescale used for this Category
0046     std::atomic<uint64_t> counter;      // number of events that fired this Category
0047     std::atomic<uint64_t> accepted;     // number of events acepted for this Category
0048     uint32_t id;                        // monotonically increasing unique id for this Category
0049
0050     CategoryEntry(uint32_t iID = 0, uint32_t t = 0)
0051         : threshold(
0052               t),  // default-constructed entries have the threshold set to 0, which means the associated Category is disabled
0053           maxPrescale(1),
0054           counter(0),
0055           accepted(0),
0056           id(iID) {}
0057
0058     CategoryEntry(CategoryEntry const& iOther)
0059         : threshold(iOther.threshold),
0060           maxPrescale(iOther.maxPrescale.load()),
0061           counter(iOther.counter.load()),
0062           accepted(iOther.counter.load()),
0063           id(iOther.id) {}
0064
0065     // caller guarantees this is called just once per event
0066     bool accept() {
0067       auto tCounter = ++counter;
0068
0069       // fail if the prescaler is disabled (threshold == 0), or if the counter is not a multiple of the prescale
0070       if (threshold == 0) {
0071         return false;
0072       }
0073
0074       if (threshold == 1) {
0075         ++accepted;
0076         return true;
0077       }
0078
0079       uint64_t dynPrescale = 1;
0080       // quasi-logarithmic increase in the prescale factor
0081       // dynamically calculating the prescale is mulit-thread stable
0082       while (tCounter > dynPrescale * threshold) {
0083         dynPrescale *= threshold;
0084       }
0085
0086       auto tMaxPrescale = maxPrescale.load();
0087       while (tMaxPrescale < dynPrescale) {
0088         maxPrescale.compare_exchange_strong(tMaxPrescale, dynPrescale);
0089       }
0090
0091       if (0 != tCounter % dynPrescale) {
0092         return false;
0093       }
0094
0095       ++accepted;
0096       return true;
0097     }
0098   };
0099
0100   typedef tbb::concurrent_unordered_map<std::string, CategoryEntry> CategoryMap;
0101
0102   // ---------- private methods -----------------------
0103
0104   /// EDFilter accept method
0105   bool filter(edm::StreamID, edm::Event&, const edm::EventSetup&) const override;
0106
0107   /// EDFilter beginJob method
0108   void beginJob() override;
0109
0110   /// EDFilter endJob method
0111   void endJob() override;
0112
0113   /// create a new entry for the given category, with the given threshold value
0114   CategoryEntry& addCategory(const std::string& category, uint32_t threshold);
0115
0116   /// return the entry for requested category, if it exists, or create a new one with the default threshold value
0117   CategoryEntry& getCategory(const std::string& category) const;
0118
0119   /// summarize to LogInfo
0120   void summary() const;
0121
0122   // ---------- member data ---------------------------
0123   uint32_t m_prescale;  // default threshold, after which messages in each Category start to be logarithmically prescaled
0124   mutable std::atomic<uint32_t> m_nextEntryID = 0;
0125   CMS_THREAD_SAFE mutable CategoryMap m_data;  // map each category name to its prescale data
0126   edm::EDPutTokenT<std::vector<edm::ErrorSummaryEntry>> m_putToken;
0127 };
0128
0129 // system include files
0130 #include <sstream>
0131 #include <iomanip>
0132 #include <memory>
0133 #include <boost/range.hpp>
0134 #include <boost/algorithm/string.hpp>
0135
0136 // user include files
0137 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0138 #include "FWCore/MessageLogger/interface/MessageSender.h"
0139
0140 //
0141 // constructors and destructor
0142 //
0143 HLTLogMonitorFilter::HLTLogMonitorFilter(const edm::ParameterSet& config) : m_prescale(), m_data() {
0144   m_prescale = config.getParameter<uint32_t>("default_threshold");
0145
0146   typedef std::vector<edm::ParameterSet> VPSet;
0147   const VPSet& categories = config.getParameter<VPSet>("categories");
0148   for (auto const& categorie : categories) {
0149     const std::string& name = categorie.getParameter<std::string>("name");
0150     uint32_t threshold = categorie.getParameter<uint32_t>("threshold");
0151     addCategory(name, threshold);
0152   }
0153
0154   m_putToken = produces<std::vector<edm::ErrorSummaryEntry>>();
0155 }
0156
0157 HLTLogMonitorFilter::~HLTLogMonitorFilter() = default;
0158
0159 //
0160 // member functions
0161 //
0162
0163 // ------------ method called on each new Event  ------------
0164 bool HLTLogMonitorFilter::filter(edm::StreamID, edm::Event& event, const edm::EventSetup& setup) const {
0165   // no LogErrors or LogWarnings, skip processing and reject the event
0166   if (not edm::FreshErrorsExist(event.streamID().value()))
0167     return false;
0168
0169   //keep track of which errors have already been seen this event
0170   struct Cache {
0171     Cache() : done(false), cachedValue(false) {}
0172     bool done;
0173     bool cachedValue;
0174   };
0175   std::vector<Cache> doneCache(m_nextEntryID);
0176
0177   // look at the LogErrors and LogWarnings, and accept the event if at least one is under threshold or matches the prescale
0178   bool accept = false;
0179   std::string category;
0180
0181   std::vector<edm::messagelogger::ErrorSummaryEntry> errorSummary{edm::LoggedErrorsSummary(event.streamID().value())};
0182   for (auto const& entry : errorSummary) {
0183     // split the message category
0184     typedef boost::split_iterator<std::string::const_iterator> splitter;
0185     for (splitter i = boost::make_split_iterator(entry.category, boost::first_finder("|", boost::is_equal()));
0186          i != splitter();
0187          ++i) {
0188       // extract the substring corresponding to the split_iterator
0189       // FIXME: this can be avoided if the m_data map is keyed on boost::sub_range<std::string>
0190       category.assign(i->begin(), i->end());
0191
0192       // access the message category, or create a new one as needed, and check the prescale
0193       auto& cat = getCategory(category);
0194       if (cat.id >= doneCache.size()) {
0195         //new categories were added so need to grow
0196         doneCache.resize(cat.id + 1);
0197       }
0198       if (not doneCache[cat.id].done) {
0199         doneCache[cat.id].cachedValue = cat.accept();
0200         doneCache[cat.id].done = true;
0201       }
0202       if (doneCache[cat.id].cachedValue)
0203         accept = true;
0204     }
0205   }
0206
0207   // harvest the errors, but only if the filter will accept the event
0208   std::vector<edm::ErrorSummaryEntry> errors;
0209   if (accept) {
0210     errors.reserve(errorSummary.size());
0211     std::transform(errorSummary.begin(), errorSummary.end(), std::back_inserter(errors), [](auto& iEntry) {
0212       edm::ErrorSummaryEntry entry;
0213       entry.category = std::move(iEntry.category);
0214       entry.module = std::move(iEntry.module);
0215       entry.severity = edm::ELseverityLevel(iEntry.severity.getLevel());
0216       entry.count = iEntry.count;
0217       return entry;
0218     });
0219   }
0220   event.emplace(m_putToken, std::move(errors));
0221
0222   return accept;
0223 }
0224
0225 // ------------ method called at the end of the Job ---------
0226 void HLTLogMonitorFilter::beginJob() { edm::EnableLoggedErrorsSummary(); }
0227 // ------------ method called at the end of the Job ---------
0228 void HLTLogMonitorFilter::endJob() {
0229   edm::DisableLoggedErrorsSummary();
0230   summary();
0231 }
0232
0233 /// create a new entry for the given category, with the given threshold value
0234 HLTLogMonitorFilter::CategoryEntry& HLTLogMonitorFilter::addCategory(const std::string& category, uint32_t threshold) {
0235   // check after inserting, as either the new CategoryEntry is needed, or an error condition is raised
0236   auto id = m_nextEntryID++;
0237   std::pair<CategoryMap::iterator, bool> result = m_data.insert(std::make_pair(category, CategoryEntry(id, threshold)));
0238   if (not result.second)
0239     throw cms::Exception("Configuration") << "Duplicate entry for category " << category;
0240   return result.first->second;
0241 }
0242
0243 /// return the entry for requested category, if it exists, or create a new one with the default threshold value
0244 HLTLogMonitorFilter::CategoryEntry& HLTLogMonitorFilter::getCategory(const std::string& category) const {
0245   // check before inserting, to avoid the construction of a CategoryEntry object
0246   auto i = m_data.find(category);
0247   if (i != m_data.end())
0248     return i->second;
0249   else {
0250     auto id = m_nextEntryID++;
0251     return m_data.insert(std::make_pair(category, CategoryEntry(id, m_prescale))).first->second;
0252   }
0253 }
0254
0255 /// summarize to LogInfo
0256 void HLTLogMonitorFilter::summary() const {
0257   std::stringstream out;
0258   out << "Log-Report ---------- HLTLogMonitorFilter Summary ------------\n"
0259       << "Log-Report  Threshold   Prescale     Issued   Accepted   Rejected Category\n";
0260
0261   std::set<std::string> sortedCategories;
0262   for (auto const& entry : m_data) {
0263     sortedCategories.insert(entry.first);
0264   }
0265
0266   for (auto const& cat : sortedCategories) {
0267     auto entry = m_data.find(cat);
0268     out << "Log-Report " << std::right << std::setw(10) << entry->second.threshold << ' ' << std::setw(10)
0269         << entry->second.maxPrescale << ' ' << std::setw(10) << entry->second.counter << ' ' << std::setw(10)
0270         << entry->second.accepted << ' ' << std::setw(10) << (entry->second.counter - entry->second.accepted) << ' '
0271         << std::left << cat << '\n';
0272   }
0273   edm::LogVerbatim("Report") << out.str();
0274 }
0275
0276 // define as a framework plugin
0277 #include "FWCore/Framework/interface/MakerMacros.h"
0278 DEFINE_FWK_MODULE(HLTLogMonitorFilter);