Back to home page

Project CMSSW displayed by LXR
 
 

    

File indexing completed on 2024-04-06 12:10:08 

0001 /*
0002  * \file QualityTester.cc
0003  *
0004  * Harvesting module that applies Quality Tests to MonitorElements.
0005  * Which tests are run an be configured using an XML-based configuration.
0006  * \author Marcel Schneider 
0007  * based on M. Zanetti's older module.
0008  *
0009  */
0010 
0011 #include "FWCore/Framework/interface/EventSetup.h"
0012 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0013 #include "FWCore/ServiceRegistry/interface/Service.h"
0014 #include "FWCore/ParameterSet/interface/FileInPath.h"
0015 #include "FWCore/Framework/interface/Event.h"
0016 #include "FWCore/Framework/interface/MakerMacros.h"
0017 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0018 #include "FWCore/Framework/interface/LuminosityBlock.h"
0019 
0020 #include "DQMServices/Core/interface/DQMEDHarvester.h"
0021 #include "DQMServices/Core/interface/QTest.h"
0022 
0023 #include <cmath>
0024 #include <memory>
0025 #include <string>
0026 #include <cstdio>
0027 #include <sstream>
0028 #include <iostream>
0029 
0030 #include <boost/property_tree/xml_parser.hpp>
0031 #include <boost/property_tree/ptree.hpp>
0032 
0033 #include <fnmatch.h>
0034 
0035 class QualityTester : public DQMEDHarvester {
0036 public:
0037   /// Constructor
0038   QualityTester(const edm::ParameterSet& ps);
0039 
0040   /// Destructor
0041   ~QualityTester() override;
0042 
0043 protected:
0044   /// perform the actual quality tests
0045   void dqmAnalyze(DQMStore::IBooker&, DQMStore::IGetter&, const edm::Event& e, const edm::EventSetup& c) override;
0046   void dqmEndLuminosityBlock(DQMStore::IBooker&,
0047                              DQMStore::IGetter&,
0048                              edm::LuminosityBlock const& lumiSeg,
0049                              edm::EventSetup const& c) override;
0050   void dqmEndRun(DQMStore::IBooker&, DQMStore::IGetter&, const edm::Run& r, const edm::EventSetup& c) override;
0051   void dqmEndJob(DQMStore::IBooker&, DQMStore::IGetter&) override;
0052 
0053 private:
0054   void performTests(DQMStore::IGetter& igetter);
0055 
0056   int nEvents;
0057   int prescaleFactor;
0058   bool getQualityTestsFromFile;
0059   std::string Label;
0060   bool testInEventloop;
0061   bool qtestOnEndRun;
0062   bool qtestOnEndJob;
0063   bool qtestOnEndLumi;
0064   std::string reportThreshold;
0065   bool verboseQT;
0066 
0067   // this vector holds and owns all the QTest objects. mostly for memory management.
0068   std::vector<std::unique_ptr<QCriterion>> qtestobjects;
0069   // we use this structure to check which tests to run on which MEs. The first
0070   // part is the pattern with wildcards matching MEs that the test should be
0071   // applied to, the second points to an object in qtestobjects. We cannot own
0072   // the objects here since more than one pattern can use the same test.
0073   std::vector<std::pair<std::string, QCriterion*>> qtestpatterns;
0074 
0075   void configureTests(std::string const& file);
0076   std::unique_ptr<QCriterion> makeQCriterion(boost::property_tree::ptree const& config);
0077 };
0078 
0079 QualityTester::QualityTester(const edm::ParameterSet& ps) : DQMEDHarvester(ps) {
0080   prescaleFactor = ps.getUntrackedParameter<int>("prescaleFactor", 1);
0081   getQualityTestsFromFile = ps.getUntrackedParameter<bool>("getQualityTestsFromFile", true);
0082   Label = ps.getUntrackedParameter<std::string>("label", "");
0083   reportThreshold = ps.getUntrackedParameter<std::string>("reportThreshold", "");
0084   testInEventloop = ps.getUntrackedParameter<bool>("testInEventloop", false);
0085   qtestOnEndRun = ps.getUntrackedParameter<bool>("qtestOnEndRun", true);
0086   qtestOnEndJob = ps.getUntrackedParameter<bool>("qtestOnEndJob", false);
0087   qtestOnEndLumi = ps.getUntrackedParameter<bool>("qtestOnEndLumi", false);
0088   verboseQT = ps.getUntrackedParameter<bool>("verboseQT", true);
0089 
0090   if (getQualityTestsFromFile) {
0091     edm::FileInPath qtlist = ps.getUntrackedParameter<edm::FileInPath>("qtList");
0092     configureTests(qtlist.fullPath());
0093   } else {
0094     assert(!"Reading from DB no longer supported.");
0095   }
0096 
0097   nEvents = 0;
0098 }
0099 
0100 QualityTester::~QualityTester() {}
0101 
0102 void QualityTester::dqmAnalyze(DQMStore::IBooker&,
0103                                DQMStore::IGetter& igetter,
0104                                const edm::Event& e,
0105                                const edm::EventSetup& c) {
0106   if (testInEventloop) {
0107     nEvents++;
0108     if (prescaleFactor > 0 && nEvents % prescaleFactor == 0) {
0109       performTests(igetter);
0110     }
0111   }
0112 }
0113 
0114 void QualityTester::dqmEndLuminosityBlock(DQMStore::IBooker&,
0115                                           DQMStore::IGetter& igetter,
0116                                           edm::LuminosityBlock const& lumiSeg,
0117                                           edm::EventSetup const& context) {
0118   if (!testInEventloop && qtestOnEndLumi) {
0119     if (prescaleFactor > 0 && lumiSeg.id().luminosityBlock() % prescaleFactor == 0) {
0120       performTests(igetter);
0121     }
0122   }
0123 }
0124 
0125 void QualityTester::dqmEndRun(DQMStore::IBooker&,
0126                               DQMStore::IGetter& igetter,
0127                               edm::Run const& run,
0128                               edm::EventSetup const& context) {
0129   if (qtestOnEndRun)
0130     performTests(igetter);
0131 }
0132 
0133 void QualityTester::dqmEndJob(DQMStore::IBooker&, DQMStore::IGetter& igetter) {
0134   if (qtestOnEndJob)
0135     performTests(igetter);
0136 }
0137 
0138 void QualityTester::performTests(DQMStore::IGetter& igetter) {
0139   edm::LogVerbatim("QualityTester") << "Running the Quality Test";
0140 
0141   auto mes = igetter.getAllContents("");
0142 
0143   for (auto me : mes) {
0144     std::string name = me->getFullname();
0145     for (auto& kv : this->qtestpatterns) {
0146       std::string& pattern = kv.first;
0147       QCriterion* qtest = kv.second;
0148       int match = fnmatch(pattern.c_str(), name.c_str(), 0);
0149       if (match == FNM_NOMATCH)
0150         continue;
0151       if (match != 0)
0152         throw cms::Exception("QualityTester")
0153             << "Something went wrong with fnmatch: pattern = '" << pattern << "' and string = '" << name << "'";
0154 
0155       // name matched, apply test.
0156       // Using the classic ME API for now.
0157       QReport* qr;
0158       DQMNet::QValue* qv;
0159       me->getQReport(/* create */ true, qtest->getName(), qr, qv);
0160       assert(qtest);  // null might be valid, maybe replace with if
0161       assert(qr);
0162       assert(qv);
0163       qtest->runTest(me, *qr, *qv);
0164       // this propagates the result into the DQMNet object flags
0165       me->syncCoreObject();
0166     }
0167   }
0168 
0169   if (!reportThreshold.empty()) {
0170     // map {red, orange, black} -> [QReport message, ...]
0171     std::map<std::string, std::vector<std::string>> theAlarms;
0172     // populate from MEs hasError, hasWarning, hasOther
0173     for (auto me : mes) {
0174       // TODO: This logic is rather broken and suppresses errors when there
0175       // are warnings (or suppresses both when there are others. But this is
0176       // how it always was, and we keep it for now for compatibility.
0177       std::vector<QReport*> report;
0178       std::string colour;
0179       if (me->hasError()) {
0180         colour = "red";
0181         report = me->getQErrors();
0182       }
0183       if (me->hasWarning()) {
0184         colour = "orange";
0185         report = me->getQWarnings();
0186       }
0187       if (me->hasOtherReport()) {
0188         colour = "black";
0189         report = me->getQOthers();
0190       }
0191       for (auto r : report) {
0192         theAlarms[colour].push_back(r->getMessage());
0193       }
0194     }
0195 
0196     // writes to stdout, because it alyways wrote to stdout.
0197     for (auto& theAlarm : theAlarms) {
0198       const std::string& alarmType = theAlarm.first;
0199       const std::vector<std::string>& msgs = theAlarm.second;
0200       if ((reportThreshold == "black") ||
0201           (reportThreshold == "orange" && (alarmType == "orange" || alarmType == "red")) ||
0202           (reportThreshold == "red" && alarmType == "red")) {
0203         std::cout << std::endl;
0204         std::cout << "Error Type: " << alarmType << std::endl;
0205         for (auto const& msg : msgs)
0206           std::cout << msg << std::endl;
0207       }
0208     }
0209     std::cout << std::endl;
0210   }
0211 }
0212 
0213 std::unique_ptr<QCriterion> QualityTester::makeQCriterion(boost::property_tree::ptree const& config) {
0214   // For whatever reasons the compiler needs the "template" keyword on ptree::get.
0215   // To save some noise in the code we have the preprocessor add it everywhere.
0216 #define get template get
0217   // TODO: make the parameter names more consistent.
0218   // TODO: add defaults where it makes sense.
0219   std::map<std::string,
0220            std::function<std::unique_ptr<QCriterion>(boost::property_tree::ptree const&, std::string& name)>>
0221       qtestmakers = {{CheckVariance::getAlgoName(),
0222                       [](auto const& config, std::string& name) {
0223                         auto test = std::make_unique<CheckVariance>(name);
0224                         test->setWarningProb(config.get<float>("warning", QCriterion::WARNING_PROB_THRESHOLD));
0225                         test->setErrorProb(config.get<float>("error", QCriterion::ERROR_PROB_THRESHOLD));
0226                         return test;
0227                       }},
0228                      {CompareLastFilledBin::getAlgoName(),
0229                       [](auto const& config, std::string& name) {
0230                         auto test = std::make_unique<CompareLastFilledBin>(name);
0231                         test->setWarningProb(config.get<float>("warning", QCriterion::WARNING_PROB_THRESHOLD));
0232                         test->setErrorProb(config.get<float>("error", QCriterion::ERROR_PROB_THRESHOLD));
0233                         test->setAverage(config.get<float>("AvVal"));
0234                         test->setMin(config.get<float>("MinVal"));
0235                         test->setMax(config.get<float>("MaxVal"));
0236                         return test;
0237                       }},
0238                      {CompareToMedian::getAlgoName(),
0239                       [](auto const& config, std::string& name) {
0240                         auto test = std::make_unique<CompareToMedian>(name);
0241                         test->setWarningProb(config.get<float>("warning", QCriterion::WARNING_PROB_THRESHOLD));
0242                         test->setErrorProb(config.get<float>("error", QCriterion::ERROR_PROB_THRESHOLD));
0243                         test->setMin(config.get<float>("MinRel"));
0244                         test->setMax(config.get<float>("MaxRel"));
0245                         test->setMinMedian(config.get<float>("MinAbs"));
0246                         test->setMaxMedian(config.get<float>("MaxAbs"));
0247                         test->setEmptyBins(config.get<int>("UseEmptyBins"));
0248                         test->setStatCut(config.get<float>("StatCut", 0));
0249                         return test;
0250                       }},
0251                      {ContentSigma::getAlgoName(),
0252                       [](auto const& config, std::string& name) {
0253                         auto test = std::make_unique<ContentSigma>(name);
0254                         test->setWarningProb(config.get<float>("warning", QCriterion::WARNING_PROB_THRESHOLD));
0255                         test->setErrorProb(config.get<float>("error", QCriterion::ERROR_PROB_THRESHOLD));
0256                         test->setNumXblocks(config.get<int>("Xblocks"));
0257                         test->setNumYblocks(config.get<int>("Yblocks"));
0258                         test->setNumNeighborsX(config.get<int>("neighboursX"));
0259                         test->setNumNeighborsY(config.get<int>("neighboursY"));
0260                         test->setToleranceNoisy(config.get<double>("toleranceNoisy"));
0261                         test->setToleranceDead(config.get<double>("toleranceDead"));
0262                         test->setNoisy(config.get<double>("noisy"));
0263                         test->setDead(config.get<double>("dead"));
0264                         test->setXMin(config.get<int>("xMin"));
0265                         test->setXMax(config.get<int>("xMax"));
0266                         test->setYMin(config.get<int>("yMin"));
0267                         test->setYMax(config.get<int>("yMax"));
0268                         return test;
0269                       }},
0270                      {ContentsWithinExpected::getAlgoName(),
0271                       [](auto const& config, std::string& name) {
0272                         auto test = std::make_unique<ContentsWithinExpected>(name);
0273                         test->setWarningProb(config.get<float>("warning", QCriterion::WARNING_PROB_THRESHOLD));
0274                         test->setErrorProb(config.get<float>("error", QCriterion::ERROR_PROB_THRESHOLD));
0275                         test->setUseEmptyBins(config.get<bool>("useEmptyBins"));
0276                         test->setMinimumEntries(config.get<int>("minEntries"));
0277                         test->setMeanTolerance(config.get<float>("toleranceMean"));
0278                         if (config.get<double>("minMean", 0) || config.get<double>("maxMean", 0))
0279                           test->setMeanRange(config.get<double>("minMean"), config.get<double>("maxMean"));
0280                         if (config.get<double>("minRMS", 0) || config.get<double>("maxRMS", 0))
0281                           test->setRMSRange(config.get<double>("minRMS"), config.get<double>("maxRMS"));
0282                         return test;
0283                       }},
0284                      {ContentsXRange::getAlgoName(),
0285                       [](auto const& config, std::string& name) {
0286                         auto test = std::make_unique<ContentsXRange>(name);
0287                         test->setWarningProb(config.get<float>("warning", QCriterion::WARNING_PROB_THRESHOLD));
0288                         test->setErrorProb(config.get<float>("error", QCriterion::ERROR_PROB_THRESHOLD));
0289                         test->setAllowedXRange(config.get<double>("xmin"), config.get<double>("xmax"));
0290                         return test;
0291                       }},
0292                      {ContentsYRange::getAlgoName(),
0293                       [](auto const& config, std::string& name) {
0294                         auto test = std::make_unique<ContentsYRange>(name);
0295                         test->setWarningProb(config.get<float>("warning", QCriterion::WARNING_PROB_THRESHOLD));
0296                         test->setErrorProb(config.get<float>("error", QCriterion::ERROR_PROB_THRESHOLD));
0297                         test->setAllowedYRange(config.get<double>("ymin"), config.get<double>("ymax"));
0298                         test->setUseEmptyBins(config.get<bool>("useEmptyBins"));
0299                         return test;
0300                       }},
0301                      {DeadChannel::getAlgoName(),
0302                       [](auto const& config, std::string& name) {
0303                         auto test = std::make_unique<DeadChannel>(name);
0304                         test->setWarningProb(config.get<float>("warning", QCriterion::WARNING_PROB_THRESHOLD));
0305                         test->setErrorProb(config.get<float>("error", QCriterion::ERROR_PROB_THRESHOLD));
0306                         test->setThreshold(config.get<float>("threshold"));
0307                         return test;
0308                       }},
0309                      {MeanWithinExpected::getAlgoName(),
0310                       [](auto const& config, std::string& name) {
0311                         auto test = std::make_unique<MeanWithinExpected>(name);
0312                         test->setWarningProb(config.get<float>("warning", QCriterion::WARNING_PROB_THRESHOLD));
0313                         test->setErrorProb(config.get<float>("error", QCriterion::ERROR_PROB_THRESHOLD));
0314                         test->setMinimumEntries(config.get<int>("minEntries", 0));
0315                         test->setExpectedMean(config.get<double>("mean"));
0316                         if (config.get<double>("useSigma", 0))
0317                           test->useSigma(config.get<double>("useSigma"));
0318                         if (config.get<int>("useRMS", 0))
0319                           test->useRMS();
0320                         if (config.get<int>("useRange", 0))
0321                           test->useRange(config.get<float>("xmin"), config.get<float>("xmax"));
0322                         return test;
0323                       }},
0324                      {NoisyChannel::getAlgoName(), [](auto const& config, std::string& name) {
0325                         auto test = std::make_unique<NoisyChannel>(name);
0326                         test->setWarningProb(config.get<float>("warning", QCriterion::WARNING_PROB_THRESHOLD));
0327                         test->setErrorProb(config.get<float>("error", QCriterion::ERROR_PROB_THRESHOLD));
0328                         test->setTolerance(config.get<double>("tolerance"));
0329                         test->setNumNeighbors(config.get<double>("neighbours"));
0330                         return test;
0331                       }}};
0332 #undef get
0333 
0334   auto maker = qtestmakers.find(config.get<std::string>("TYPE"));
0335   // Check if the type is known, error out otherwise.
0336   if (maker == qtestmakers.end())
0337     return nullptr;
0338 
0339   // The QTest XML format has structure
0340   // <QTEST><TYPE>QTestClass</TYPE><PARAM name="thing">value</PARAM>...</QTEST>
0341   // but that is a pain to read with property_tree. So we reorder the structure
0342   // and add a child "thing" with data "value" for each param to a new tree.
0343   // Then the qtestmakers can just config.get<type>("thing").
0344   boost::property_tree::ptree reordered;
0345   for (auto kv : config) {
0346     // TODO: case sensitive?
0347     if (kv.first == "PARAM") {
0348       reordered.put(kv.second.get<std::string>("<xmlattr>.name"), kv.second.data());
0349     }
0350   }
0351 
0352   auto name = config.get<std::string>("<xmlattr>.name");
0353   return maker->second(reordered, name);
0354 }
0355 
0356 void QualityTester::configureTests(std::string const& file) {
0357   // We read QTests from the config file into this structure, before
0358   // transforming into the final [(pattern, *qtest), ...] structure.
0359   struct TestItem {
0360     std::unique_ptr<QCriterion> qtest;
0361     std::vector<std::string> pathpatterns;
0362   };
0363   std::map<std::string, TestItem> qtestmap;
0364 
0365   // read XML using property tree. Should throw on error.
0366   boost::property_tree::ptree xml;
0367   boost::property_tree::read_xml(file, xml);
0368 
0369   auto it = xml.find("TESTSCONFIGURATION");
0370   if (it == xml.not_found()) {
0371     throw cms::Exception("QualityTester") << "QTest XML needs to have a TESTSCONFIGURATION node.";
0372   }
0373   auto& testconfig = it->second;
0374   for (auto& kv : testconfig) {
0375     // QTEST describes a QTest object (actually QCriterion) with its parameters
0376     if (kv.first == "QTEST") {
0377       auto& qtestconfig = kv.second;
0378       auto name = qtestconfig.get<std::string>("<xmlattr>.name");
0379       auto value = makeQCriterion(qtestconfig);
0380       // LINK and QTEST can be in any order, so this element may or may not exist
0381       qtestmap[name].qtest = std::move(value);
0382     }  // else
0383     // LINK associates one ore more QTest referenced by name to a ME path pattern.
0384     if (kv.first == "LINK") {
0385       auto& linkconfig = kv.second;
0386       auto pathpattern = linkconfig.get<std::string>("<xmlattr>.name");
0387       for (auto& subkv : linkconfig) {
0388         if (subkv.first == "TestName") {
0389           std::string testname = subkv.second.data();
0390           bool enabled = subkv.second.get<bool>("<xmlattr>.activate");
0391           if (enabled) {
0392             // LINK and QTEST can be in any order, so this element may or may not exist
0393             qtestmap[testname].pathpatterns.push_back(pathpattern);
0394           }
0395         }
0396       }
0397     }
0398     // else: unknown tag, but that is fine, its XML
0399   }
0400 
0401   // now that we have all the objects created and references resolved, flatten
0402   // the structure into something more useful for evaluating tests.
0403   this->qtestobjects.clear();
0404   this->qtestpatterns.clear();
0405   for (auto& kv : qtestmap) {
0406     QCriterion* bareptr = kv.second.qtest.get();
0407     for (auto& p : kv.second.pathpatterns) {
0408       this->qtestpatterns.push_back(std::make_pair(p, bareptr));
0409     }
0410     this->qtestobjects.push_back(std::move(kv.second.qtest));
0411   }
0412   // we could sort the patterns here to allow more performant matching
0413   // (using a suffix-array to handle the "*" in the beginning)
0414 }
0415 
0416 DEFINE_FWK_MODULE(QualityTester);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.2.1 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!