File indexing completed on 2022-06-22 01:41:24
0001
0002
0003
0004
0005
0006
0007 #include <atomic>
0008 #include <fstream>
0009 #include <sstream>
0010 #include <vector>
0011 #include <string>
0012 #include <memory>
0013 #include <algorithm>
0014
0015 #include <fmt/printf.h>
0016
0017 #include "DataFormats/Common/interface/Handle.h"
0018 #include "DataFormats/Common/interface/TriggerResults.h"
0019 #include "EventFilter/Utilities/interface/EvFDaqDirector.h"
0020 #include "EventFilter/Utilities/interface/FastMonitor.h"
0021 #include "EventFilter/Utilities/interface/FastMonitoringService.h"
0022 #include "EventFilter/Utilities/interface/JSONSerializer.h"
0023 #include "EventFilter/Utilities/interface/JsonMonitorable.h"
0024 #include "FWCore/Framework/interface/ESHandle.h"
0025 #include "FWCore/Framework/interface/Event.h"
0026 #include "FWCore/Framework/interface/LuminosityBlock.h"
0027 #include "FWCore/Framework/interface/Run.h"
0028 #include "FWCore/Framework/interface/global/EDAnalyzer.h"
0029 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0030 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0031 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0032 #include "FWCore/ServiceRegistry/interface/Service.h"
0033 #include "FWCore/Utilities/interface/Adler32Calculator.h"
0034 #include "HLTrigger/HLTcore/interface/HLTConfigProvider.h"
0035
0036 struct HLTriggerJSONMonitoringData {
0037
0038 struct stream {
0039 unsigned int processed;
0040 std::vector<unsigned int> hltWasRun;
0041 std::vector<unsigned int> hltL1s;
0042 std::vector<unsigned int> hltPre;
0043 std::vector<unsigned int> hltAccept;
0044 std::vector<unsigned int> hltReject;
0045 std::vector<unsigned int> hltErrors;
0046 std::vector<unsigned int> datasets;
0047 };
0048
0049
0050 struct run {
0051 std::string streamDestination;
0052 std::string streamMergeType;
0053 std::string baseRunDir;
0054 std::string jsdFileName;
0055
0056 HLTConfigProvider hltConfig;
0057 std::vector<int> posL1s;
0058 std::vector<int> posPre;
0059 std::vector<std::vector<unsigned int>> datasets;
0060 std::vector<unsigned int> indicesOfTriggerPaths;
0061 };
0062
0063
0064 struct lumisection {
0065 jsoncollector::HistoJ<unsigned int> processed;
0066 jsoncollector::HistoJ<unsigned int> hltWasRun;
0067 jsoncollector::HistoJ<unsigned int> hltL1s;
0068 jsoncollector::HistoJ<unsigned int> hltPre;
0069 jsoncollector::HistoJ<unsigned int> hltAccept;
0070 jsoncollector::HistoJ<unsigned int> hltReject;
0071 jsoncollector::HistoJ<unsigned int> hltErrors;
0072 jsoncollector::HistoJ<unsigned int> datasets;
0073 };
0074 };
0075
0076 class HLTriggerJSONMonitoring : public edm::global::EDAnalyzer<
0077
0078 edm::StreamCache<HLTriggerJSONMonitoringData::stream>,
0079
0080 edm::RunCache<HLTriggerJSONMonitoringData::run>,
0081
0082 edm::LuminosityBlockSummaryCache<HLTriggerJSONMonitoringData::lumisection>> {
0083 public:
0084
0085 explicit HLTriggerJSONMonitoring(const edm::ParameterSet&);
0086
0087
0088 ~HLTriggerJSONMonitoring() override = default;
0089
0090
0091 static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0092
0093
0094 void analyze(edm::StreamID, edm::Event const&, edm::EventSetup const&) const override;
0095
0096
0097
0098
0099 std::unique_ptr<HLTriggerJSONMonitoringData::stream> beginStream(edm::StreamID) const override;
0100
0101
0102 void streamBeginLuminosityBlock(edm::StreamID, edm::LuminosityBlock const&, edm::EventSetup const&) const override;
0103
0104
0105
0106
0107 std::shared_ptr<HLTriggerJSONMonitoringData::run> globalBeginRun(edm::Run const&,
0108 edm::EventSetup const&) const override;
0109
0110
0111 void globalEndRun(edm::Run const&, edm::EventSetup const&) const override;
0112
0113
0114
0115
0116 std::shared_ptr<HLTriggerJSONMonitoringData::lumisection> globalBeginLuminosityBlockSummary(
0117 edm::LuminosityBlock const&, edm::EventSetup const&) const override;
0118
0119
0120 void streamEndLuminosityBlockSummary(edm::StreamID,
0121 edm::LuminosityBlock const&,
0122 edm::EventSetup const&,
0123 HLTriggerJSONMonitoringData::lumisection*) const override;
0124
0125
0126 void globalEndLuminosityBlockSummary(edm::LuminosityBlock const&,
0127 edm::EventSetup const&,
0128 HLTriggerJSONMonitoringData::lumisection*) const override;
0129
0130 private:
0131 static constexpr const char* streamName_ = "streamHLTRates";
0132
0133 static constexpr const char* datasetPathNamePrefix_ = "Dataset_";
0134
0135 static void writeJsdFile(HLTriggerJSONMonitoringData::run const&);
0136 static void writeIniFile(HLTriggerJSONMonitoringData::run const&, unsigned int);
0137
0138
0139 const edm::InputTag triggerResults_;
0140 const edm::EDGetTokenT<edm::TriggerResults> triggerResultsToken_;
0141 };
0142
0143
0144 HLTriggerJSONMonitoring::HLTriggerJSONMonitoring(edm::ParameterSet const& config)
0145 : triggerResults_(config.getParameter<edm::InputTag>("triggerResults")),
0146 triggerResultsToken_(consumes(triggerResults_)) {}
0147
0148
0149 void HLTriggerJSONMonitoring::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0150 edm::ParameterSetDescription desc;
0151 desc.add<edm::InputTag>("triggerResults", edm::InputTag("TriggerResults", "", "@currentProcess"));
0152 descriptions.add("HLTriggerJSONMonitoring", desc);
0153 }
0154
0155
0156 std::unique_ptr<HLTriggerJSONMonitoringData::stream> HLTriggerJSONMonitoring::beginStream(edm::StreamID) const {
0157 return std::make_unique<HLTriggerJSONMonitoringData::stream>();
0158 }
0159
0160
0161 std::shared_ptr<HLTriggerJSONMonitoringData::run> HLTriggerJSONMonitoring::globalBeginRun(
0162 edm::Run const& run, edm::EventSetup const& setup) const {
0163 auto rundata = std::make_shared<HLTriggerJSONMonitoringData::run>();
0164
0165
0166 if (edm::Service<evf::EvFDaqDirector>().isAvailable()) {
0167 rundata->streamDestination = edm::Service<evf::EvFDaqDirector>()->getStreamDestinations(streamName_);
0168 rundata->streamMergeType =
0169 edm::Service<evf::EvFDaqDirector>()->getStreamMergeType(streamName_, evf::MergeTypeJSNDATA);
0170 rundata->baseRunDir = edm::Service<evf::EvFDaqDirector>()->baseRunDir();
0171 } else {
0172 rundata->streamDestination = "";
0173 rundata->streamMergeType = "";
0174 rundata->baseRunDir = ".";
0175 }
0176
0177
0178 bool changed = true;
0179 if (not rundata->hltConfig.init(run, setup, triggerResults_.process(), changed)) {
0180 edm::LogError("HLTriggerJSONMonitoring") << "HLTConfigProvider initialization failed!";
0181 } else if (changed) {
0182
0183 auto const& triggerNames = rundata->hltConfig.triggerNames();
0184 auto const triggerNamesSize = triggerNames.size();
0185
0186
0187 rundata->indicesOfTriggerPaths.clear();
0188 rundata->indicesOfTriggerPaths.reserve(triggerNamesSize);
0189 for (auto triggerNameIdx = 0u; triggerNameIdx < triggerNamesSize; ++triggerNameIdx) {
0190
0191 if (triggerNames[triggerNameIdx].find(datasetPathNamePrefix_) != 0) {
0192 rundata->indicesOfTriggerPaths.emplace_back(triggerNameIdx);
0193 }
0194 }
0195 auto const triggersSize = rundata->indicesOfTriggerPaths.size();
0196
0197
0198 auto const& datasets = rundata->hltConfig.datasetContents();
0199 auto const& datasetNames = rundata->hltConfig.datasetNames();
0200 auto const datasetsSize = datasetNames.size();
0201 rundata->datasets.resize(datasetsSize);
0202 for (auto ds = 0u; ds < datasetsSize; ++ds) {
0203 auto& dataset = rundata->datasets[ds];
0204
0205
0206
0207 auto const datasetPathName = datasetPathNamePrefix_ + datasetNames[ds];
0208 auto const datasetPathExists =
0209 std::find(triggerNames.begin(), triggerNames.end(), datasetPathName) != triggerNames.end();
0210 if (datasetPathExists) {
0211
0212
0213
0214 dataset.reserve(1);
0215 auto const index = rundata->hltConfig.triggerIndex(datasetPathName);
0216 if (index < triggerNamesSize)
0217 dataset.push_back(index);
0218 } else {
0219 auto const paths = datasets[ds].size();
0220 dataset.reserve(paths);
0221 for (auto p = 0u; p < paths; p++) {
0222 auto const index = rundata->hltConfig.triggerIndex(datasets[ds][p]);
0223 if (index < triggerNamesSize)
0224 dataset.push_back(index);
0225 }
0226 }
0227 }
0228
0229
0230 rundata->posL1s.resize(triggersSize);
0231 rundata->posPre.resize(triggersSize);
0232 for (auto i = 0u; i < triggersSize; ++i) {
0233 rundata->posL1s[i] = -1;
0234 rundata->posPre[i] = -1;
0235 auto const& moduleLabels = rundata->hltConfig.moduleLabels(i);
0236 for (auto j = 0u; j < moduleLabels.size(); ++j) {
0237 auto const& label = rundata->hltConfig.moduleType(moduleLabels[j]);
0238 if (label == "HLTL1TSeed")
0239 rundata->posL1s[i] = j;
0240 else if (label == "HLTPrescaler")
0241 rundata->posPre[i] = j;
0242 }
0243 }
0244 }
0245
0246
0247 rundata->jsdFileName = fmt::sprintf("run%06d_ls0000_streamHLTRates_pid%05d.jsd", run.run(), getpid());
0248 writeJsdFile(*rundata);
0249
0250
0251
0252 writeIniFile(*rundata, run.run());
0253
0254 return rundata;
0255 }
0256
0257
0258 void HLTriggerJSONMonitoring::globalEndRun(edm::Run const&, edm::EventSetup const&) const {}
0259
0260
0261 void HLTriggerJSONMonitoring::analyze(edm::StreamID sid, edm::Event const& event, edm::EventSetup const&) const {
0262 auto& stream = *streamCache(sid);
0263 auto const& rundata = *runCache(event.getRun().index());
0264
0265 ++stream.processed;
0266
0267
0268 if (not rundata.hltConfig.inited())
0269 return;
0270
0271
0272 edm::Handle<edm::TriggerResults> handle;
0273 if (not event.getByToken(triggerResultsToken_, handle) or not handle.isValid()) {
0274 edm::LogError("HLTriggerJSONMonitoring")
0275 << "TriggerResults with label [" + triggerResults_.encode() + "] not present or invalid";
0276 return;
0277 }
0278 edm::TriggerResults const& results = *handle;
0279 assert(results.size() == rundata.hltConfig.triggerNames().size());
0280
0281
0282 for (auto idx = 0u; idx < rundata.indicesOfTriggerPaths.size(); ++idx) {
0283 auto const triggerPathIdx = rundata.indicesOfTriggerPaths[idx];
0284 auto const& status = results[triggerPathIdx];
0285 if (status.wasrun()) {
0286 ++stream.hltWasRun[idx];
0287 if (status.accept()) {
0288 ++stream.hltL1s[idx];
0289 ++stream.hltPre[idx];
0290 ++stream.hltAccept[idx];
0291 } else {
0292 int const index = (int)status.index();
0293 if (index > rundata.posL1s[idx])
0294 ++stream.hltL1s[idx];
0295 if (index > rundata.posPre[idx])
0296 ++stream.hltPre[idx];
0297 if (status.error())
0298 ++stream.hltErrors[idx];
0299 else
0300 ++stream.hltReject[idx];
0301 }
0302 }
0303 }
0304
0305
0306
0307 for (auto i = 0u; i < rundata.datasets.size(); ++i)
0308 if (std::any_of(rundata.datasets[i].begin(), rundata.datasets[i].end(), [&](unsigned int path) {
0309 return results.accept(path);
0310 }))
0311 ++stream.datasets[i];
0312 }
0313
0314
0315 std::shared_ptr<HLTriggerJSONMonitoringData::lumisection> HLTriggerJSONMonitoring::globalBeginLuminosityBlockSummary(
0316 edm::LuminosityBlock const& lumi, edm::EventSetup const&) const {
0317 unsigned int triggers = 0;
0318 unsigned int datasets = 0;
0319 auto const& rundata = *runCache(lumi.getRun().index());
0320 if (rundata.hltConfig.inited()) {
0321 triggers = rundata.indicesOfTriggerPaths.size();
0322 datasets = rundata.hltConfig.datasetNames().size();
0323 };
0324
0325
0326
0327 auto lumidata = std::make_shared<HLTriggerJSONMonitoringData::lumisection>(HLTriggerJSONMonitoringData::lumisection{
0328 jsoncollector::HistoJ<unsigned int>(1),
0329 jsoncollector::HistoJ<unsigned int>(triggers),
0330 jsoncollector::HistoJ<unsigned int>(triggers),
0331 jsoncollector::HistoJ<unsigned int>(triggers),
0332 jsoncollector::HistoJ<unsigned int>(triggers),
0333 jsoncollector::HistoJ<unsigned int>(triggers),
0334 jsoncollector::HistoJ<unsigned int>(triggers),
0335 jsoncollector::HistoJ<unsigned int>(datasets)
0336 });
0337
0338 lumidata->processed.update(0);
0339 for (unsigned int i = 0; i < triggers; ++i)
0340 lumidata->hltWasRun.update(0);
0341 for (unsigned int i = 0; i < triggers; ++i)
0342 lumidata->hltL1s.update(0);
0343 for (unsigned int i = 0; i < triggers; ++i)
0344 lumidata->hltPre.update(0);
0345 for (unsigned int i = 0; i < triggers; ++i)
0346 lumidata->hltAccept.update(0);
0347 for (unsigned int i = 0; i < triggers; ++i)
0348 lumidata->hltReject.update(0);
0349 for (unsigned int i = 0; i < triggers; ++i)
0350 lumidata->hltErrors.update(0);
0351 for (unsigned int i = 0; i < datasets; ++i)
0352 lumidata->datasets.update(0);
0353
0354 return lumidata;
0355 }
0356
0357
0358 void HLTriggerJSONMonitoring::streamBeginLuminosityBlock(edm::StreamID sid,
0359 edm::LuminosityBlock const& lumi,
0360 edm::EventSetup const&) const {
0361 auto& stream = *streamCache(sid);
0362
0363 unsigned int triggers = 0;
0364 unsigned int datasets = 0;
0365 auto const& rundata = *runCache(lumi.getRun().index());
0366 if (rundata.hltConfig.inited()) {
0367 triggers = rundata.indicesOfTriggerPaths.size();
0368 datasets = rundata.hltConfig.datasetNames().size();
0369 };
0370
0371
0372 stream.processed = 0;
0373 stream.hltWasRun.assign(triggers, 0);
0374 stream.hltL1s.assign(triggers, 0);
0375 stream.hltPre.assign(triggers, 0);
0376 stream.hltAccept.assign(triggers, 0);
0377 stream.hltReject.assign(triggers, 0);
0378 stream.hltErrors.assign(triggers, 0);
0379 stream.datasets.assign(datasets, 0);
0380 }
0381
0382
0383 void HLTriggerJSONMonitoring::streamEndLuminosityBlockSummary(edm::StreamID sid,
0384 edm::LuminosityBlock const& lumi,
0385 edm::EventSetup const&,
0386 HLTriggerJSONMonitoringData::lumisection* lumidata) const {
0387 auto const& stream = *streamCache(sid);
0388 auto const& rundata = *runCache(lumi.getRun().index());
0389 lumidata->processed.value()[0] += stream.processed;
0390
0391
0392 if (not rundata.hltConfig.inited())
0393 return;
0394
0395 auto const triggers = rundata.indicesOfTriggerPaths.size();
0396 for (auto i = 0u; i < triggers; ++i) {
0397 lumidata->hltWasRun.value()[i] += stream.hltWasRun[i];
0398 lumidata->hltL1s.value()[i] += stream.hltL1s[i];
0399 lumidata->hltPre.value()[i] += stream.hltPre[i];
0400 lumidata->hltAccept.value()[i] += stream.hltAccept[i];
0401 lumidata->hltReject.value()[i] += stream.hltReject[i];
0402 lumidata->hltErrors.value()[i] += stream.hltErrors[i];
0403 }
0404 auto const datasets = rundata.hltConfig.datasetNames().size();
0405 for (auto i = 0u; i < datasets; ++i)
0406 lumidata->datasets.value()[i] += stream.datasets[i];
0407 }
0408
0409
0410 void HLTriggerJSONMonitoring::globalEndLuminosityBlockSummary(edm::LuminosityBlock const& lumi,
0411 edm::EventSetup const&,
0412 HLTriggerJSONMonitoringData::lumisection* lumidata) const {
0413 unsigned int ls = lumi.luminosityBlock();
0414 unsigned int run = lumi.run();
0415
0416 bool writeFiles = true;
0417 if (edm::Service<evf::MicroStateService>().isAvailable()) {
0418 evf::FastMonitoringService* fms =
0419 (evf::FastMonitoringService*)(edm::Service<evf::MicroStateService>().operator->());
0420 if (fms)
0421 writeFiles = fms->shouldWriteFiles(ls);
0422 }
0423 if (not writeFiles)
0424 return;
0425
0426 unsigned int processed = lumidata->processed.value().at(0);
0427 auto const& rundata = *runCache(lumi.getRun().index());
0428 Json::StyledWriter writer;
0429
0430
0431 char hostname[33];
0432 gethostname(hostname, 32);
0433 std::string sourceHost(hostname);
0434
0435
0436 std::stringstream sOutDef;
0437 sOutDef << rundata.baseRunDir << "/"
0438 << "output_" << getpid() << ".jsd";
0439
0440 std::string jsndataFileList = "";
0441 unsigned int jsndataSize = 0;
0442 unsigned int jsndataAdler32 = 1;
0443
0444 if (processed) {
0445
0446 Json::Value jsndata;
0447 jsndata[jsoncollector::DataPoint::SOURCE] = sourceHost;
0448 jsndata[jsoncollector::DataPoint::DEFINITION] = rundata.jsdFileName;
0449 jsndata[jsoncollector::DataPoint::DATA].append(lumidata->processed.toJsonValue());
0450 jsndata[jsoncollector::DataPoint::DATA].append(lumidata->hltWasRun.toJsonValue());
0451 jsndata[jsoncollector::DataPoint::DATA].append(lumidata->hltL1s.toJsonValue());
0452 jsndata[jsoncollector::DataPoint::DATA].append(lumidata->hltPre.toJsonValue());
0453 jsndata[jsoncollector::DataPoint::DATA].append(lumidata->hltAccept.toJsonValue());
0454 jsndata[jsoncollector::DataPoint::DATA].append(lumidata->hltReject.toJsonValue());
0455 jsndata[jsoncollector::DataPoint::DATA].append(lumidata->hltErrors.toJsonValue());
0456 jsndata[jsoncollector::DataPoint::DATA].append(lumidata->datasets.toJsonValue());
0457
0458 auto jsndataFileName = fmt::sprintf("run%06d_ls%04d_streamHLTRates_pid%05d.jsndata", run, ls, getpid());
0459
0460 std::string result = writer.write(jsndata);
0461 std::ofstream jsndataFile(rundata.baseRunDir + "/" + jsndataFileName);
0462 jsndataFile << result;
0463 jsndataFile.close();
0464
0465 jsndataFileList = jsndataFileName;
0466 jsndataSize = result.size();
0467 jsndataAdler32 = cms::Adler32(result.c_str(), result.size());
0468 }
0469
0470
0471 unsigned int jsnProcessed = processed;
0472 unsigned int jsnAccepted = processed;
0473 unsigned int jsnErrorEvents = 0;
0474 unsigned int jsnRetCodeMask = 0;
0475 std::string jsnInputFiles = "";
0476 unsigned int jsnHLTErrorEvents = 0;
0477
0478 Json::Value jsn;
0479 jsn[jsoncollector::DataPoint::SOURCE] = sourceHost;
0480 jsn[jsoncollector::DataPoint::DEFINITION] = sOutDef.str();
0481 jsn[jsoncollector::DataPoint::DATA].append(jsnProcessed);
0482 jsn[jsoncollector::DataPoint::DATA].append(jsnAccepted);
0483 jsn[jsoncollector::DataPoint::DATA].append(jsnErrorEvents);
0484 jsn[jsoncollector::DataPoint::DATA].append(jsnRetCodeMask);
0485 jsn[jsoncollector::DataPoint::DATA].append(jsndataFileList);
0486 jsn[jsoncollector::DataPoint::DATA].append(jsndataSize);
0487 jsn[jsoncollector::DataPoint::DATA].append(jsnInputFiles);
0488 jsn[jsoncollector::DataPoint::DATA].append(jsndataAdler32);
0489 jsn[jsoncollector::DataPoint::DATA].append(rundata.streamDestination);
0490 jsn[jsoncollector::DataPoint::DATA].append(rundata.streamMergeType);
0491 jsn[jsoncollector::DataPoint::DATA].append(jsnHLTErrorEvents);
0492
0493 auto jsnFileName = fmt::sprintf("run%06d_ls%04d_streamHLTRates_pid%05d.jsn", run, ls, getpid());
0494 std::ofstream jsnFile(rundata.baseRunDir + "/" + jsnFileName);
0495 jsnFile << writer.write(jsn);
0496 jsnFile.close();
0497 }
0498
0499 void HLTriggerJSONMonitoring::writeJsdFile(HLTriggerJSONMonitoringData::run const& rundata) {
0500 std::ofstream file(rundata.baseRunDir + "/" + rundata.jsdFileName);
0501 file << R"""({
0502 "data" : [
0503 { "name" : "Processed", "type" : "integer", "operation" : "histo"},
0504 { "name" : "Path-WasRun", "type" : "integer", "operation" : "histo"},
0505 { "name" : "Path-AfterL1Seed", "type" : "integer", "operation" : "histo"},
0506 { "name" : "Path-AfterPrescale", "type" : "integer", "operation" : "histo"},
0507 { "name" : "Path-Accepted", "type" : "integer", "operation" : "histo"},
0508 { "name" : "Path-Rejected", "type" : "integer", "operation" : "histo"},
0509 { "name" : "Path-Errors", "type" : "integer", "operation" : "histo"},
0510 { "name" : "Dataset-Accepted", "type" : "integer", "operation" : "histo"}
0511 ]
0512 }
0513 )""";
0514 file.close();
0515 }
0516
0517 void HLTriggerJSONMonitoring::writeIniFile(HLTriggerJSONMonitoringData::run const& rundata, unsigned int run) {
0518 Json::Value content;
0519
0520 Json::Value triggerNames(Json::arrayValue);
0521 for (auto idx : rundata.indicesOfTriggerPaths)
0522 triggerNames.append(rundata.hltConfig.triggerNames()[idx]);
0523 content["Path-Names"] = triggerNames;
0524
0525 Json::Value datasetNames(Json::arrayValue);
0526 for (auto const& name : rundata.hltConfig.datasetNames())
0527 datasetNames.append(name);
0528 content["Dataset-Names"] = datasetNames;
0529
0530 std::string iniFileName = fmt::sprintf("run%06d_ls0000_streamHLTRates_pid%05d.ini", run, getpid());
0531 std::ofstream file(rundata.baseRunDir + "/" + iniFileName);
0532 Json::StyledWriter writer;
0533 file << writer.write(content);
0534 file.close();
0535 }
0536
0537
0538 #include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
0539 #include "FWCore/Framework/interface/MakerMacros.h"
0540 DEFINE_FWK_MODULE(HLTriggerJSONMonitoring);