File indexing completed on 2025-03-14 23:36:19
0001 #ifndef FWCore_MessageLogger_JobReport_h
0002 #define FWCore_MessageLogger_JobReport_h
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018
0019
0020
0021
0022
0023
0024
0025
0026
0027
0028
0029
0030
0031
0032
0033
0034
0035
0036
0037
0038
0039
0040
0041
0042
0043
0044
0045
0046
0047
0048
0049
0050
0051
0052
0053
0054
0055
0056
0057
0058
0059
0060
0061
0062
0063
0064
0065
0066
0067
0068
0069
0070
0071
0072
0073
0074
0075
0076
0077
0078
0079
0080
0081
0082
0083
0084 #include "DataFormats/Provenance/interface/RunLumiEventNumber.h"
0085 #include "DataFormats/Provenance/interface/ParameterSetID.h"
0086 #include "DataFormats/Provenance/interface/ProcessConfigurationID.h"
0087 #include "FWCore/Utilities/interface/InputType.h"
0088 #include "FWCore/Utilities/interface/get_underlying_safe.h"
0089
0090 #include <atomic>
0091 #include <cstddef>
0092 #include <iosfwd>
0093 #include <map>
0094 #include <memory>
0095 #include <mutex>
0096 #include <set>
0097 #include <string>
0098 #include <vector>
0099
0100 #include "oneapi/tbb/concurrent_unordered_map.h"
0101 #include "oneapi/tbb/concurrent_vector.h"
0102
0103 namespace edm {
0104
0105 class JobReport {
0106 public:
0107 typedef unsigned int RunNumber;
0108 typedef std::size_t Token;
0109
0110 struct RunReport {
0111 RunNumber runNumber;
0112 std::map<unsigned int, unsigned long> lumiSectionsToNEvents;
0113 };
0114
0115
0116
0117
0118
0119
0120
0121
0122
0123
0124 struct InputFile {
0125 typedef std::vector<std::string> StringVector;
0126
0127 std::string logicalFileName;
0128 std::string physicalFileName;
0129 std::string catalog;
0130 std::string inputType;
0131 std::string inputSourceClassName;
0132 std::string moduleLabel;
0133 std::string guid;
0134 std::size_t numEventsRead;
0135 StringVector branchNames;
0136 std::map<RunNumber, RunReport> runReports;
0137 bool fileHasBeenClosed;
0138 std::set<std::string> fastClonedBranches;
0139 };
0140
0141
0142
0143
0144
0145
0146
0147
0148
0149
0150 struct OutputFile {
0151 typedef InputFile::StringVector StringVector;
0152
0153 std::string logicalFileName;
0154 std::string physicalFileName;
0155 std::string catalog;
0156 std::string outputModuleClassName;
0157 std::string moduleLabel;
0158 std::string guid;
0159 std::string dataType;
0160 std::string branchHash;
0161 std::size_t numEventsWritten;
0162 StringVector branchNames;
0163 std::vector<Token> contributingInputs;
0164 oneapi::tbb::concurrent_vector<Token> contributingInputsSecSource;
0165 std::map<std::string, bool> fastCopyingInputs;
0166 std::map<RunNumber, RunReport> runReports;
0167 bool fileHasBeenClosed;
0168 };
0169
0170 class AtomicLongLong {
0171 public:
0172 AtomicLongLong() : value_(0) {}
0173 AtomicLongLong(AtomicLongLong const& r) : value_(r.value_.load()) {}
0174 std::atomic<long long>& value() { return value_; }
0175 std::atomic<long long> const& value() const { return value_; }
0176
0177 private:
0178 std::atomic<long long> value_;
0179 };
0180
0181 struct JobReportImpl {
0182 JobReportImpl(JobReportImpl const&) = delete;
0183 JobReportImpl& operator=(JobReportImpl const&) = delete;
0184 JobReportImpl(JobReportImpl&&) = delete;
0185 JobReportImpl& operator=(JobReportImpl&&) = delete;
0186
0187 InputFile& getInputFileForToken(InputType inputType, Token t);
0188 OutputFile& getOutputFileForToken(Token t);
0189
0190
0191
0192
0193
0194
0195
0196 void insertInputForOutputs(InputType inputType, Token t);
0197
0198
0199
0200
0201
0202 void associateLumiSection(JobReport::Token token,
0203 unsigned int runNumber,
0204 unsigned int lumiSection,
0205 unsigned long nEvents);
0206
0207
0208
0209
0210
0211 void associateInputLumiSection(unsigned int runNumber, unsigned int lumiSection);
0212
0213
0214
0215
0216 void associateRun(JobReport::Token token, unsigned int runNumber);
0217
0218
0219
0220
0221 void associateInputRun(unsigned int runNumber);
0222
0223
0224
0225
0226
0227
0228 void writeInputFile(InputFile const& f);
0229
0230
0231
0232
0233
0234
0235
0236
0237
0238
0239
0240
0241
0242
0243
0244 void writeOutputFile(OutputFile const& f);
0245
0246
0247
0248
0249 void flushFiles(void);
0250
0251 JobReportImpl(std::ostream* iOst) : printedReadBranches_(false), ost_(iOst) {}
0252
0253 std::ostream const* ost() const { return get_underlying_safe(ost_); }
0254 std::ostream*& ost() { return get_underlying_safe(ost_); }
0255
0256 std::vector<InputFile> inputFiles_;
0257 oneapi::tbb::concurrent_vector<InputFile> inputFilesSecSource_;
0258 oneapi::tbb::concurrent_vector<OutputFile> outputFiles_;
0259 std::map<std::string, long long> readBranches_;
0260 std::map<std::string, long long> readBranchesSecFile_;
0261 oneapi::tbb::concurrent_unordered_map<std::string, AtomicLongLong> readBranchesSecSource_;
0262 std::atomic<bool> printedReadBranches_;
0263 std::vector<InputFile>::size_type lastOpenedPrimaryInputFile_;
0264 edm::propagate_const<std::ostream*> ost_;
0265 };
0266
0267 JobReport();
0268
0269 JobReport(std::ostream* outputStream);
0270
0271 JobReport& operator=(JobReport const&) = delete;
0272 JobReport(JobReport const&) = delete;
0273
0274 ~JobReport();
0275
0276
0277 void reportProcess(std::string_view processName,
0278 ProcessConfigurationID const& reducedProcessID,
0279 ParameterSetID const& psetID);
0280
0281
0282
0283
0284 Token inputFileOpened(std::string const& physicalFileName,
0285 std::string const& logicalFileName,
0286 std::string const& catalog,
0287 std::string const& inputType,
0288 std::string const& inputSourceClassName,
0289 std::string const& moduleLabel,
0290 std::string const& guid,
0291 std::vector<std::string> const& branchNames);
0292
0293
0294
0295 void eventReadFromFile(InputType inputType, Token fileToken);
0296
0297
0298
0299
0300
0301
0302
0303 void reportDataType(Token fileToken, std::string const& dataType);
0304
0305
0306
0307
0308 void inputFileClosed(InputType inputType, Token fileToken);
0309
0310
0311
0312
0313 Token outputFileOpened(std::string const& physicalFileName,
0314 std::string const& logicalFileName,
0315 std::string const& catalog,
0316 std::string const& outputModuleClassName,
0317 std::string const& moduleLabel,
0318 std::string const& guid,
0319 std::string const& dataType,
0320 std::string const& branchHash,
0321 std::vector<std::string> const& branchNames);
0322
0323
0324
0325
0326
0327 void eventWrittenToFile(Token fileToken, RunNumber_t run, EventNumber_t event);
0328
0329
0330
0331
0332 void outputFileClosed(Token fileToken);
0333
0334 void reportSkippedEvent(RunNumber_t run, EventNumber_t event);
0335
0336
0337
0338
0339
0340 void reportRunNumber(JobReport::Token token, unsigned int run);
0341
0342
0343
0344
0345
0346 void reportLumiSection(JobReport::Token token, unsigned int run, unsigned int lumiSectId, unsigned long nEvents = 0);
0347
0348
0349
0350
0351
0352
0353 void reportInputLumiSection(unsigned int run, unsigned int lumiSectId);
0354
0355
0356
0357
0358 void reportInputRunNumber(unsigned int run);
0359
0360
0361
0362
0363
0364
0365 void reportError(std::string const& shortDesc, std::string const& longDesc, int const& exitCode);
0366
0367
0368
0369
0370 void reportShutdownSignal();
0371
0372
0373
0374
0375
0376
0377 void reportSkippedFile(std::string const& pfn, std::string const& lfn);
0378
0379 void reportFallbackAttempt(std::string const& pfn, std::string const& lfn, std::string const& err);
0380
0381 void reportAnalysisFile(std::string const& fileName, std::map<std::string, std::string> const& fileData);
0382
0383
0384
0385
0386
0387
0388
0389 void reportMemoryInfo(std::vector<std::string> const& memoryData);
0390
0391
0392
0393
0394
0395
0396
0397 void reportMessageInfo(std::map<std::string, double> const& messageData);
0398
0399
0400
0401
0402
0403
0404
0405 void reportReadBranches();
0406
0407
0408 void reportReadBranch(InputType inputType, std::string const& branchName);
0409
0410
0411 void reportFastClonedBranches(std::set<std::string> const& fastClonedBranches, long long nEvents);
0412
0413
0414
0415
0416 void reportRandomStateFile(std::string const& name);
0417
0418
0419
0420
0421
0422 void reportFastCopyingStatus(Token t, std::string const& inputFileName, bool fastCopying);
0423
0424
0425
0426
0427
0428
0429
0430
0431 void reportPerformanceSummary(std::string const& metricClass, std::map<std::string, std::string> const& metrics);
0432
0433 void reportPerformanceForModule(std::string const& metricClass,
0434 std::string const& moduleName,
0435 std::map<std::string, std::string> const& metrics);
0436
0437
0438 std::string dumpFiles(void);
0439
0440 protected:
0441 edm::propagate_const<std::unique_ptr<JobReportImpl>>& impl() { return impl_; }
0442
0443 private:
0444 void temporarilyCloseXML();
0445 edm::propagate_const<std::unique_ptr<JobReportImpl>> impl_;
0446 std::mutex write_mutex;
0447 bool errorLogged_ = false;
0448 };
0449
0450 std::ostream& operator<<(std::ostream& os, JobReport::InputFile const& f);
0451 std::ostream& operator<<(std::ostream& os, JobReport::OutputFile const& f);
0452 }
0453 #endif