File indexing completed on 2023-10-25 09:47:31
0001 #ifndef FWCore_MessageLogger_JobReport_h
0002 #define FWCore_MessageLogger_JobReport_h
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018
0019
0020
0021
0022
0023
0024
0025
0026
0027
0028
0029
0030
0031
0032
0033
0034
0035
0036
0037
0038
0039
0040
0041
0042
0043
0044
0045
0046
0047
0048
0049
0050
0051
0052
0053
0054
0055
0056
0057
0058
0059
0060
0061
0062
0063
0064
0065
0066
0067
0068
0069
0070
0071
0072
0073
0074
0075
0076
0077
0078
0079
0080
0081
0082
0083
0084 #include "DataFormats/Provenance/interface/RunLumiEventNumber.h"
0085 #include "FWCore/Utilities/interface/InputType.h"
0086 #include "FWCore/Utilities/interface/get_underlying_safe.h"
0087
0088 #include <atomic>
0089 #include <cstddef>
0090 #include <iosfwd>
0091 #include <map>
0092 #include <memory>
0093 #include <mutex>
0094 #include <set>
0095 #include <string>
0096 #include <vector>
0097
0098 #include "oneapi/tbb/concurrent_unordered_map.h"
0099 #include "oneapi/tbb/concurrent_vector.h"
0100
0101 namespace edm {
0102
0103 class JobReport {
0104 public:
0105 typedef unsigned int RunNumber;
0106 typedef std::size_t Token;
0107
0108 struct RunReport {
0109 RunNumber runNumber;
0110 std::map<unsigned int, unsigned long> lumiSectionsToNEvents;
0111 };
0112
0113
0114
0115
0116
0117
0118
0119
0120
0121
0122 struct InputFile {
0123 typedef std::vector<std::string> StringVector;
0124
0125 std::string logicalFileName;
0126 std::string physicalFileName;
0127 std::string catalog;
0128 std::string inputType;
0129 std::string inputSourceClassName;
0130 std::string moduleLabel;
0131 std::string guid;
0132 std::size_t numEventsRead;
0133 StringVector branchNames;
0134 std::map<RunNumber, RunReport> runReports;
0135 bool fileHasBeenClosed;
0136 std::set<std::string> fastClonedBranches;
0137 };
0138
0139
0140
0141
0142
0143
0144
0145
0146
0147
0148 struct OutputFile {
0149 typedef InputFile::StringVector StringVector;
0150
0151 std::string logicalFileName;
0152 std::string physicalFileName;
0153 std::string catalog;
0154 std::string outputModuleClassName;
0155 std::string moduleLabel;
0156 std::string guid;
0157 std::string dataType;
0158 std::string branchHash;
0159 std::size_t numEventsWritten;
0160 StringVector branchNames;
0161 std::vector<Token> contributingInputs;
0162 oneapi::tbb::concurrent_vector<Token> contributingInputsSecSource;
0163 std::map<std::string, bool> fastCopyingInputs;
0164 std::map<RunNumber, RunReport> runReports;
0165 bool fileHasBeenClosed;
0166 };
0167
0168 class AtomicLongLong {
0169 public:
0170 AtomicLongLong() : value_(0) {}
0171 AtomicLongLong(AtomicLongLong const& r) : value_(r.value_.load()) {}
0172 std::atomic<long long>& value() { return value_; }
0173 std::atomic<long long> const& value() const { return value_; }
0174
0175 private:
0176 std::atomic<long long> value_;
0177 };
0178
0179 struct JobReportImpl {
0180 JobReportImpl& operator=(JobReportImpl const&) = delete;
0181 JobReportImpl(JobReportImpl const&) = delete;
0182
0183 InputFile& getInputFileForToken(InputType inputType, Token t);
0184 OutputFile& getOutputFileForToken(Token t);
0185
0186
0187
0188
0189
0190
0191
0192 void insertInputForOutputs(InputType inputType, Token t);
0193
0194
0195
0196
0197
0198 void associateLumiSection(JobReport::Token token,
0199 unsigned int runNumber,
0200 unsigned int lumiSection,
0201 unsigned long nEvents);
0202
0203
0204
0205
0206
0207 void associateInputLumiSection(unsigned int runNumber, unsigned int lumiSection);
0208
0209
0210
0211
0212 void associateRun(JobReport::Token token, unsigned int runNumber);
0213
0214
0215
0216
0217 void associateInputRun(unsigned int runNumber);
0218
0219
0220
0221
0222
0223
0224 void writeInputFile(InputFile const& f);
0225
0226
0227
0228
0229
0230
0231
0232
0233
0234
0235
0236
0237
0238
0239
0240 void writeOutputFile(OutputFile const& f);
0241
0242
0243
0244
0245 void flushFiles(void);
0246
0247 JobReportImpl(std::ostream* iOst) : printedReadBranches_(false), ost_(iOst) {}
0248
0249 std::ostream const* ost() const { return get_underlying_safe(ost_); }
0250 std::ostream*& ost() { return get_underlying_safe(ost_); }
0251
0252 std::vector<InputFile> inputFiles_;
0253 oneapi::tbb::concurrent_vector<InputFile> inputFilesSecSource_;
0254 oneapi::tbb::concurrent_vector<OutputFile> outputFiles_;
0255 std::map<std::string, long long> readBranches_;
0256 std::map<std::string, long long> readBranchesSecFile_;
0257 oneapi::tbb::concurrent_unordered_map<std::string, AtomicLongLong> readBranchesSecSource_;
0258 std::atomic<bool> printedReadBranches_;
0259 std::vector<InputFile>::size_type lastOpenedPrimaryInputFile_;
0260 edm::propagate_const<std::ostream*> ost_;
0261 };
0262
0263 JobReport();
0264
0265 JobReport(std::ostream* outputStream);
0266
0267 JobReport& operator=(JobReport const&) = delete;
0268 JobReport(JobReport const&) = delete;
0269
0270 ~JobReport();
0271
0272
0273
0274
0275 Token inputFileOpened(std::string const& physicalFileName,
0276 std::string const& logicalFileName,
0277 std::string const& catalog,
0278 std::string const& inputType,
0279 std::string const& inputSourceClassName,
0280 std::string const& moduleLabel,
0281 std::string const& guid,
0282 std::vector<std::string> const& branchNames);
0283
0284
0285
0286 void eventReadFromFile(InputType inputType, Token fileToken);
0287
0288
0289
0290
0291
0292
0293
0294 void reportDataType(Token fileToken, std::string const& dataType);
0295
0296
0297
0298
0299 void inputFileClosed(InputType inputType, Token fileToken);
0300
0301
0302
0303
0304 Token outputFileOpened(std::string const& physicalFileName,
0305 std::string const& logicalFileName,
0306 std::string const& catalog,
0307 std::string const& outputModuleClassName,
0308 std::string const& moduleLabel,
0309 std::string const& guid,
0310 std::string const& dataType,
0311 std::string const& branchHash,
0312 std::vector<std::string> const& branchNames);
0313
0314
0315
0316
0317
0318 void eventWrittenToFile(Token fileToken, RunNumber_t run, EventNumber_t event);
0319
0320
0321
0322
0323 void outputFileClosed(Token fileToken);
0324
0325 void reportSkippedEvent(RunNumber_t run, EventNumber_t event);
0326
0327
0328
0329
0330
0331 void reportRunNumber(JobReport::Token token, unsigned int run);
0332
0333
0334
0335
0336
0337 void reportLumiSection(JobReport::Token token, unsigned int run, unsigned int lumiSectId, unsigned long nEvents = 0);
0338
0339
0340
0341
0342
0343
0344 void reportInputLumiSection(unsigned int run, unsigned int lumiSectId);
0345
0346
0347
0348
0349 void reportInputRunNumber(unsigned int run);
0350
0351
0352
0353
0354
0355
0356 void reportError(std::string const& shortDesc, std::string const& longDesc, int const& exitCode);
0357
0358
0359
0360
0361 void reportShutdownSignal();
0362
0363
0364
0365
0366
0367
0368 void reportSkippedFile(std::string const& pfn, std::string const& lfn);
0369
0370 void reportFallbackAttempt(std::string const& pfn, std::string const& lfn, std::string const& err);
0371
0372 void reportAnalysisFile(std::string const& fileName, std::map<std::string, std::string> const& fileData);
0373
0374
0375
0376
0377
0378
0379
0380 void reportMemoryInfo(std::vector<std::string> const& memoryData);
0381
0382
0383
0384
0385
0386
0387
0388 void reportMessageInfo(std::map<std::string, double> const& messageData);
0389
0390
0391
0392
0393
0394
0395
0396 void reportReadBranches();
0397
0398
0399 void reportReadBranch(InputType inputType, std::string const& branchName);
0400
0401
0402 void reportFastClonedBranches(std::set<std::string> const& fastClonedBranches, long long nEvents);
0403
0404
0405
0406
0407 void reportRandomStateFile(std::string const& name);
0408
0409
0410
0411
0412
0413 void reportFastCopyingStatus(Token t, std::string const& inputFileName, bool fastCopying);
0414
0415
0416
0417
0418
0419
0420
0421
0422 void reportPerformanceSummary(std::string const& metricClass, std::map<std::string, std::string> const& metrics);
0423
0424 void reportPerformanceForModule(std::string const& metricClass,
0425 std::string const& moduleName,
0426 std::map<std::string, std::string> const& metrics);
0427
0428
0429 std::string dumpFiles(void);
0430
0431 protected:
0432 edm::propagate_const<std::unique_ptr<JobReportImpl>>& impl() { return impl_; }
0433
0434 private:
0435 void temporarilyCloseXML();
0436 edm::propagate_const<std::unique_ptr<JobReportImpl>> impl_;
0437 std::mutex write_mutex;
0438 bool errorLogged_ = false;
0439 };
0440
0441 std::ostream& operator<<(std::ostream& os, JobReport::InputFile const& f);
0442 std::ostream& operator<<(std::ostream& os, JobReport::OutputFile const& f);
0443 }
0444 #endif