StreamerIO/plugins/DQMFileIterator.h

0001 #ifndef DQMServices_StreamerIO_DQMFileIterator_h
0002 #define DQMServices_StreamerIO_DQMFileIterator_h
0003
0004 #include <chrono>
0005 #include <map>
0006 #include <string>
0007 #include <unordered_set>
0008 #include <vector>
0009
0010 #include "FWCore/ServiceRegistry/interface/Service.h"
0011
0012 namespace edm {
0013   class ParameterSet;
0014   class ParameterSetDescription;
0015 }  // namespace edm
0016
0017 namespace dqmservices {
0018
0019   class DQMMonitoringService;
0020
0021   class DQMFileIterator {
0022   public:
0023     struct LumiEntry {
0024       std::string filename;
0025       std::string run_path;
0026
0027       unsigned int file_ls;
0028       std::size_t n_events_processed;
0029       std::size_t n_events_accepted;
0030       std::string datafn;
0031
0032       static LumiEntry load_json(const std::string& run_path,
0033                                  const std::string& filename,
0034                                  int lumiNumber,
0035                                  int datafn_position);
0036
0037       std::string get_data_path() const;
0038       std::string get_json_path() const;
0039       std::string state;
0040     };
0041
0042     struct EorEntry {
0043       bool loaded = false;
0044       std::string filename;
0045       std::string run_path;
0046
0047       std::size_t n_events;
0048       std::size_t n_lumi;
0049
0050       static EorEntry load_json(const std::string& run_path, const std::string& filename);
0051     };
0052
0053     enum State {
0054       OPEN = 0,
0055       EOR_CLOSING = 1,  // EoR file found, but lumis are still pending
0056       EOR = 2,
0057     };
0058
0059     DQMFileIterator(edm::ParameterSet const& pset);
0060     ~DQMFileIterator() = default;
0061     void initialise(int run, const std::string&, const std::string&);
0062
0063     State state() const { return state_; }
0064
0065     /* methods to iterate the actual files */
0066
0067     /* nextLumiNumber_ is the first unprocessed lumi number
0068    * lumiReady() returns if the next lumi is ready to be loaded
0069    * open() opens a file and advances the pointer to the next lumi
0070    *
0071    * front() a reference to the description (LumiEntry)
0072    * pop() advances to the next lumi
0073    */
0074     bool lumiReady();
0075     LumiEntry open();
0076
0077     void pop();
0078
0079     /* control */
0080     void reset();
0081     void update_state();
0082
0083     /* misc helpers for input sources */
0084     void logFileAction(const std::string& msg, const std::string& fileName = "") const;
0085     void logLumiState(const LumiEntry& lumi, const std::string& msg);
0086
0087     void delay();
0088
0089     unsigned int runNumber() const { return runNumber_; }
0090     unsigned int lastLumiFound();
0091     void advanceToLumi(unsigned int lumi, std::string reason);
0092
0093     static void fillDescription(edm::ParameterSetDescription& d);
0094
0095   private:
0096     unsigned int runNumber_;
0097     std::string runInputDir_;
0098     std::string streamLabel_;
0099     unsigned long delayMillis_;
0100     long nextLumiTimeoutMillis_;
0101     long forceFileCheckTimeoutMillis_;
0102     bool flagScanOnce_;
0103
0104     // file name position in the json file
0105     unsigned int datafnPosition_;
0106     std::vector<std::string> runPath_;
0107
0108     EorEntry eor_;
0109     State state_;
0110
0111     unsigned int nextLumiNumber_;
0112     std::map<unsigned int, LumiEntry> lumiSeen_;
0113     std::unordered_set<std::string> filesSeen_;
0114
0115     /* this should be different,
0116    * since time between hosts might be not in sync */
0117     unsigned runPathMTime_;
0118     std::chrono::high_resolution_clock::time_point runPathLastCollect_;
0119
0120     /* this is for missing lumi files */
0121     std::chrono::high_resolution_clock::time_point lastLumiLoad_;
0122
0123     unsigned mtimeHash() const;
0124     void collect(bool ignoreTimers);
0125     void monUpdateLumi(const LumiEntry& lumi);
0126
0127     /* this is for monitoring */
0128     edm::Service<DQMMonitoringService> mon_;
0129   };
0130
0131 }  // namespace dqmservices
0132
0133 #endif  // DQMServices_StreamerIO_DQMFileIterator_h