StreamerIO/plugins/DQMFileIterator.h

0001 #ifndef IOPool_DQMStreamer_DQMFilerIterator_h
0002 #define IOPool_DQMStreamer_DQMFilerIterator_h
0003
0004 #include "FWCore/ServiceRegistry/interface/Service.h"
0005 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0006 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0007
0008 #include <filesystem>
0009
0010 #include <map>
0011 #include <unordered_set>
0012 #include <chrono>
0013
0014 #include "DQMMonitoringService.h"
0015
0016 namespace dqmservices {
0017
0018   class DQMFileIterator {
0019   public:
0020     struct LumiEntry {
0021       std::string filename;
0022       std::string run_path;
0023
0024       unsigned int file_ls;
0025       std::size_t n_events_processed;
0026       std::size_t n_events_accepted;
0027       std::string datafn;
0028
0029       static LumiEntry load_json(const std::string& run_path,
0030                                  const std::string& filename,
0031                                  int lumiNumber,
0032                                  int datafn_position);
0033
0034       std::string get_data_path() const;
0035       std::string get_json_path() const;
0036       std::string state;
0037     };
0038
0039     struct EorEntry {
0040       bool loaded = false;
0041       std::string filename;
0042       std::string run_path;
0043
0044       std::size_t n_events;
0045       std::size_t n_lumi;
0046
0047       static EorEntry load_json(const std::string& run_path, const std::string& filename);
0048     };
0049
0050     enum State {
0051       OPEN = 0,
0052       EOR_CLOSING = 1,  // EoR file found, but lumis are still pending
0053       EOR = 2,
0054     };
0055
0056     DQMFileIterator(edm::ParameterSet const& pset);
0057     ~DQMFileIterator();
0058     void initialise(int run, const std::string&, const std::string&);
0059
0060     State state();
0061
0062     /* methods to iterate the actual files */
0063
0064     /* nextLumiNumber_ is the first unprocessed lumi number
0065    * lumiReady() returns if the next lumi is ready to be loaded
0066    * open() opens a file and advances the pointer to the next lumi
0067    *
0068    * front() a reference to the description (LumiEntry)
0069    * pop() advances to the next lumi
0070    */
0071     bool lumiReady();
0072     LumiEntry open();
0073
0074     void pop();
0075
0076     /* control */
0077     void reset();
0078     void update_state();
0079
0080     /* misc helpers for input sources */
0081     void logFileAction(const std::string& msg, const std::string& fileName = "") const;
0082     void logLumiState(const LumiEntry& lumi, const std::string& msg);
0083
0084     void delay();
0085
0086     unsigned int runNumber();
0087     unsigned int lastLumiFound();
0088     void advanceToLumi(unsigned int lumi, std::string reason);
0089
0090     static void fillDescription(edm::ParameterSetDescription& d);
0091
0092   private:
0093     unsigned int runNumber_;
0094     std::string runInputDir_;
0095     std::string streamLabel_;
0096     unsigned long delayMillis_;
0097     long nextLumiTimeoutMillis_;
0098     long forceFileCheckTimeoutMillis_;
0099     bool flagScanOnce_;
0100
0101     // file name position in the json file
0102     unsigned int datafnPosition_;
0103     std::vector<std::string> runPath_;
0104
0105     EorEntry eor_;
0106     State state_;
0107
0108     unsigned int nextLumiNumber_;
0109     std::map<unsigned int, LumiEntry> lumiSeen_;
0110     std::unordered_set<std::string> filesSeen_;
0111
0112     /* this should be different,
0113    * since time between hosts might be not in sync */
0114     unsigned runPathMTime_;
0115     std::chrono::high_resolution_clock::time_point runPathLastCollect_;
0116
0117     /* this is for missing lumi files */
0118     std::chrono::high_resolution_clock::time_point lastLumiLoad_;
0119
0120     unsigned mtimeHash() const;
0121     void collect(bool ignoreTimers);
0122     void monUpdateLumi(const LumiEntry& lumi);
0123
0124     /* this is for monitoring */
0125     edm::Service<DQMMonitoringService> mon_;
0126   };
0127
0128 }  // namespace dqmservices
0129
0130 #endif