Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2022-01-16 23:25:22

0001 #ifndef CondCore_ESSources_CondDBESSource_h
0002 #define CondCore_ESSources_CondDBESSource_h
0003 //
0004 // Package:    CondCore/ESSources
0005 // Class:      CondDBESSource
0006 //
0007 /*
0008  Description: EventSetup source module for serving data from offline database
0009 */
0010 //
0011 // Author:      Zhen Xie
0012 //
0013 
0014 // Some comments on concurrency. Several things are working together
0015 // to prevent concurrency issues when this module is executing.
0016 // Some of these things are in this module and some are in the Framework.
0017 // Here is a list of these things:
0018 //
0019 //   1. There is a single mutex that is a data member of CondDBESSource.
0020 //   This is locked near the beginning of setIntervalFor and also
0021 //   near the beginning of ::DataProxy::prefetch so that these functions
0022 //   will never run concurrently. All the ::DataProxy objects have a
0023 //   pointer to this mutex stored in their ESSourceDataProxyTemplate
0024 //   base class.
0025 //
0026 //   2. CondDBESSource contains a single SerialTaskQueue. The tasks
0027 //   that run the prefetch function are placed in this SerialTaskQueue.
0028 //   This allows only one ::DataProxy::prefetch function to run at a
0029 //   time. All the ::DataProxy objects have a pointer to this SerialTaskQueue
0030 //   stored in their ESSourceDataProxyTemplate base class. Note that
0031 //   locking the mutex is inside the task that runs prefetch.
0032 //   Since these tasks are serialized by the SerialTaskQueue,
0033 //   the mutex will never be locked by another prefetch call
0034 //   when prefetch is called. The mutex is really only protecting
0035 //   setIntervalFor calls from each other and from prefetch calls.
0036 //
0037 //   3. An ESSource is not allowed to get data from the EventSetup
0038 //   while its DataProxy prefetch function runs, preventing deadlocks
0039 //   and ensuring the mutex does not need to be recursive.
0040 //
0041 //   4. The WaitingTaskList in ESSourceDataProxyBase (a base class of
0042 //   ::DataProxy) is used to notify other tasks waiting for prefetch
0043 //   to complete that the data is available (other tasks created and
0044 //   managed by the Framework).
0045 //
0046 //   5. There is an atomic<bool> in ESSourceDataProxyBase which
0047 //   prevents the prefetch function being run more than once for the
0048 //   same IOV and DataProxy.
0049 //
0050 //   6. The Framework ensures calls are sequenced such that a call to
0051 //   setIntervalFor is made and completes, then all related calls to
0052 //   DataProxy::initializeForNewIOV are made before another call to
0053 //   setIntervalFor is made.  It is configurable how many
0054 //   IOVs can be running concurrently. The Framework will not call
0055 //   initializeForNewIOV or start running a new IOV unless the
0056 //   number of active IOVs is less than that configured number.
0057 //
0058 //   7. The Framework guarantees that after a call is made to
0059 //   DataProxy::initializeForNewIOV for a particular
0060 //   EventSetupRecordKey and iovIndex, all calls to DataProxy::make
0061 //   associated with that whose data is requested will be completed
0062 //   and processing of luminosity blocks associated with that will
0063 //   be completed before another call to DataProxy::initializeForNewIOV
0064 //   is made for that EventSetupRecordKey and iovIndex.
0065 
0066 // system include files
0067 #include <string>
0068 #include <map>
0069 #include <memory>
0070 #include <set>
0071 #include <mutex>
0072 // user include files
0073 #include "CondCore/CondDB/interface/ConnectionPool.h"
0074 
0075 #include "FWCore/Framework/interface/DataProxyProvider.h"
0076 #include "FWCore/Framework/interface/EventSetupRecordIntervalFinder.h"
0077 #include "FWCore/Concurrency/interface/SerialTaskQueue.h"
0078 
0079 namespace edm {
0080   class ParameterSet;
0081 }
0082 
0083 namespace cond {
0084   class DataProxyWrapperBase;
0085 }
0086 
0087 class CondDBESSource : public edm::eventsetup::DataProxyProvider, public edm::EventSetupRecordIntervalFinder {
0088 public:
0089   using DataKey = edm::eventsetup::DataKey;
0090   using EventSetupRecordKey = edm::eventsetup::EventSetupRecordKey;
0091   typedef std::shared_ptr<cond::DataProxyWrapperBase> ProxyP;
0092   typedef std::multimap<std::string, ProxyP> ProxyMap;
0093 
0094   typedef enum { NOREFRESH, REFRESH_ALWAYS, REFRESH_OPEN_IOVS, REFRESH_EACH_RUN, RECONNECT_EACH_RUN } RefreshPolicy;
0095 
0096   explicit CondDBESSource(const edm::ParameterSet&);
0097   ~CondDBESSource() override;
0098 
0099 protected:
0100   void setIntervalFor(const EventSetupRecordKey&, const edm::IOVSyncValue&, edm::ValidityInterval&) override;
0101 
0102   KeyedProxiesVector registerProxies(const EventSetupRecordKey&, unsigned int iovIndex) override;
0103 
0104   void initConcurrentIOVs(const EventSetupRecordKey& key, unsigned int nConcurrentIOVs) override;
0105 
0106   bool isConcurrentFinder() const override { return true; }
0107 
0108 private:
0109   // ----------member data ---------------------------
0110 
0111   cond::persistency::ConnectionPool m_connection;
0112   std::string m_connectionString;
0113   std::string m_frontierKey;
0114 
0115   // Container of DataProxy, implemented as multi-map keyed by records
0116   ProxyMap m_proxies;
0117 
0118   typedef std::map<std::string, cond::GTEntry_t> TagCollection;
0119   // the collections of tag, record/label used in this ESSource
0120   TagCollection m_tagCollection;
0121   std::map<std::string, std::pair<cond::Time_t, bool> > m_refreshTimeForRecord;
0122   std::map<std::string, std::pair<cond::persistency::Session, std::string> > m_sessionPool;
0123   std::map<std::string, std::pair<cond::persistency::Session, std::string> > m_sessionPoolForLumiConditions;
0124   std::map<std::string, unsigned int> m_lastRecordRuns;
0125 
0126   edm::SerialTaskQueue m_queue;
0127   std::mutex m_mutex;
0128 
0129   struct Stats {
0130     int nData;
0131     int nSet;
0132     int nRun;
0133     int nLumi;
0134     int nRefresh;
0135     int nActualRefresh;
0136     int nReconnect;
0137     int nActualReconnect;
0138   };
0139 
0140   Stats m_stats;
0141 
0142   unsigned int m_lastRun;
0143   unsigned int m_lastLumi;
0144   RefreshPolicy m_policy;
0145 
0146   bool m_doDump;
0147 
0148 private:
0149   void fillList(const std::string& pfn,
0150                 std::vector<std::string>& pfnList,
0151                 const unsigned int listSize,
0152                 const std::string& type);
0153 
0154   void fillTagCollectionFromGT(const std::string& connectionString,
0155                                const std::string& prefix,
0156                                const std::string& postfix,
0157                                const std::string& roottag,
0158                                std::set<cond::GTEntry_t>& tagcoll,
0159                                cond::GTMetadata_t& gtMetadata);
0160 
0161   void fillTagCollectionFromDB(const std::vector<std::string>& connectionStringList,
0162                                const std::vector<std::string>& prefixList,
0163                                const std::vector<std::string>& postfixList,
0164                                const std::vector<std::string>& roottagList,
0165                                std::map<std::string, cond::GTEntry_t>& replacement,
0166                                cond::GTMetadata_t& gtMetadata);
0167 };
0168 #endif