CondHDF5ESSource/plugins/CondHDF5ESSource.cc

0001 // -*- C++ -*-
0002 //
0003 // Package:     CondCore/HDF5ESSource
0004 // Class  :     CondHDF5ESSource
0005 //
0006 // Implementation:
0007 //     [Notes on implementation]
0008 //
0009 // Original Author:  Christopher Jones
0010 //         Created:  Fri, 16 Jun 2023 15:17:53 GMT
0011 //
0012
0013 // system include files
0014 #include <cassert>
0015 #include <iostream>
0016
0017 // user include files
0018 #include "FWCore/Framework/interface/EventSetupRecordIntervalFinder.h"
0019 #include "FWCore/Framework/interface/ESProductResolverProvider.h"
0020 #include "FWCore/Framework/interface/IOVSyncValue.h"
0021 #include "FWCore/Framework/interface/SourceFactory.h"
0022 #include "FWCore/Framework/interface/ValidityInterval.h"
0023 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0024 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0025 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0026 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0027 #include "FWCore/Concurrency/interface/SerialTaskQueue.h"
0028
0029 #include "CondFormats/SerializationHelper/interface/SerializationHelperFactory.h"
0030
0031 #include "IOVSyncValue.h"
0032 #include "DataProduct.h"
0033 #include "Record.h"
0034 #include "HDF5ProductResolver.h"
0035 #include "convertSyncValue.h"
0036 #include "h5_File.h"
0037 #include "h5_Group.h"
0038 #include "h5_DataSet.h"
0039 #include "h5_Attribute.h"
0040 #include "Compression.h"
0041
0042 using namespace cond::hdf5;
0043
0044 class CondHDF5ESSource : public edm::EventSetupRecordIntervalFinder, public edm::eventsetup::ESProductResolverProvider {
0045 public:
0046   using EventSetupRecordKey = edm::eventsetup::EventSetupRecordKey;
0047   explicit CondHDF5ESSource(edm::ParameterSet const&);
0048
0049   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0050
0051 private:
0052   bool isConcurrentFinder() const final { return true; }
0053   void setIntervalFor(EventSetupRecordKey const&, edm::IOVSyncValue const&, edm::ValidityInterval&) final;
0054   KeyedResolversVector registerResolvers(EventSetupRecordKey const&, unsigned int iovIndex) final;
0055
0056   edm::SerialTaskQueue queue_;
0057   std::mutex mutex_;
0058   std::vector<Record> records_;
0059   std::string filename_;
0060   cms::h5::File file_;
0061   Compression compression_ = Compression::kNone;
0062 };
0063
0064 //
0065 // constants, enums and typedefs
0066 //
0067
0068 //
0069 // static data member definitions
0070 //
0071 namespace {
0072   cond::hdf5::Compression nameToEnum(std::string const& iName) {
0073     if (iName == "zlib") {
0074       return Compression::kZLIB;
0075     } else if (iName == "lzma") {
0076       return Compression::kLZMA;
0077     } else if (iName == "none") {
0078       return Compression::kNone;
0079     } else {
0080       throw cms::Exception("BadCompressionType") << "unknown compression type used in file '" << iName << "'";
0081     }
0082     return Compression::kNone;
0083   }
0084 }  // namespace
0085
0086 //
0087 // constructors and destructor
0088 //
0089 CondHDF5ESSource::CondHDF5ESSource(edm::ParameterSet const& iPSet)
0090     : filename_(iPSet.getUntrackedParameter<std::string>("filename")),
0091       file_(filename_, cms::h5::File::kReadOnly),
0092       compression_(nameToEnum(file_.findAttribute("default_payload_compressor")->readString())) {
0093   const auto globalTagsGroup = file_.findGroup("GlobalTags");
0094   const auto chosenTag = globalTagsGroup->findGroup(iPSet.getParameter<std::string>("globalTag"));
0095   const auto tagsDataSet = chosenTag->findDataSet("Tags");
0096   const auto recordsGroup = file_.findGroup("Records");
0097
0098   std::vector<hobj_ref_t> tags = tagsDataSet->readRefs();
0099
0100   std::set<std::string> recordsToExclude;
0101   {
0102     auto exclude = iPSet.getParameter<std::vector<std::string>>("excludeRecords");
0103     recordsToExclude = std::set(exclude.begin(), exclude.end());
0104   }
0105
0106   for (auto t : tags) {
0107     auto tagGroup = file_.derefGroup(t);
0108     Record record;
0109     record.name_ = tagGroup->findAttribute("record")->readString();
0110     //std::cout << record.name_ << std::endl;
0111
0112     if (recordsToExclude.end() != recordsToExclude.find(record.name_)) {
0113       //std::cout << "excluding " << record.name_ << std::endl;
0114       continue;
0115     }
0116
0117     auto recordGroup = recordsGroup->findGroup(record.name_);
0118     //std::cout << "found record group" << std::endl;
0119     auto dataProductsGroup = recordGroup->findGroup("DataProducts");
0120     //std::cout << "found DataProducts group" << std::endl;
0121
0122     for (size_t i = 0; i < dataProductsGroup->getNumObjs(); ++i) {
0123       std::string productGroupName = dataProductsGroup->getObjnameByIdx(i);
0124       //std::cout << "looking for " << productGroupName << std::endl;
0125       auto dataProductGroup = dataProductsGroup->findGroup(productGroupName);
0126
0127       auto const typeAttr = dataProductGroup->findAttribute("type");
0128       std::string typeName = typeAttr->readString();
0129       //loading the factory should also trigger registering the Record and DataProduct keys
0130       cond::serialization::SerializationHelperFactory::get()->create(typeName);
0131       std::string name = productGroupName.substr(typeName.size() + 1, productGroupName.size());
0132       if (name.size() == 1 and name[0] == '-') {
0133         name = std::string();
0134       }
0135       record.dataProducts_.emplace_back(std::move(name), std::move(typeName));
0136     }
0137
0138     {
0139       auto const typeAttr = tagGroup->findAttribute("time_type");
0140       std::string typeName = typeAttr->readString();
0141       record.iovIsRunLumi_ = (typeName == "run_lumi");
0142     }
0143
0144     std::vector<hobj_ref_t> payloadRefForIOVs;
0145     {
0146       auto const firstDataSet = tagGroup->findDataSet("first");
0147       auto const lastDataSet = tagGroup->findDataSet("last");
0148
0149       record.iovFirsts_ = firstDataSet->readSyncValues();
0150       record.iovLasts_ = lastDataSet->readSyncValues();
0151
0152       {
0153         auto const payloadDataSet = tagGroup->findDataSet("payload");
0154         payloadRefForIOVs = payloadDataSet->readRefs();
0155         assert(payloadRefForIOVs.size() == record.iovFirsts_.size() * record.dataProducts_.size());
0156       }
0157     }
0158     size_t dataProductIndex = 0;
0159     for (auto r : payloadRefForIOVs) {
0160       record.dataProducts_[dataProductIndex].payloadForIOVs_.push_back(r);
0161       ++dataProductIndex;
0162       if (dataProductIndex >= record.dataProducts_.size()) {
0163         dataProductIndex = 0;
0164       }
0165     }
0166
0167     //now that we've loaded a plugin that is associated to the record, the record should be registered
0168     auto key =
0169         edm::eventsetup::EventSetupRecordKey(edm::eventsetup::heterocontainer::HCTypeTag::findType(record.name_));
0170     assert(key != edm::eventsetup::heterocontainer::HCTypeTag());
0171     //tell system we retrieve this Record
0172     findingRecordWithKey(key);
0173     usingRecordWithKey(key);
0174
0175     records_.emplace_back(std::move(record));
0176   }
0177   std::sort(records_.begin(), records_.end(), [](auto const& l, auto const& r) { return l.name_ < r.name_; });
0178 }
0179
0180 void CondHDF5ESSource::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0181   edm::ParameterSetDescription desc;
0182   desc.addUntracked<std::string>("filename")->setComment("HDF5 file containing the conditions");
0183   desc.add<std::string>("globalTag")->setComment("Which global tag to use from the file");
0184   desc.add<std::vector<std::string>>("excludeRecords", std::vector<std::string>())
0185       ->setComment("List of Records that should not be read from the file");
0186
0187   descriptions.addDefault(desc);
0188 }
0189
0190 void CondHDF5ESSource::setIntervalFor(EventSetupRecordKey const& iRecordKey,
0191                                       edm::IOVSyncValue const& iSync,
0192                                       edm::ValidityInterval& iIOV) {
0193   using namespace cond::hdf5;
0194
0195   auto const itRecord =
0196       std::lower_bound(records_.begin(), records_.end(), iRecordKey.name(), [](auto const& iE, auto const& iV) {
0197         return iE.name_ < iV;
0198       });
0199   assert(itRecord != records_.end());
0200   auto const& record = *itRecord;
0201   assert(record.name_ == iRecordKey.name());
0202   auto sync = convertSyncValue(iSync, record.iovIsRunLumi_);
0203   auto itFound = findMatchingFirst(record.iovFirsts_, sync);
0204   if (itFound == record.iovFirsts_.end()) {
0205     //std::cout << "BAD SYNC for record " << iRecordKey.name() << std::endl;
0206     iIOV = edm::ValidityInterval::invalidInterval();
0207     return;
0208   }
0209   iIOV = edm::ValidityInterval{
0210       convertSyncValue(*itFound, record.iovIsRunLumi_),
0211       convertSyncValue(record.iovLasts_[itFound - record.iovFirsts_.begin()], record.iovIsRunLumi_)};
0212 }
0213
0214 CondHDF5ESSource::KeyedResolversVector CondHDF5ESSource::registerResolvers(EventSetupRecordKey const& iRecordKey,
0215                                                                            unsigned int iovIndex) {
0216   CondHDF5ESSource::KeyedResolversVector returnValue;
0217
0218   //std::cout << "Register proxies called " << iRecordKey.name() << std::endl;
0219   auto const itRecord =
0220       std::lower_bound(records_.begin(), records_.end(), iRecordKey.name(), [](auto const& iE, auto const& iV) {
0221         return iE.name_ < iV;
0222       });
0223   assert(itRecord != records_.end());
0224   auto const& record = *itRecord;
0225   assert(record.name_ == iRecordKey.name());
0226   for (auto const& dataProduct : record.dataProducts_) {
0227     //std::cout << "Making DataProduct " << dataProduct.type_ << " '" << dataProduct.name_ << "' for Record "
0228     //          << record.name_ << std::endl;
0229     auto helper = cond::serialization::SerializationHelperFactory::get()->create(dataProduct.type_);
0230     returnValue.emplace_back(
0231         edm::eventsetup::DataKey(edm::eventsetup::heterocontainer::HCTypeTag::findType(dataProduct.type_),
0232                                  dataProduct.name_.c_str()),
0233         std::make_shared<HDF5ProductResolver>(
0234             &queue_, std::move(helper), &file_, filename_, compression_, &record, &dataProduct));
0235   }
0236   return returnValue;
0237 }
0238
0239 DEFINE_FWK_EVENTSETUP_SOURCE(CondHDF5ESSource);