CondHDF5ESSource/plugins/CondHDF5ESSource.cc

0001 // -*- C++ -*-
0002 //
0003 // Package:     CondCore/HDF5ESSource
0004 // Class  :     CondHDF5ESSource
0005 //
0006 // Implementation:
0007 //     [Notes on implementation]
0008 //
0009 // Original Author:  Christopher Jones
0010 //         Created:  Fri, 16 Jun 2023 15:17:53 GMT
0011 //
0012
0013 // system include files
0014 #include <cassert>
0015 #include <iostream>
0016
0017 // user include files
0018 #include "FWCore/Framework/interface/EventSetupRecordIntervalFinder.h"
0019 #include "FWCore/Framework/interface/ESProductResolverProvider.h"
0020 #include "FWCore/Framework/interface/ESModuleProducesInfo.h"
0021 #include "FWCore/Framework/interface/IOVSyncValue.h"
0022 #include "FWCore/Framework/interface/SourceFactory.h"
0023 #include "FWCore/Framework/interface/ValidityInterval.h"
0024 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0025 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0026 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0027 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0028 #include "FWCore/Concurrency/interface/SerialTaskQueue.h"
0029
0030 #include "CondFormats/SerializationHelper/interface/SerializationHelperFactory.h"
0031
0032 #include "IOVSyncValue.h"
0033 #include "DataProduct.h"
0034 #include "Record.h"
0035 #include "HDF5ProductResolver.h"
0036 #include "convertSyncValue.h"
0037 #include "h5_File.h"
0038 #include "h5_Group.h"
0039 #include "h5_DataSet.h"
0040 #include "h5_Attribute.h"
0041 #include "Compression.h"
0042
0043 using namespace cond::hdf5;
0044
0045 class CondHDF5ESSource : public edm::EventSetupRecordIntervalFinder, public edm::eventsetup::ESProductResolverProvider {
0046 public:
0047   using EventSetupRecordKey = edm::eventsetup::EventSetupRecordKey;
0048   explicit CondHDF5ESSource(edm::ParameterSet const&);
0049
0050   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0051
0052 private:
0053   bool isConcurrentFinder() const final { return true; }
0054   void setIntervalFor(EventSetupRecordKey const&, edm::IOVSyncValue const&, edm::ValidityInterval&) final;
0055   KeyedResolversVector registerResolvers(EventSetupRecordKey const&, unsigned int iovIndex) final;
0056   std::vector<edm::eventsetup::ESModuleProducesInfo> producesInfo() const final;
0057
0058   edm::SerialTaskQueue queue_;
0059   std::mutex mutex_;
0060   std::vector<Record> records_;
0061   std::string filename_;
0062   cms::h5::File file_;
0063   Compression compression_ = Compression::kNone;
0064 };
0065
0066 //
0067 // constants, enums and typedefs
0068 //
0069
0070 //
0071 // static data member definitions
0072 //
0073 namespace {
0074   cond::hdf5::Compression nameToEnum(std::string const& iName) {
0075     if (iName == "zlib") {
0076       return Compression::kZLIB;
0077     } else if (iName == "lzma") {
0078       return Compression::kLZMA;
0079     } else if (iName == "none") {
0080       return Compression::kNone;
0081     } else {
0082       throw cms::Exception("BadCompressionType") << "unknown compression type used in file '" << iName << "'";
0083     }
0084     return Compression::kNone;
0085   }
0086 }  // namespace
0087
0088 //
0089 // constructors and destructor
0090 //
0091 CondHDF5ESSource::CondHDF5ESSource(edm::ParameterSet const& iPSet)
0092     : filename_(iPSet.getUntrackedParameter<std::string>("filename")),
0093       file_(filename_, cms::h5::File::kReadOnly),
0094       compression_(nameToEnum(file_.findAttribute("default_payload_compressor")->readString())) {
0095   const auto globalTagsGroup = file_.findGroup("GlobalTags");
0096   const auto chosenTag = globalTagsGroup->findGroup(iPSet.getParameter<std::string>("globalTag"));
0097   const auto tagsDataSet = chosenTag->findDataSet("Tags");
0098   const auto recordsGroup = file_.findGroup("Records");
0099
0100   std::vector<hobj_ref_t> tags = tagsDataSet->readRefs();
0101
0102   std::set<std::string> recordsToExclude;
0103   {
0104     auto exclude = iPSet.getParameter<std::vector<std::string>>("excludeRecords");
0105     recordsToExclude = std::set(exclude.begin(), exclude.end());
0106   }
0107
0108   for (auto t : tags) {
0109     auto tagGroup = file_.derefGroup(t);
0110     Record record;
0111     record.name_ = tagGroup->findAttribute("record")->readString();
0112     //std::cout << record.name_ << std::endl;
0113
0114     if (recordsToExclude.end() != recordsToExclude.find(record.name_)) {
0115       //std::cout << "excluding " << record.name_ << std::endl;
0116       continue;
0117     }
0118
0119     auto recordGroup = recordsGroup->findGroup(record.name_);
0120     //std::cout << "found record group" << std::endl;
0121     auto dataProductsGroup = recordGroup->findGroup("DataProducts");
0122     //std::cout << "found DataProducts group" << std::endl;
0123
0124     for (size_t i = 0; i < dataProductsGroup->getNumObjs(); ++i) {
0125       std::string productGroupName = dataProductsGroup->getObjnameByIdx(i);
0126       //std::cout << "looking for " << productGroupName << std::endl;
0127       auto dataProductGroup = dataProductsGroup->findGroup(productGroupName);
0128
0129       auto const typeAttr = dataProductGroup->findAttribute("type");
0130       std::string typeName = typeAttr->readString();
0131       //loading the factory should also trigger registering the Record and DataProduct keys
0132       cond::serialization::SerializationHelperFactory::get()->create(typeName);
0133       std::string name = productGroupName.substr(typeName.size() + 1, productGroupName.size());
0134       if (name.size() == 1 and name[0] == '-') {
0135         name = std::string();
0136       }
0137       record.dataProducts_.emplace_back(std::move(name), std::move(typeName));
0138     }
0139
0140     {
0141       auto const typeAttr = tagGroup->findAttribute("time_type");
0142       std::string typeName = typeAttr->readString();
0143       record.iovIsRunLumi_ = (typeName == "run_lumi");
0144     }
0145
0146     std::vector<hobj_ref_t> payloadRefForIOVs;
0147     {
0148       auto const firstDataSet = tagGroup->findDataSet("first");
0149       auto const lastDataSet = tagGroup->findDataSet("last");
0150
0151       record.iovFirsts_ = firstDataSet->readSyncValues();
0152       record.iovLasts_ = lastDataSet->readSyncValues();
0153
0154       {
0155         auto const payloadDataSet = tagGroup->findDataSet("payload");
0156         payloadRefForIOVs = payloadDataSet->readRefs();
0157         assert(payloadRefForIOVs.size() == record.iovFirsts_.size() * record.dataProducts_.size());
0158       }
0159     }
0160     size_t dataProductIndex = 0;
0161     for (auto r : payloadRefForIOVs) {
0162       record.dataProducts_[dataProductIndex].payloadForIOVs_.push_back(r);
0163       ++dataProductIndex;
0164       if (dataProductIndex >= record.dataProducts_.size()) {
0165         dataProductIndex = 0;
0166       }
0167     }
0168
0169     //now that we've loaded a plugin that is associated to the record, the record should be registered
0170     auto key =
0171         edm::eventsetup::EventSetupRecordKey(edm::eventsetup::heterocontainer::HCTypeTag::findType(record.name_));
0172     assert(key != edm::eventsetup::heterocontainer::HCTypeTag());
0173     //tell system we retrieve this Record
0174     findingRecordWithKey(key);
0175     usingRecordWithKey(key);
0176
0177     records_.emplace_back(std::move(record));
0178   }
0179   std::sort(records_.begin(), records_.end(), [](auto const& l, auto const& r) { return l.name_ < r.name_; });
0180 }
0181
0182 void CondHDF5ESSource::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0183   edm::ParameterSetDescription desc;
0184   desc.addUntracked<std::string>("filename")->setComment("HDF5 file containing the conditions");
0185   desc.add<std::string>("globalTag")->setComment("Which global tag to use from the file");
0186   desc.add<std::vector<std::string>>("excludeRecords", std::vector<std::string>())
0187       ->setComment("List of Records that should not be read from the file");
0188
0189   descriptions.addDefault(desc);
0190 }
0191
0192 void CondHDF5ESSource::setIntervalFor(EventSetupRecordKey const& iRecordKey,
0193                                       edm::IOVSyncValue const& iSync,
0194                                       edm::ValidityInterval& iIOV) {
0195   using namespace cond::hdf5;
0196
0197   auto const itRecord =
0198       std::lower_bound(records_.begin(), records_.end(), iRecordKey.name(), [](auto const& iE, auto const& iV) {
0199         return iE.name_ < iV;
0200       });
0201   assert(itRecord != records_.end());
0202   auto const& record = *itRecord;
0203   assert(record.name_ == iRecordKey.name());
0204   auto sync = convertSyncValue(iSync, record.iovIsRunLumi_);
0205   auto itFound = findMatchingFirst(record.iovFirsts_, sync);
0206   if (itFound == record.iovFirsts_.end()) {
0207     //std::cout << "BAD SYNC for record " << iRecordKey.name() << std::endl;
0208     iIOV = edm::ValidityInterval::invalidInterval();
0209     return;
0210   }
0211   iIOV = edm::ValidityInterval{
0212       convertSyncValue(*itFound, record.iovIsRunLumi_),
0213       convertSyncValue(record.iovLasts_[itFound - record.iovFirsts_.begin()], record.iovIsRunLumi_)};
0214 }
0215
0216 CondHDF5ESSource::KeyedResolversVector CondHDF5ESSource::registerResolvers(EventSetupRecordKey const& iRecordKey,
0217                                                                            unsigned int iovIndex) {
0218   CondHDF5ESSource::KeyedResolversVector returnValue;
0219
0220   //std::cout << "Register proxies called " << iRecordKey.name() << std::endl;
0221   auto const itRecord =
0222       std::lower_bound(records_.begin(), records_.end(), iRecordKey.name(), [](auto const& iE, auto const& iV) {
0223         return iE.name_ < iV;
0224       });
0225   assert(itRecord != records_.end());
0226   auto const& record = *itRecord;
0227   assert(record.name_ == iRecordKey.name());
0228   for (auto const& dataProduct : record.dataProducts_) {
0229     //std::cout << "Making DataProduct " << dataProduct.type_ << " '" << dataProduct.name_ << "' for Record "
0230     //          << record.name_ << std::endl;
0231     auto helper = cond::serialization::SerializationHelperFactory::get()->create(dataProduct.type_);
0232     returnValue.emplace_back(
0233         edm::eventsetup::DataKey(edm::eventsetup::heterocontainer::HCTypeTag::findType(dataProduct.type_),
0234                                  dataProduct.name_.c_str()),
0235         std::make_shared<HDF5ProductResolver>(
0236             &queue_, std::move(helper), &file_, filename_, compression_, &record, &dataProduct));
0237   }
0238   return returnValue;
0239 }
0240
0241 std::vector<edm::eventsetup::ESModuleProducesInfo> CondHDF5ESSource::producesInfo() const {
0242   std::vector<edm::eventsetup::ESModuleProducesInfo> returnValue;
0243   auto size = 0;
0244   for (auto const& recInfo : records_) {
0245     size += recInfo.dataProducts_.size();
0246   }
0247   returnValue.reserve(size);
0248
0249   for (auto const& recInfo : records_) {
0250     EventSetupRecordKey rec{edm::eventsetup::heterocontainer::HCTypeTag::findType(recInfo.name_)};
0251     for (auto const& dataProduct : recInfo.dataProducts_) {
0252       unsigned int index = returnValue.size();
0253       edm::eventsetup::DataKey key{edm::eventsetup::heterocontainer::HCTypeTag::findType(dataProduct.type_),
0254                                    dataProduct.name_.c_str()};
0255       returnValue.emplace_back(rec, key, index);
0256     }
0257   }
0258
0259   return returnValue;
0260 }
0261
0262 DEFINE_FWK_EVENTSETUP_SOURCE(CondHDF5ESSource);