File indexing completed on 2025-02-14 03:16:46
0001 #ifndef IOPool_Output_PoolOutputModule_h
0002 #define IOPool_Output_PoolOutputModule_h
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013 #include <array>
0014 #include <map>
0015 #include <memory>
0016 #include <set>
0017 #include <string>
0018 #include <vector>
0019 #include <regex>
0020
0021 #include "IOPool/Common/interface/RootServiceChecker.h"
0022 #include "FWCore/Framework/interface/Frameworkfwd.h"
0023 #include "FWCore/Framework/interface/one/OutputModule.h"
0024 #include "FWCore/Utilities/interface/BranchType.h"
0025 #include "FWCore/Utilities/interface/propagate_const.h"
0026 #include "DataFormats/Provenance/interface/ProductDependencies.h"
0027 #include "DataFormats/Provenance/interface/BranchID.h"
0028 #include "DataFormats/Provenance/interface/BranchType.h"
0029 #include "DataFormats/Provenance/interface/ParentageID.h"
0030 #include "DataFormats/Provenance/interface/ProductRegistry.h"
0031
0032 class TTree;
0033 namespace edm {
0034
0035 class EDGetToken;
0036 class ModuleCallingContext;
0037 class ParameterSet;
0038 class RootOutputFile;
0039 class ConfigurationDescriptions;
0040 class ProductProvenanceRetriever;
0041
0042 class PoolOutputModule : public one::OutputModule<WatchInputFiles> {
0043 public:
0044 enum DropMetaData { DropNone, DropDroppedPrior, DropPrior, DropAll };
0045 explicit PoolOutputModule(ParameterSet const& ps);
0046 ~PoolOutputModule() override;
0047 PoolOutputModule(PoolOutputModule const&) = delete;
0048 PoolOutputModule& operator=(PoolOutputModule const&) = delete;
0049 std::string const& fileName() const { return fileName_; }
0050 std::string const& logicalFileName() const { return logicalFileName_; }
0051 int compressionLevel() const { return compressionLevel_; }
0052 std::string const& compressionAlgorithm() const { return compressionAlgorithm_; }
0053 int basketSize() const { return basketSize_; }
0054 int eventAuxiliaryBasketSize() const { return eventAuxBasketSize_; }
0055 int eventAutoFlushSize() const { return eventAutoFlushSize_; }
0056 int splitLevel() const { return splitLevel_; }
0057 std::string const& basketOrder() const { return basketOrder_; }
0058 int treeMaxVirtualSize() const { return treeMaxVirtualSize_; }
0059 bool overrideInputFileSplitLevels() const { return overrideInputFileSplitLevels_; }
0060 bool compactEventAuxiliary() const { return compactEventAuxiliary_; }
0061 bool mergeJob() const { return mergeJob_; }
0062 DropMetaData const& dropMetaData() const { return dropMetaData_; }
0063 std::string const& catalog() const { return catalog_; }
0064 std::string const& moduleLabel() const { return moduleLabel_; }
0065 unsigned int maxFileSize() const { return maxFileSize_; }
0066 int inputFileCount() const { return inputFileCount_; }
0067 int whyNotFastClonable() const { return whyNotFastClonable_; }
0068
0069 std::string const& currentFileName() const;
0070
0071 static void fillDescription(ParameterSetDescription& desc);
0072 static void fillDescriptions(ConfigurationDescriptions& descriptions);
0073
0074 using OutputModule::selectorConfig;
0075
0076 struct AuxItem {
0077 AuxItem();
0078 ~AuxItem() {}
0079 int basketSize_;
0080 };
0081 using AuxItemArray = std::array<AuxItem, numberOfRunLumiEventProductTrees>;
0082 AuxItemArray const& auxItems() const { return auxItems_; }
0083
0084 struct OutputItem {
0085 class Sorter {
0086 public:
0087 explicit Sorter(TTree* tree);
0088 bool operator()(OutputItem const& lh, OutputItem const& rh) const;
0089
0090 private:
0091 std::shared_ptr<std::map<std::string, int>> treeMap_;
0092 };
0093
0094 explicit OutputItem(ProductDescription const* bd, EDGetToken const& token, int splitLevel, int basketSize);
0095
0096 BranchID branchID() const { return productDescription_->branchID(); }
0097 std::string const& branchName() const { return productDescription_->branchName(); }
0098
0099 bool operator<(OutputItem const& rh) const { return *productDescription_ < *rh.productDescription_; }
0100
0101 ProductDescription const* productDescription() const { return productDescription_; }
0102 EDGetToken token() const { return token_; }
0103 void const* const product() const { return product_; }
0104 void const*& product() { return product_; }
0105 void setProduct(void const* iProduct) { product_ = iProduct; }
0106 int splitLevel() const { return splitLevel_; }
0107 int basketSize() const { return basketSize_; }
0108
0109 private:
0110 ProductDescription const* productDescription_;
0111 EDGetToken token_;
0112 void const* product_;
0113 int splitLevel_;
0114 int basketSize_;
0115 };
0116
0117 using OutputItemList = std::vector<OutputItem>;
0118
0119 struct SpecialSplitLevelForBranch {
0120 SpecialSplitLevelForBranch(std::string const& iBranchName, int iSplitLevel)
0121 : branch_(convert(iBranchName)),
0122 splitLevel_(iSplitLevel < 1 ? 1 : iSplitLevel)
0123 {}
0124 bool match(std::string const& iBranchName) const;
0125 std::regex convert(std::string const& iGlobBranchExpression) const;
0126
0127 std::regex branch_;
0128 int splitLevel_;
0129 };
0130
0131 struct AliasForBranch {
0132 AliasForBranch(std::string const& iBranchName, std::string const& iAlias)
0133 : branch_{convert(iBranchName)}, alias_{iAlias} {}
0134
0135 bool match(std::string const& iBranchName) const;
0136 std::regex convert(std::string const& iGlobBranchExpression) const;
0137
0138 std::regex branch_;
0139 std::string alias_;
0140 };
0141
0142 std::vector<OutputItemList> const& selectedOutputItemList() const { return selectedOutputItemList_; }
0143
0144 std::vector<OutputItemList>& selectedOutputItemList() { return selectedOutputItemList_; }
0145
0146 ProductDependencies const& productDependencies() const { return productDependencies_; }
0147
0148 std::vector<AliasForBranch> const& aliasForBranches() const { return aliasForBranches_; }
0149
0150 protected:
0151
0152 bool shouldWeCloseFile() const override;
0153 void write(EventForOutput const& e) override;
0154
0155 virtual std::pair<std::string, std::string> physicalAndLogicalNameForNewFile();
0156 virtual void doExtrasAfterCloseFile();
0157
0158 private:
0159 void preActionBeforeRunEventAsync(WaitingTaskHolder iTask,
0160 ModuleCallingContext const& iModuleCallingContext,
0161 Principal const& iPrincipal) const noexcept override;
0162
0163 void openFile(FileBlock const& fb) override;
0164 void respondToOpenInputFile(FileBlock const& fb) override;
0165 void respondToCloseInputFile(FileBlock const& fb) override;
0166 void writeLuminosityBlock(LuminosityBlockForOutput const&) override;
0167 void writeRun(RunForOutput const&) override;
0168 void writeProcessBlock(ProcessBlockForOutput const&) override;
0169 bool isFileOpen() const override;
0170 void reallyOpenFile();
0171 void reallyCloseFile() override;
0172 void beginJob() override;
0173 void initialRegistry(edm::ProductRegistry const& iReg) override;
0174
0175 void setProcessesWithSelectedMergeableRunProducts(std::set<std::string> const&) override;
0176
0177 using BranchParents = std::map<BranchID, std::set<ParentageID>>;
0178 void updateBranchParentsForOneBranch(ProductProvenanceRetriever const* provRetriever, BranchID const& branchID);
0179 void updateBranchParents(EventForOutput const& e);
0180 void fillDependencyGraph();
0181
0182 void startEndFile();
0183 void writeFileFormatVersion();
0184 void writeFileIdentifier();
0185 void writeIndexIntoFile();
0186 void writeStoredMergeableRunProductMetadata();
0187 void writeProcessHistoryRegistry();
0188 void writeParameterSetRegistry();
0189 void writeProductDescriptionRegistry();
0190 void writeParentageRegistry();
0191 void writeBranchIDListRegistry();
0192 void writeThinnedAssociationsHelper();
0193 void writeProductDependencies();
0194 void writeEventAuxiliary();
0195 void writeProcessBlockHelper();
0196 void finishEndFile();
0197
0198 void fillSelectedItemList(BranchType branchtype,
0199 std::string const& processName,
0200 TTree* theInputTree,
0201 OutputItemList&);
0202 void beginInputFile(FileBlock const& fb);
0203
0204 RootServiceChecker rootServiceChecker_;
0205 AuxItemArray auxItems_;
0206 std::vector<OutputItemList> selectedOutputItemList_;
0207 std::vector<SpecialSplitLevelForBranch> specialSplitLevelForBranches_;
0208 std::vector<AliasForBranch> aliasForBranches_;
0209 std::unique_ptr<edm::ProductRegistry const> reg_;
0210 std::string const fileName_;
0211 std::string const logicalFileName_;
0212 std::string const catalog_;
0213 unsigned int const maxFileSize_;
0214 int const compressionLevel_;
0215 std::string const compressionAlgorithm_;
0216 int const basketSize_;
0217 int const eventAuxBasketSize_;
0218 int const eventAutoFlushSize_;
0219 int const splitLevel_;
0220 std::string basketOrder_;
0221 int const treeMaxVirtualSize_;
0222 int whyNotFastClonable_;
0223 DropMetaData dropMetaData_;
0224 std::string const moduleLabel_;
0225 bool initializedFromInput_;
0226 int outputFileCount_;
0227 int inputFileCount_;
0228 BranchParents branchParents_;
0229 ProductDependencies productDependencies_;
0230 std::vector<BranchID> producedBranches_;
0231 bool overrideInputFileSplitLevels_;
0232 bool compactEventAuxiliary_;
0233 bool mergeJob_;
0234 edm::propagate_const<std::unique_ptr<RootOutputFile>> rootOutputFile_;
0235 std::string statusFileName_;
0236 std::string overrideGUID_;
0237 std::vector<std::string> processesWithSelectedMergeableRunProducts_;
0238 };
0239 }
0240
0241 #endif