Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2025-02-14 03:16:46

0001 #ifndef IOPool_Output_PoolOutputModule_h
0002 #define IOPool_Output_PoolOutputModule_h
0003 
0004 //////////////////////////////////////////////////////////////////////
0005 //
0006 // Class PoolOutputModule. Output module to POOL file
0007 //
0008 // Oringinal Author: Luca Lista
0009 // Current Author: Bill Tanenbaum
0010 //
0011 //////////////////////////////////////////////////////////////////////
0012 
0013 #include <array>
0014 #include <map>
0015 #include <memory>
0016 #include <set>
0017 #include <string>
0018 #include <vector>
0019 #include <regex>
0020 
0021 #include "IOPool/Common/interface/RootServiceChecker.h"
0022 #include "FWCore/Framework/interface/Frameworkfwd.h"
0023 #include "FWCore/Framework/interface/one/OutputModule.h"
0024 #include "FWCore/Utilities/interface/BranchType.h"
0025 #include "FWCore/Utilities/interface/propagate_const.h"
0026 #include "DataFormats/Provenance/interface/ProductDependencies.h"
0027 #include "DataFormats/Provenance/interface/BranchID.h"
0028 #include "DataFormats/Provenance/interface/BranchType.h"
0029 #include "DataFormats/Provenance/interface/ParentageID.h"
0030 #include "DataFormats/Provenance/interface/ProductRegistry.h"
0031 
0032 class TTree;
0033 namespace edm {
0034 
0035   class EDGetToken;
0036   class ModuleCallingContext;
0037   class ParameterSet;
0038   class RootOutputFile;
0039   class ConfigurationDescriptions;
0040   class ProductProvenanceRetriever;
0041 
0042   class PoolOutputModule : public one::OutputModule<WatchInputFiles> {
0043   public:
0044     enum DropMetaData { DropNone, DropDroppedPrior, DropPrior, DropAll };
0045     explicit PoolOutputModule(ParameterSet const& ps);
0046     ~PoolOutputModule() override;
0047     PoolOutputModule(PoolOutputModule const&) = delete;             // Disallow copying and moving
0048     PoolOutputModule& operator=(PoolOutputModule const&) = delete;  // Disallow copying and moving
0049     std::string const& fileName() const { return fileName_; }
0050     std::string const& logicalFileName() const { return logicalFileName_; }
0051     int compressionLevel() const { return compressionLevel_; }
0052     std::string const& compressionAlgorithm() const { return compressionAlgorithm_; }
0053     int basketSize() const { return basketSize_; }
0054     int eventAuxiliaryBasketSize() const { return eventAuxBasketSize_; }
0055     int eventAutoFlushSize() const { return eventAutoFlushSize_; }
0056     int splitLevel() const { return splitLevel_; }
0057     std::string const& basketOrder() const { return basketOrder_; }
0058     int treeMaxVirtualSize() const { return treeMaxVirtualSize_; }
0059     bool overrideInputFileSplitLevels() const { return overrideInputFileSplitLevels_; }
0060     bool compactEventAuxiliary() const { return compactEventAuxiliary_; }
0061     bool mergeJob() const { return mergeJob_; }
0062     DropMetaData const& dropMetaData() const { return dropMetaData_; }
0063     std::string const& catalog() const { return catalog_; }
0064     std::string const& moduleLabel() const { return moduleLabel_; }
0065     unsigned int maxFileSize() const { return maxFileSize_; }
0066     int inputFileCount() const { return inputFileCount_; }
0067     int whyNotFastClonable() const { return whyNotFastClonable_; }
0068 
0069     std::string const& currentFileName() const;
0070 
0071     static void fillDescription(ParameterSetDescription& desc);
0072     static void fillDescriptions(ConfigurationDescriptions& descriptions);
0073 
0074     using OutputModule::selectorConfig;
0075 
0076     struct AuxItem {
0077       AuxItem();
0078       ~AuxItem() {}
0079       int basketSize_;
0080     };
0081     using AuxItemArray = std::array<AuxItem, numberOfRunLumiEventProductTrees>;
0082     AuxItemArray const& auxItems() const { return auxItems_; }
0083 
0084     struct OutputItem {
0085       class Sorter {
0086       public:
0087         explicit Sorter(TTree* tree);
0088         bool operator()(OutputItem const& lh, OutputItem const& rh) const;
0089 
0090       private:
0091         std::shared_ptr<std::map<std::string, int>> treeMap_;
0092       };
0093 
0094       explicit OutputItem(ProductDescription const* bd, EDGetToken const& token, int splitLevel, int basketSize);
0095 
0096       BranchID branchID() const { return productDescription_->branchID(); }
0097       std::string const& branchName() const { return productDescription_->branchName(); }
0098 
0099       bool operator<(OutputItem const& rh) const { return *productDescription_ < *rh.productDescription_; }
0100 
0101       ProductDescription const* productDescription() const { return productDescription_; }
0102       EDGetToken token() const { return token_; }
0103       void const* const product() const { return product_; }
0104       void const*& product() { return product_; }
0105       void setProduct(void const* iProduct) { product_ = iProduct; }
0106       int splitLevel() const { return splitLevel_; }
0107       int basketSize() const { return basketSize_; }
0108 
0109     private:
0110       ProductDescription const* productDescription_;
0111       EDGetToken token_;
0112       void const* product_;
0113       int splitLevel_;
0114       int basketSize_;
0115     };
0116 
0117     using OutputItemList = std::vector<OutputItem>;
0118 
0119     struct SpecialSplitLevelForBranch {
0120       SpecialSplitLevelForBranch(std::string const& iBranchName, int iSplitLevel)
0121           : branch_(convert(iBranchName)),
0122             splitLevel_(iSplitLevel < 1 ? 1 : iSplitLevel)  //minimum is 1
0123       {}
0124       bool match(std::string const& iBranchName) const;
0125       std::regex convert(std::string const& iGlobBranchExpression) const;
0126 
0127       std::regex branch_;
0128       int splitLevel_;
0129     };
0130 
0131     struct AliasForBranch {
0132       AliasForBranch(std::string const& iBranchName, std::string const& iAlias)
0133           : branch_{convert(iBranchName)}, alias_{iAlias} {}
0134 
0135       bool match(std::string const& iBranchName) const;
0136       std::regex convert(std::string const& iGlobBranchExpression) const;
0137 
0138       std::regex branch_;
0139       std::string alias_;
0140     };
0141 
0142     std::vector<OutputItemList> const& selectedOutputItemList() const { return selectedOutputItemList_; }
0143 
0144     std::vector<OutputItemList>& selectedOutputItemList() { return selectedOutputItemList_; }
0145 
0146     ProductDependencies const& productDependencies() const { return productDependencies_; }
0147 
0148     std::vector<AliasForBranch> const& aliasForBranches() const { return aliasForBranches_; }
0149 
0150   protected:
0151     ///allow inheriting classes to override but still be able to call this method in the overridden version
0152     bool shouldWeCloseFile() const override;
0153     void write(EventForOutput const& e) override;
0154 
0155     virtual std::pair<std::string, std::string> physicalAndLogicalNameForNewFile();
0156     virtual void doExtrasAfterCloseFile();
0157 
0158   private:
0159     void preActionBeforeRunEventAsync(WaitingTaskHolder iTask,
0160                                       ModuleCallingContext const& iModuleCallingContext,
0161                                       Principal const& iPrincipal) const noexcept override;
0162 
0163     void openFile(FileBlock const& fb) override;
0164     void respondToOpenInputFile(FileBlock const& fb) override;
0165     void respondToCloseInputFile(FileBlock const& fb) override;
0166     void writeLuminosityBlock(LuminosityBlockForOutput const&) override;
0167     void writeRun(RunForOutput const&) override;
0168     void writeProcessBlock(ProcessBlockForOutput const&) override;
0169     bool isFileOpen() const override;
0170     void reallyOpenFile();
0171     void reallyCloseFile() override;
0172     void beginJob() override;
0173     void initialRegistry(edm::ProductRegistry const& iReg) override;
0174 
0175     void setProcessesWithSelectedMergeableRunProducts(std::set<std::string> const&) override;
0176 
0177     using BranchParents = std::map<BranchID, std::set<ParentageID>>;
0178     void updateBranchParentsForOneBranch(ProductProvenanceRetriever const* provRetriever, BranchID const& branchID);
0179     void updateBranchParents(EventForOutput const& e);
0180     void fillDependencyGraph();
0181 
0182     void startEndFile();
0183     void writeFileFormatVersion();
0184     void writeFileIdentifier();
0185     void writeIndexIntoFile();
0186     void writeStoredMergeableRunProductMetadata();
0187     void writeProcessHistoryRegistry();
0188     void writeParameterSetRegistry();
0189     void writeProductDescriptionRegistry();
0190     void writeParentageRegistry();
0191     void writeBranchIDListRegistry();
0192     void writeThinnedAssociationsHelper();
0193     void writeProductDependencies();
0194     void writeEventAuxiliary();
0195     void writeProcessBlockHelper();
0196     void finishEndFile();
0197 
0198     void fillSelectedItemList(BranchType branchtype,
0199                               std::string const& processName,
0200                               TTree* theInputTree,
0201                               OutputItemList&);
0202     void beginInputFile(FileBlock const& fb);
0203 
0204     RootServiceChecker rootServiceChecker_;
0205     AuxItemArray auxItems_;
0206     std::vector<OutputItemList> selectedOutputItemList_;
0207     std::vector<SpecialSplitLevelForBranch> specialSplitLevelForBranches_;
0208     std::vector<AliasForBranch> aliasForBranches_;
0209     std::unique_ptr<edm::ProductRegistry const> reg_;
0210     std::string const fileName_;
0211     std::string const logicalFileName_;
0212     std::string const catalog_;
0213     unsigned int const maxFileSize_;
0214     int const compressionLevel_;
0215     std::string const compressionAlgorithm_;
0216     int const basketSize_;
0217     int const eventAuxBasketSize_;
0218     int const eventAutoFlushSize_;
0219     int const splitLevel_;
0220     std::string basketOrder_;
0221     int const treeMaxVirtualSize_;
0222     int whyNotFastClonable_;
0223     DropMetaData dropMetaData_;
0224     std::string const moduleLabel_;
0225     bool initializedFromInput_;
0226     int outputFileCount_;
0227     int inputFileCount_;
0228     BranchParents branchParents_;
0229     ProductDependencies productDependencies_;
0230     std::vector<BranchID> producedBranches_;
0231     bool overrideInputFileSplitLevels_;
0232     bool compactEventAuxiliary_;
0233     bool mergeJob_;
0234     edm::propagate_const<std::unique_ptr<RootOutputFile>> rootOutputFile_;
0235     std::string statusFileName_;
0236     std::string overrideGUID_;
0237     std::vector<std::string> processesWithSelectedMergeableRunProducts_;
0238   };
0239 }  // namespace edm
0240 
0241 #endif