Output/interface/PoolOutputModule.h

0001 #ifndef IOPool_Output_PoolOutputModule_h
0002 #define IOPool_Output_PoolOutputModule_h
0003
0004 //////////////////////////////////////////////////////////////////////
0005 //
0006 // Class PoolOutputModule. Output module to POOL file
0007 //
0008 // Oringinal Author: Luca Lista
0009 // Current Author: Bill Tanenbaum
0010 //
0011 //////////////////////////////////////////////////////////////////////
0012
0013 #include <array>
0014 #include <map>
0015 #include <memory>
0016 #include <set>
0017 #include <string>
0018 #include <vector>
0019 #include <regex>
0020
0021 #include "IOPool/Common/interface/RootServiceChecker.h"
0022 #include "FWCore/Framework/interface/Frameworkfwd.h"
0023 #include "FWCore/Framework/interface/one/OutputModule.h"
0024 #include "FWCore/Utilities/interface/BranchType.h"
0025 #include "FWCore/Utilities/interface/propagate_const.h"
0026 #include "DataFormats/Provenance/interface/BranchChildren.h"
0027 #include "DataFormats/Provenance/interface/BranchID.h"
0028 #include "DataFormats/Provenance/interface/BranchType.h"
0029 #include "DataFormats/Provenance/interface/ParentageID.h"
0030
0031 class TTree;
0032 namespace edm {
0033
0034   class EDGetToken;
0035   class ModuleCallingContext;
0036   class ParameterSet;
0037   class RootOutputFile;
0038   class ConfigurationDescriptions;
0039   class ProductProvenanceRetriever;
0040
0041   class PoolOutputModule : public one::OutputModule<WatchInputFiles> {
0042   public:
0043     enum DropMetaData { DropNone, DropDroppedPrior, DropPrior, DropAll };
0044     explicit PoolOutputModule(ParameterSet const& ps);
0045     ~PoolOutputModule() override;
0046     PoolOutputModule(PoolOutputModule const&) = delete;             // Disallow copying and moving
0047     PoolOutputModule& operator=(PoolOutputModule const&) = delete;  // Disallow copying and moving
0048     std::string const& fileName() const { return fileName_; }
0049     std::string const& logicalFileName() const { return logicalFileName_; }
0050     int compressionLevel() const { return compressionLevel_; }
0051     std::string const& compressionAlgorithm() const { return compressionAlgorithm_; }
0052     int basketSize() const { return basketSize_; }
0053     int eventAuxiliaryBasketSize() const { return eventAuxBasketSize_; }
0054     int eventAutoFlushSize() const { return eventAutoFlushSize_; }
0055     int splitLevel() const { return splitLevel_; }
0056     std::string const& basketOrder() const { return basketOrder_; }
0057     int treeMaxVirtualSize() const { return treeMaxVirtualSize_; }
0058     bool overrideInputFileSplitLevels() const { return overrideInputFileSplitLevels_; }
0059     bool compactEventAuxiliary() const { return compactEventAuxiliary_; }
0060     bool mergeJob() const { return mergeJob_; }
0061     DropMetaData const& dropMetaData() const { return dropMetaData_; }
0062     std::string const& catalog() const { return catalog_; }
0063     std::string const& moduleLabel() const { return moduleLabel_; }
0064     unsigned int maxFileSize() const { return maxFileSize_; }
0065     int inputFileCount() const { return inputFileCount_; }
0066     int whyNotFastClonable() const { return whyNotFastClonable_; }
0067
0068     std::string const& currentFileName() const;
0069
0070     static void fillDescription(ParameterSetDescription& desc);
0071     static void fillDescriptions(ConfigurationDescriptions& descriptions);
0072
0073     using OutputModule::selectorConfig;
0074
0075     struct AuxItem {
0076       AuxItem();
0077       ~AuxItem() {}
0078       int basketSize_;
0079     };
0080     using AuxItemArray = std::array<AuxItem, numberOfRunLumiEventProductTrees>;
0081     AuxItemArray const& auxItems() const { return auxItems_; }
0082
0083     struct OutputItem {
0084       class Sorter {
0085       public:
0086         explicit Sorter(TTree* tree);
0087         bool operator()(OutputItem const& lh, OutputItem const& rh) const;
0088
0089       private:
0090         std::shared_ptr<std::map<std::string, int>> treeMap_;
0091       };
0092
0093       explicit OutputItem(BranchDescription const* bd, EDGetToken const& token, int splitLevel, int basketSize);
0094
0095       BranchID branchID() const { return branchDescription_->branchID(); }
0096       std::string const& branchName() const { return branchDescription_->branchName(); }
0097
0098       bool operator<(OutputItem const& rh) const { return *branchDescription_ < *rh.branchDescription_; }
0099
0100       BranchDescription const* branchDescription() const { return branchDescription_; }
0101       EDGetToken token() const { return token_; }
0102       void const* const product() const { return product_; }
0103       void const*& product() { return product_; }
0104       void setProduct(void const* iProduct) { product_ = iProduct; }
0105       int splitLevel() const { return splitLevel_; }
0106       int basketSize() const { return basketSize_; }
0107
0108     private:
0109       BranchDescription const* branchDescription_;
0110       EDGetToken token_;
0111       void const* product_;
0112       int splitLevel_;
0113       int basketSize_;
0114     };
0115
0116     using OutputItemList = std::vector<OutputItem>;
0117
0118     struct SpecialSplitLevelForBranch {
0119       SpecialSplitLevelForBranch(std::string const& iBranchName, int iSplitLevel)
0120           : branch_(convert(iBranchName)),
0121             splitLevel_(iSplitLevel < 1 ? 1 : iSplitLevel)  //minimum is 1
0122       {}
0123       bool match(std::string const& iBranchName) const;
0124       std::regex convert(std::string const& iGlobBranchExpression) const;
0125
0126       std::regex branch_;
0127       int splitLevel_;
0128     };
0129
0130     std::vector<OutputItemList> const& selectedOutputItemList() const { return selectedOutputItemList_; }
0131
0132     std::vector<OutputItemList>& selectedOutputItemList() { return selectedOutputItemList_; }
0133
0134     BranchChildren const& branchChildren() const { return branchChildren_; }
0135
0136   protected:
0137     ///allow inheriting classes to override but still be able to call this method in the overridden version
0138     bool shouldWeCloseFile() const override;
0139     void write(EventForOutput const& e) override;
0140
0141     virtual std::pair<std::string, std::string> physicalAndLogicalNameForNewFile();
0142     virtual void doExtrasAfterCloseFile();
0143
0144   private:
0145     void preActionBeforeRunEventAsync(WaitingTaskHolder iTask,
0146                                       ModuleCallingContext const& iModuleCallingContext,
0147                                       Principal const& iPrincipal) const noexcept override;
0148
0149     void openFile(FileBlock const& fb) override;
0150     void respondToOpenInputFile(FileBlock const& fb) override;
0151     void respondToCloseInputFile(FileBlock const& fb) override;
0152     void writeLuminosityBlock(LuminosityBlockForOutput const&) override;
0153     void writeRun(RunForOutput const&) override;
0154     void writeProcessBlock(ProcessBlockForOutput const&) override;
0155     bool isFileOpen() const override;
0156     void reallyOpenFile();
0157     void reallyCloseFile() override;
0158     void beginJob() override;
0159
0160     void setProcessesWithSelectedMergeableRunProducts(std::set<std::string> const&) override;
0161
0162     using BranchParents = std::map<BranchID, std::set<ParentageID>>;
0163     void updateBranchParentsForOneBranch(ProductProvenanceRetriever const* provRetriever, BranchID const& branchID);
0164     void updateBranchParents(EventForOutput const& e);
0165     void fillDependencyGraph();
0166
0167     void startEndFile();
0168     void writeFileFormatVersion();
0169     void writeFileIdentifier();
0170     void writeIndexIntoFile();
0171     void writeStoredMergeableRunProductMetadata();
0172     void writeProcessHistoryRegistry();
0173     void writeParameterSetRegistry();
0174     void writeProductDescriptionRegistry();
0175     void writeParentageRegistry();
0176     void writeBranchIDListRegistry();
0177     void writeThinnedAssociationsHelper();
0178     void writeProductDependencies();
0179     void writeEventAuxiliary();
0180     void writeProcessBlockHelper();
0181     void finishEndFile();
0182
0183     void fillSelectedItemList(BranchType branchtype,
0184                               std::string const& processName,
0185                               TTree* theInputTree,
0186                               OutputItemList&);
0187     void beginInputFile(FileBlock const& fb);
0188
0189     RootServiceChecker rootServiceChecker_;
0190     AuxItemArray auxItems_;
0191     std::vector<OutputItemList> selectedOutputItemList_;
0192     std::vector<SpecialSplitLevelForBranch> specialSplitLevelForBranches_;
0193     std::string const fileName_;
0194     std::string const logicalFileName_;
0195     std::string const catalog_;
0196     unsigned int const maxFileSize_;
0197     int const compressionLevel_;
0198     std::string const compressionAlgorithm_;
0199     int const basketSize_;
0200     int const eventAuxBasketSize_;
0201     int const eventAutoFlushSize_;
0202     int const splitLevel_;
0203     std::string basketOrder_;
0204     int const treeMaxVirtualSize_;
0205     int whyNotFastClonable_;
0206     DropMetaData dropMetaData_;
0207     std::string const moduleLabel_;
0208     bool initializedFromInput_;
0209     int outputFileCount_;
0210     int inputFileCount_;
0211     BranchParents branchParents_;
0212     BranchChildren branchChildren_;
0213     std::vector<BranchID> producedBranches_;
0214     bool overrideInputFileSplitLevels_;
0215     bool compactEventAuxiliary_;
0216     bool mergeJob_;
0217     edm::propagate_const<std::unique_ptr<RootOutputFile>> rootOutputFile_;
0218     std::string statusFileName_;
0219     std::string overrideGUID_;
0220     std::vector<std::string> processesWithSelectedMergeableRunProducts_;
0221   };
0222 }  // namespace edm
0223
0224 #endif