Framework/interface/OutputModuleCore.h

0001 #ifndef FWCore_Framework_one_OutputModuleCore_h
0002 #define FWCore_Framework_one_OutputModuleCore_h
0003 // -*- C++ -*-
0004 //
0005 // Package:     FWCore/Framework
0006 // Class  :     OutputModuleCore
0007 //
0008 /**\class OutputModuleCore OutputModuleCore.h "FWCore/Framework/interface/one/OutputModuleCore.h"
0009
0010  Description: Base class for all 'one' OutputModules
0011
0012  Usage:
0013     <usage>
0014
0015 */
0016 //
0017 // Original Author:  Chris Jones
0018 //         Created:  Wed, 31 Jul 2013 15:37:16 GMT
0019 //
0020
0021 // system include files
0022 #include <array>
0023 #include <functional>
0024 #include <memory>
0025 #include <string>
0026 #include <vector>
0027 #include <map>
0028 #include <atomic>
0029 #include <set>
0030
0031 // user include files
0032 #include "DataFormats/Provenance/interface/BranchDescription.h"
0033 #include "DataFormats/Provenance/interface/BranchID.h"
0034 #include "DataFormats/Provenance/interface/BranchIDList.h"
0035 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0036 #include "DataFormats/Provenance/interface/SelectedProducts.h"
0037
0038 #include "FWCore/Common/interface/FWCoreCommonFwd.h"
0039 #include "FWCore/Common/interface/OutputProcessBlockHelper.h"
0040 #include "FWCore/Framework/interface/TriggerResultsBasedEventSelector.h"
0041 #include "FWCore/Framework/interface/Frameworkfwd.h"
0042 #include "FWCore/Framework/interface/ProductSelectorRules.h"
0043 #include "FWCore/Framework/interface/ProductSelector.h"
0044 #include "FWCore/Framework/interface/EDConsumerBase.h"
0045 #include "FWCore/Framework/interface/getAllTriggerNames.h"
0046 #include "FWCore/ParameterSet/interface/ParameterSetfwd.h"
0047 #include "FWCore/Concurrency/interface/WaitingTaskHolder.h"
0048 #include "FWCore/Utilities/interface/propagate_const.h"
0049
0050 // forward declarations
0051 namespace edm {
0052
0053   class MergeableRunProductMetadata;
0054   class ModuleCallingContext;
0055   class PreallocationConfiguration;
0056   class ActivityRegistry;
0057   class ThinnedAssociationsHelper;
0058
0059   template <typename T>
0060   class OutputModuleCommunicatorT;
0061
0062   namespace maker {
0063     template <typename T>
0064     class ModuleHolderT;
0065   }
0066
0067   namespace core {
0068
0069     class OutputModuleCore : public EDConsumerBase {
0070     public:
0071       template <typename U>
0072       friend class edm::maker::ModuleHolderT;
0073       template <typename T>
0074       friend class ::edm::WorkerT;
0075       template <typename T>
0076       friend class ::edm::OutputModuleCommunicatorT;
0077       typedef OutputModuleCore ModuleType;
0078
0079       explicit OutputModuleCore(ParameterSet const& pset);
0080       ~OutputModuleCore() override;
0081
0082       OutputModuleCore(OutputModuleCore const&) = delete;             // Disallow copying and moving
0083       OutputModuleCore& operator=(OutputModuleCore const&) = delete;  // Disallow copying and moving
0084
0085       /// Accessor for maximum number of events to be written.
0086       /// -1 is used for unlimited.
0087       int maxEvents() const { return maxEvents_; }
0088
0089       /// Accessor for remaining number of events to be written.
0090       /// -1 is used for unlimited.
0091       int remainingEvents() const { return remainingEvents_; }
0092
0093       bool selected(BranchDescription const& desc) const;
0094
0095       void selectProducts(ProductRegistry const& preg, ThinnedAssociationsHelper const&, ProcessBlockHelperBase const&);
0096       std::string const& processName() const { return process_name_; }
0097       SelectedProductsForBranchType const& keptProducts() const { return keptProducts_; }
0098       std::array<bool, NumBranchTypes> const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; }
0099
0100       static void fillDescription(
0101           ParameterSetDescription& desc,
0102           std::vector<std::string> const& iDefaultOutputCommands = ProductSelectorRules::defaultSelectionStrings());
0103       static void fillDescriptions(ConfigurationDescriptions& descriptions);
0104       static const std::string& baseType();
0105       static void prevalidate(ConfigurationDescriptions&);
0106
0107       bool wantAllEvents() const { return wantAllEvents_; }
0108
0109       BranchIDLists const* branchIDLists() const;
0110
0111       OutputProcessBlockHelper const& outputProcessBlockHelper() const { return outputProcessBlockHelper_; }
0112
0113       ThinnedAssociationsHelper const* thinnedAssociationsHelper() const;
0114
0115       const ModuleDescription& moduleDescription() const { return moduleDescription_; }
0116
0117       void callWhenNewProductsRegistered(std::function<void(BranchDescription const&)> const& func) {
0118         callWhenNewProductsRegistered_ = func;
0119       }
0120
0121     protected:
0122       ModuleDescription const& description() const;
0123
0124       ParameterSetID selectorConfig() const { return selector_config_id_; }
0125
0126       void doPreallocate_(PreallocationConfiguration const&);
0127       virtual void preallocLumis(unsigned int);
0128
0129       void doBeginJob_();
0130       void doEndJob();
0131       bool doEvent_(EventTransitionInfo const&, ActivityRegistry*, ModuleCallingContext const*);
0132       void doBeginProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {}
0133       void doAccessInputProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {}
0134       void doEndProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {}
0135       bool doBeginRun(RunTransitionInfo const&, ModuleCallingContext const*);
0136       bool doEndRun(RunTransitionInfo const&, ModuleCallingContext const*);
0137       bool doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*);
0138       bool doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*);
0139
0140       void setEventSelectionInfo(
0141           std::map<std::string, std::vector<std::pair<std::string, int>>> const& outputModulePathPositions,
0142           bool anyProductProduced);
0143
0144       void configure(OutputModuleDescription const& desc);
0145
0146       std::map<BranchID::value_type, BranchID::value_type> const& droppedBranchIDToKeptBranchID() {
0147         return droppedBranchIDToKeptBranchID_;
0148       }
0149
0150       //inheriting classes decrement this in doEvent in a manner that will be thread-safe for that class
0151       std::atomic<int> remainingEvents_;
0152
0153     private:
0154       int maxEvents_;
0155
0156       // TODO: Give OutputModule
0157       // an interface (protected?) that supplies client code with the
0158       // needed functionality *without* giving away implementation
0159       // details ... don't just return a reference to keptProducts_, because
0160       // we are looking to have the flexibility to change the
0161       // implementation of keptProducts_ without modifying clients. When this
0162       // change is made, we'll have a one-time-only task of modifying
0163       // clients (classes derived from OutputModule) to use the
0164       // newly-introduced interface.
0165       // TODO: Consider using shared pointers here?
0166
0167       // keptProducts_ are pointers to the BranchDescription objects describing
0168       // the branches we are to write.
0169       //
0170       // We do not own the BranchDescriptions to which we point.
0171       SelectedProductsForBranchType keptProducts_;
0172       std::array<bool, NumBranchTypes> hasNewlyDroppedBranch_;
0173
0174       std::string process_name_;
0175       ProductSelectorRules productSelectorRules_;
0176       ProductSelector productSelector_;
0177       ModuleDescription moduleDescription_;
0178
0179       bool wantAllEvents_;
0180       std::vector<detail::TriggerResultsBasedEventSelector> selectors_;
0181       ParameterSet selectEvents_;
0182       std::vector<EDGetToken> tokensForEndPaths_;  //needed for FinalPath
0183       bool onFinalPath_ = false;
0184       // ID of the ParameterSet that configured the event selector
0185       // subsystem.
0186       ParameterSetID selector_config_id_;
0187
0188       // needed because of possible EDAliases.
0189       // filled in only if key and value are different.
0190       std::map<BranchID::value_type, BranchID::value_type> droppedBranchIDToKeptBranchID_;
0191       edm::propagate_const<std::unique_ptr<BranchIDLists>> branchIDLists_;
0192       BranchIDLists const* origBranchIDLists_;
0193
0194       edm::propagate_const<std::unique_ptr<ThinnedAssociationsHelper>> thinnedAssociationsHelper_;
0195       std::map<BranchID, bool> keepAssociation_;
0196
0197       OutputProcessBlockHelper outputProcessBlockHelper_;
0198
0199       std::function<void(BranchDescription const&)> callWhenNewProductsRegistered_;
0200
0201       //------------------------------------------------------------------
0202       // private member functions
0203       //------------------------------------------------------------------
0204       void updateBranchIDListsWithKeptAliases();
0205
0206       void doWriteProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*);
0207       void doWriteRun(RunPrincipal const& rp, ModuleCallingContext const*, MergeableRunProductMetadata const*);
0208       void doWriteLuminosityBlock(LuminosityBlockPrincipal const& lbp, ModuleCallingContext const*);
0209       void doOpenFile(FileBlock const& fb);
0210       void doRespondToOpenInputFile(FileBlock const& fb);
0211       void doRespondToCloseInputFile(FileBlock const& fb);
0212       void doRespondToCloseOutputFile() {}
0213       void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {}
0214
0215       /// Tell the OutputModule that is must end the current file.
0216       void doCloseFile();
0217
0218       void registerProductsAndCallbacks(OutputModuleCore const*, ProductRegistry*);
0219
0220       bool needToRunSelection() const;
0221       std::vector<ProductResolverIndexAndSkipBit> productsUsedBySelection() const;
0222       bool prePrefetchSelection(StreamID id, EventPrincipal const&, ModuleCallingContext const*);
0223
0224       // Do the end-of-file tasks; this is only called internally, after
0225       // the appropriate tests have been done.
0226       virtual void reallyCloseFile();
0227
0228       /// Ask the OutputModule if we should end the current file.
0229       virtual bool shouldWeCloseFile() const { return false; }
0230
0231       virtual void write(EventForOutput const&) = 0;
0232
0233       virtual void beginJob() {}
0234       virtual void endJob() {}
0235       virtual void writeLuminosityBlock(LuminosityBlockForOutput const&) = 0;
0236       virtual void writeRun(RunForOutput const&) = 0;
0237       virtual void writeProcessBlock(ProcessBlockForOutput const&) {}
0238       virtual void openFile(FileBlock const&) {}
0239       virtual bool isFileOpen() const { return true; }
0240
0241       virtual void doBeginRun_(RunForOutput const&) {}
0242       virtual void doEndRun_(RunForOutput const&) {}
0243       virtual void doBeginLuminosityBlock_(LuminosityBlockForOutput const&) {}
0244       virtual void doEndLuminosityBlock_(LuminosityBlockForOutput const&) {}
0245       virtual void doRespondToOpenInputFile_(FileBlock const&) {}
0246       virtual void doRespondToCloseInputFile_(FileBlock const&) {}
0247
0248       virtual void setProcessesWithSelectedMergeableRunProducts(std::set<std::string> const&) {}
0249
0250       bool hasAccumulator() const { return false; }
0251
0252       void keepThisBranch(BranchDescription const& desc,
0253                           std::map<BranchID, BranchDescription const*>& trueBranchIDToKeptBranchDesc,
0254                           std::set<BranchID>& keptProductsInEvent);
0255
0256       void setModuleDescription(ModuleDescription const& md) { moduleDescription_ = md; }
0257
0258       bool limitReached() const { return remainingEvents_ == 0; }
0259     };
0260   }  // namespace core
0261 }  // namespace edm
0262 #endif