Framework/interface/OutputModuleCore.h

0001 #ifndef FWCore_Framework_one_OutputModuleCore_h
0002 #define FWCore_Framework_one_OutputModuleCore_h
0003 // -*- C++ -*-
0004 //
0005 // Package:     FWCore/Framework
0006 // Class  :     OutputModuleCore
0007 //
0008 /**\class OutputModuleCore OutputModuleCore.h "FWCore/Framework/interface/one/OutputModuleCore.h"
0009
0010  Description: Base class for all 'one' OutputModules
0011
0012  Usage:
0013     <usage>
0014
0015 */
0016 //
0017 // Original Author:  Chris Jones
0018 //         Created:  Wed, 31 Jul 2013 15:37:16 GMT
0019 //
0020
0021 // system include files
0022 #include <array>
0023 #include <functional>
0024 #include <memory>
0025 #include <string>
0026 #include <vector>
0027 #include <map>
0028 #include <atomic>
0029 #include <set>
0030
0031 // user include files
0032 #include "DataFormats/Provenance/interface/ProductDescription.h"
0033 #include "DataFormats/Provenance/interface/BranchID.h"
0034 #include "DataFormats/Provenance/interface/BranchIDList.h"
0035 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0036 #include "DataFormats/Provenance/interface/SelectedProducts.h"
0037
0038 #include "FWCore/Common/interface/FWCoreCommonFwd.h"
0039 #include "FWCore/Common/interface/OutputProcessBlockHelper.h"
0040 #include "FWCore/Framework/interface/TriggerResultsBasedEventSelector.h"
0041 #include "FWCore/Framework/interface/Frameworkfwd.h"
0042 #include "FWCore/Framework/interface/ProductSelectorRules.h"
0043 #include "FWCore/Framework/interface/ProductSelector.h"
0044 #include "FWCore/Framework/interface/EDConsumerBase.h"
0045 #include "FWCore/Framework/interface/getAllTriggerNames.h"
0046 #include "FWCore/ParameterSet/interface/ParameterSetfwd.h"
0047 #include "FWCore/Concurrency/interface/WaitingTaskHolder.h"
0048 #include "FWCore/Utilities/interface/propagate_const.h"
0049
0050 // forward declarations
0051 namespace edm {
0052
0053   class MergeableRunProductMetadata;
0054   class ModuleCallingContext;
0055   class PreallocationConfiguration;
0056   class ActivityRegistry;
0057   class ThinnedAssociationsHelper;
0058   class SignallingProductRegistryFiller;
0059
0060   template <typename T>
0061   class OutputModuleCommunicatorT;
0062
0063   namespace maker {
0064     template <typename T>
0065     class ModuleHolderT;
0066   }
0067
0068   namespace core {
0069
0070     class OutputModuleCore : public EDConsumerBase {
0071     public:
0072       template <typename U>
0073       friend class edm::maker::ModuleHolderT;
0074       template <typename T>
0075       friend class ::edm::WorkerT;
0076       template <typename T>
0077       friend class ::edm::OutputModuleCommunicatorT;
0078       typedef OutputModuleCore ModuleType;
0079
0080       explicit OutputModuleCore(ParameterSet const& pset);
0081       ~OutputModuleCore() override;
0082
0083       OutputModuleCore(OutputModuleCore const&) = delete;             // Disallow copying and moving
0084       OutputModuleCore& operator=(OutputModuleCore const&) = delete;  // Disallow copying and moving
0085
0086       /// Accessor for maximum number of events to be written.
0087       /// -1 is used for unlimited.
0088       int maxEvents() const { return maxEvents_; }
0089
0090       /// Accessor for remaining number of events to be written.
0091       /// -1 is used for unlimited.
0092       int remainingEvents() const { return remainingEvents_; }
0093
0094       bool selected(ProductDescription const& desc) const;
0095
0096       void selectProducts(ProductRegistry const& preg, ThinnedAssociationsHelper const&, ProcessBlockHelperBase const&);
0097       std::string const& processName() const { return process_name_; }
0098       SelectedProductsForBranchType const& keptProducts() const { return keptProducts_; }
0099       std::array<bool, NumBranchTypes> const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; }
0100
0101       static void fillDescription(
0102           ParameterSetDescription& desc,
0103           std::vector<std::string> const& iDefaultOutputCommands = ProductSelectorRules::defaultSelectionStrings());
0104       static void fillDescriptions(ConfigurationDescriptions& descriptions);
0105       static const std::string& baseType();
0106       static void prevalidate(ConfigurationDescriptions&);
0107
0108       bool wantAllEvents() const { return wantAllEvents_; }
0109
0110       BranchIDLists const* branchIDLists() const;
0111
0112       OutputProcessBlockHelper const& outputProcessBlockHelper() const { return outputProcessBlockHelper_; }
0113
0114       ThinnedAssociationsHelper const* thinnedAssociationsHelper() const;
0115
0116       const ModuleDescription& moduleDescription() const { return moduleDescription_; }
0117
0118       void callWhenNewProductsRegistered(std::function<void(ProductDescription const&)> const& func) {
0119         callWhenNewProductsRegistered_ = func;
0120       }
0121
0122     protected:
0123       ModuleDescription const& description() const;
0124
0125       ParameterSetID selectorConfig() const { return selector_config_id_; }
0126
0127       void doPreallocate_(PreallocationConfiguration const&);
0128       virtual void preallocLumis(unsigned int);
0129
0130       void doBeginJob_();
0131       void doEndJob();
0132       bool doEvent_(EventTransitionInfo const&, ActivityRegistry*, ModuleCallingContext const*);
0133       void doBeginProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {}
0134       void doAccessInputProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {}
0135       void doEndProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {}
0136       bool doBeginRun(RunTransitionInfo const&, ModuleCallingContext const*);
0137       bool doEndRun(RunTransitionInfo const&, ModuleCallingContext const*);
0138       bool doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*);
0139       bool doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*);
0140
0141       void setEventSelectionInfo(
0142           std::map<std::string, std::vector<std::pair<std::string, int>>> const& outputModulePathPositions,
0143           bool anyProductProduced);
0144
0145       void configure(OutputModuleDescription const& desc);
0146
0147       std::map<BranchID::value_type, BranchID::value_type> const& droppedBranchIDToKeptBranchID() {
0148         return droppedBranchIDToKeptBranchID_;
0149       }
0150
0151       //inheriting classes decrement this in doEvent in a manner that will be thread-safe for that class
0152       std::atomic<int> remainingEvents_;
0153
0154     private:
0155       int maxEvents_;
0156
0157       // TODO: Give OutputModule
0158       // an interface (protected?) that supplies client code with the
0159       // needed functionality *without* giving away implementation
0160       // details ... don't just return a reference to keptProducts_, because
0161       // we are looking to have the flexibility to change the
0162       // implementation of keptProducts_ without modifying clients. When this
0163       // change is made, we'll have a one-time-only task of modifying
0164       // clients (classes derived from OutputModule) to use the
0165       // newly-introduced interface.
0166       // TODO: Consider using shared pointers here?
0167
0168       // keptProducts_ are pointers to the ProductDescription objects describing
0169       // the branches we are to write.
0170       //
0171       // We do not own the ProductDescriptions to which we point.
0172       SelectedProductsForBranchType keptProducts_;
0173       std::array<bool, NumBranchTypes> hasNewlyDroppedBranch_;
0174
0175       std::string process_name_;
0176       ProductSelectorRules productSelectorRules_;
0177       ProductSelector productSelector_;
0178       ModuleDescription moduleDescription_;
0179
0180       bool wantAllEvents_;
0181       std::vector<detail::TriggerResultsBasedEventSelector> selectors_;
0182       ParameterSet selectEvents_;
0183       // ID of the ParameterSet that configured the event selector
0184       // subsystem.
0185       ParameterSetID selector_config_id_;
0186
0187       // needed because of possible EDAliases.
0188       // filled in only if key and value are different.
0189       std::map<BranchID::value_type, BranchID::value_type> droppedBranchIDToKeptBranchID_;
0190       edm::propagate_const<std::unique_ptr<BranchIDLists>> branchIDLists_;
0191       BranchIDLists const* origBranchIDLists_;
0192
0193       edm::propagate_const<std::unique_ptr<ThinnedAssociationsHelper>> thinnedAssociationsHelper_;
0194       std::map<BranchID, bool> keepAssociation_;
0195
0196       OutputProcessBlockHelper outputProcessBlockHelper_;
0197
0198       std::function<void(ProductDescription const&)> callWhenNewProductsRegistered_;
0199
0200       //------------------------------------------------------------------
0201       // private member functions
0202       //------------------------------------------------------------------
0203       void updateBranchIDListsWithKeptAliases();
0204
0205       void doWriteProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*);
0206       void doWriteRun(RunPrincipal const& rp, ModuleCallingContext const*, MergeableRunProductMetadata const*);
0207       void doWriteLuminosityBlock(LuminosityBlockPrincipal const& lbp, ModuleCallingContext const*);
0208       void doOpenFile(FileBlock const& fb);
0209       void doRespondToOpenInputFile(FileBlock const& fb);
0210       void doRespondToCloseInputFile(FileBlock const& fb);
0211       void doRespondToCloseOutputFile() {}
0212       void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {}
0213
0214       /// Tell the OutputModule that is must end the current file.
0215       void doCloseFile();
0216
0217       void registerProductsAndCallbacks(OutputModuleCore const*, SignallingProductRegistryFiller*);
0218
0219       bool needToRunSelection() const noexcept;
0220       std::vector<ProductResolverIndexAndSkipBit> productsUsedBySelection() const noexcept;
0221       bool prePrefetchSelection(StreamID id, EventPrincipal const&, ModuleCallingContext const*);
0222
0223       // Do the end-of-file tasks; this is only called internally, after
0224       // the appropriate tests have been done.
0225       virtual void reallyCloseFile();
0226
0227       /// Ask the OutputModule if we should end the current file.
0228       virtual bool shouldWeCloseFile() const { return false; }
0229
0230       virtual void write(EventForOutput const&) = 0;
0231
0232       /// @brief  called after data product selection has finished
0233       /// @param  iReg ProductRegistry at the start of the job
0234       virtual void initialRegistry(edm::ProductRegistry const& iReg) {}
0235       virtual void beginJob() {}
0236       virtual void endJob() {}
0237       virtual void writeLuminosityBlock(LuminosityBlockForOutput const&) = 0;
0238       virtual void writeRun(RunForOutput const&) = 0;
0239       virtual void writeProcessBlock(ProcessBlockForOutput const&) {}
0240       virtual void openFile(FileBlock const&) {}
0241       virtual bool isFileOpen() const { return true; }
0242
0243       virtual void doBeginRun_(RunForOutput const&) {}
0244       virtual void doEndRun_(RunForOutput const&) {}
0245       virtual void doBeginLuminosityBlock_(LuminosityBlockForOutput const&) {}
0246       virtual void doEndLuminosityBlock_(LuminosityBlockForOutput const&) {}
0247       virtual void doRespondToOpenInputFile_(FileBlock const&) {}
0248       virtual void doRespondToCloseInputFile_(FileBlock const&) {}
0249
0250       virtual void setProcessesWithSelectedMergeableRunProducts(std::set<std::string> const&) {}
0251
0252       bool hasAccumulator() const noexcept { return false; }
0253
0254       void keepThisBranch(ProductDescription const& desc,
0255                           std::map<BranchID, ProductDescription const*>& trueBranchIDToKeptBranchDesc,
0256                           std::set<BranchID>& keptProductsInEvent);
0257
0258       void setModuleDescription(ModuleDescription const& md) { moduleDescription_ = md; }
0259
0260       bool limitReached() const { return remainingEvents_ == 0; }
0261     };
0262   }  // namespace core
0263 }  // namespace edm
0264 #endif