Framework/interface/OutputModuleCore.h

0001 #ifndef FWCore_Framework_one_OutputModuleCore_h
0002 #define FWCore_Framework_one_OutputModuleCore_h
0003 // -*- C++ -*-
0004 //
0005 // Package:     FWCore/Framework
0006 // Class  :     OutputModuleCore
0007 //
0008 /**\class OutputModuleCore OutputModuleCore.h "FWCore/Framework/interface/one/OutputModuleCore.h"
0009
0010  Description: Base class for all 'one' OutputModules
0011
0012  Usage:
0013     <usage>
0014
0015 */
0016 //
0017 // Original Author:  Chris Jones
0018 //         Created:  Wed, 31 Jul 2013 15:37:16 GMT
0019 //
0020
0021 // system include files
0022 #include <array>
0023 #include <memory>
0024 #include <string>
0025 #include <vector>
0026 #include <map>
0027 #include <atomic>
0028 #include <set>
0029
0030 // user include files
0031 #include "DataFormats/Provenance/interface/BranchID.h"
0032 #include "DataFormats/Provenance/interface/BranchIDList.h"
0033 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0034 #include "DataFormats/Provenance/interface/SelectedProducts.h"
0035
0036 #include "FWCore/Common/interface/FWCoreCommonFwd.h"
0037 #include "FWCore/Common/interface/OutputProcessBlockHelper.h"
0038 #include "FWCore/Framework/interface/TriggerResultsBasedEventSelector.h"
0039 #include "FWCore/Framework/interface/Frameworkfwd.h"
0040 #include "FWCore/Framework/interface/ProductSelectorRules.h"
0041 #include "FWCore/Framework/interface/ProductSelector.h"
0042 #include "FWCore/Framework/interface/EDConsumerBase.h"
0043 #include "FWCore/Framework/interface/getAllTriggerNames.h"
0044 #include "FWCore/ParameterSet/interface/ParameterSetfwd.h"
0045 #include "FWCore/Concurrency/interface/WaitingTaskHolder.h"
0046 #include "FWCore/Utilities/interface/propagate_const.h"
0047
0048 // forward declarations
0049 namespace edm {
0050
0051   class MergeableRunProductMetadata;
0052   class ModuleCallingContext;
0053   class PreallocationConfiguration;
0054   class ActivityRegistry;
0055   class ThinnedAssociationsHelper;
0056
0057   template <typename T>
0058   class OutputModuleCommunicatorT;
0059
0060   namespace maker {
0061     template <typename T>
0062     class ModuleHolderT;
0063   }
0064
0065   namespace core {
0066
0067     class OutputModuleCore : public EDConsumerBase {
0068     public:
0069       template <typename U>
0070       friend class edm::maker::ModuleHolderT;
0071       template <typename T>
0072       friend class ::edm::WorkerT;
0073       template <typename T>
0074       friend class ::edm::OutputModuleCommunicatorT;
0075       typedef OutputModuleCore ModuleType;
0076
0077       explicit OutputModuleCore(ParameterSet const& pset);
0078       ~OutputModuleCore() override;
0079
0080       OutputModuleCore(OutputModuleCore const&) = delete;             // Disallow copying and moving
0081       OutputModuleCore& operator=(OutputModuleCore const&) = delete;  // Disallow copying and moving
0082
0083       /// Accessor for maximum number of events to be written.
0084       /// -1 is used for unlimited.
0085       int maxEvents() const { return maxEvents_; }
0086
0087       /// Accessor for remaining number of events to be written.
0088       /// -1 is used for unlimited.
0089       int remainingEvents() const { return remainingEvents_; }
0090
0091       bool selected(BranchDescription const& desc) const;
0092
0093       void selectProducts(ProductRegistry const& preg, ThinnedAssociationsHelper const&, ProcessBlockHelperBase const&);
0094       std::string const& processName() const { return process_name_; }
0095       SelectedProductsForBranchType const& keptProducts() const { return keptProducts_; }
0096       std::array<bool, NumBranchTypes> const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; }
0097
0098       static void fillDescription(
0099           ParameterSetDescription& desc,
0100           std::vector<std::string> const& iDefaultOutputCommands = ProductSelectorRules::defaultSelectionStrings());
0101       static void fillDescriptions(ConfigurationDescriptions& descriptions);
0102       static const std::string& baseType();
0103       static void prevalidate(ConfigurationDescriptions&);
0104
0105       bool wantAllEvents() const { return wantAllEvents_; }
0106
0107       BranchIDLists const* branchIDLists() const;
0108
0109       OutputProcessBlockHelper const& outputProcessBlockHelper() const { return outputProcessBlockHelper_; }
0110
0111       ThinnedAssociationsHelper const* thinnedAssociationsHelper() const;
0112
0113       const ModuleDescription& moduleDescription() const { return moduleDescription_; }
0114
0115     protected:
0116       ModuleDescription const& description() const;
0117
0118       ParameterSetID selectorConfig() const { return selector_config_id_; }
0119
0120       void doPreallocate_(PreallocationConfiguration const&);
0121       virtual void preallocLumis(unsigned int);
0122
0123       void doBeginJob_();
0124       void doEndJob();
0125       bool doEvent_(EventTransitionInfo const&, ActivityRegistry*, ModuleCallingContext const*);
0126       void doBeginProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {}
0127       void doAccessInputProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {}
0128       void doEndProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {}
0129       bool doBeginRun(RunTransitionInfo const&, ModuleCallingContext const*);
0130       bool doEndRun(RunTransitionInfo const&, ModuleCallingContext const*);
0131       bool doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*);
0132       bool doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*);
0133
0134       void setEventSelectionInfo(
0135           std::map<std::string, std::vector<std::pair<std::string, int>>> const& outputModulePathPositions,
0136           bool anyProductProduced);
0137
0138       void configure(OutputModuleDescription const& desc);
0139
0140       std::map<BranchID::value_type, BranchID::value_type> const& droppedBranchIDToKeptBranchID() {
0141         return droppedBranchIDToKeptBranchID_;
0142       }
0143
0144       //inheriting classes decrement this in doEvent in a manner that will be thread-safe for that class
0145       std::atomic<int> remainingEvents_;
0146
0147     private:
0148       int maxEvents_;
0149
0150       // TODO: Give OutputModule
0151       // an interface (protected?) that supplies client code with the
0152       // needed functionality *without* giving away implementation
0153       // details ... don't just return a reference to keptProducts_, because
0154       // we are looking to have the flexibility to change the
0155       // implementation of keptProducts_ without modifying clients. When this
0156       // change is made, we'll have a one-time-only task of modifying
0157       // clients (classes derived from OutputModule) to use the
0158       // newly-introduced interface.
0159       // TODO: Consider using shared pointers here?
0160
0161       // keptProducts_ are pointers to the BranchDescription objects describing
0162       // the branches we are to write.
0163       //
0164       // We do not own the BranchDescriptions to which we point.
0165       SelectedProductsForBranchType keptProducts_;
0166       std::array<bool, NumBranchTypes> hasNewlyDroppedBranch_;
0167
0168       std::string process_name_;
0169       ProductSelectorRules productSelectorRules_;
0170       ProductSelector productSelector_;
0171       ModuleDescription moduleDescription_;
0172
0173       bool wantAllEvents_;
0174       std::vector<detail::TriggerResultsBasedEventSelector> selectors_;
0175       ParameterSet selectEvents_;
0176       std::vector<EDGetToken> tokensForEndPaths_;  //needed for FinalPath
0177       bool onFinalPath_ = false;
0178       // ID of the ParameterSet that configured the event selector
0179       // subsystem.
0180       ParameterSetID selector_config_id_;
0181
0182       // needed because of possible EDAliases.
0183       // filled in only if key and value are different.
0184       std::map<BranchID::value_type, BranchID::value_type> droppedBranchIDToKeptBranchID_;
0185       edm::propagate_const<std::unique_ptr<BranchIDLists>> branchIDLists_;
0186       BranchIDLists const* origBranchIDLists_;
0187
0188       edm::propagate_const<std::unique_ptr<ThinnedAssociationsHelper>> thinnedAssociationsHelper_;
0189       std::map<BranchID, bool> keepAssociation_;
0190
0191       OutputProcessBlockHelper outputProcessBlockHelper_;
0192
0193       //------------------------------------------------------------------
0194       // private member functions
0195       //------------------------------------------------------------------
0196       void updateBranchIDListsWithKeptAliases();
0197
0198       void doWriteProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*);
0199       void doWriteRun(RunPrincipal const& rp, ModuleCallingContext const*, MergeableRunProductMetadata const*);
0200       void doWriteLuminosityBlock(LuminosityBlockPrincipal const& lbp, ModuleCallingContext const*);
0201       void doOpenFile(FileBlock const& fb);
0202       void doRespondToOpenInputFile(FileBlock const& fb);
0203       void doRespondToCloseInputFile(FileBlock const& fb);
0204       void doRespondToCloseOutputFile() {}
0205       void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {}
0206
0207       /// Tell the OutputModule that is must end the current file.
0208       void doCloseFile();
0209
0210       void registerProductsAndCallbacks(OutputModuleCore const*, ProductRegistry const*) {}
0211
0212       bool needToRunSelection() const;
0213       std::vector<ProductResolverIndexAndSkipBit> productsUsedBySelection() const;
0214       bool prePrefetchSelection(StreamID id, EventPrincipal const&, ModuleCallingContext const*);
0215
0216       // Do the end-of-file tasks; this is only called internally, after
0217       // the appropriate tests have been done.
0218       virtual void reallyCloseFile();
0219
0220       /// Ask the OutputModule if we should end the current file.
0221       virtual bool shouldWeCloseFile() const { return false; }
0222
0223       virtual void write(EventForOutput const&) = 0;
0224
0225       virtual void beginJob() {}
0226       virtual void endJob() {}
0227       virtual void writeLuminosityBlock(LuminosityBlockForOutput const&) = 0;
0228       virtual void writeRun(RunForOutput const&) = 0;
0229       virtual void writeProcessBlock(ProcessBlockForOutput const&) {}
0230       virtual void openFile(FileBlock const&) {}
0231       virtual bool isFileOpen() const { return true; }
0232
0233       virtual void doBeginRun_(RunForOutput const&) {}
0234       virtual void doEndRun_(RunForOutput const&) {}
0235       virtual void doBeginLuminosityBlock_(LuminosityBlockForOutput const&) {}
0236       virtual void doEndLuminosityBlock_(LuminosityBlockForOutput const&) {}
0237       virtual void doRespondToOpenInputFile_(FileBlock const&) {}
0238       virtual void doRespondToCloseInputFile_(FileBlock const&) {}
0239
0240       virtual void setProcessesWithSelectedMergeableRunProducts(std::set<std::string> const&) {}
0241
0242       bool hasAccumulator() const { return false; }
0243
0244       void keepThisBranch(BranchDescription const& desc,
0245                           std::map<BranchID, BranchDescription const*>& trueBranchIDToKeptBranchDesc,
0246                           std::set<BranchID>& keptProductsInEvent);
0247
0248       void setModuleDescription(ModuleDescription const& md) { moduleDescription_ = md; }
0249
0250       bool limitReached() const { return remainingEvents_ == 0; }
0251     };
0252   }  // namespace core
0253 }  // namespace edm
0254 #endif