TestProcessor/interface/TestProcessor.h

0001 #ifndef FWCore_TestProcessor_TestProcessor_h
0002 #define FWCore_TestProcessor_TestProcessor_h
0003 // -*- C++ -*-
0004 //
0005 // Package:     FWCore/TestProcessor
0006 // Class  :     TestProcessor
0007 //
0008 /**\class TestProcessor TestProcessor.h "TestProcessor.h"
0009
0010  Description: [one line class summary]
0011
0012  Usage:
0013     <usage>
0014
0015 */
0016 //
0017 // Original Author:  Chris Jones
0018 //         Created:  Mon, 30 Apr 2018 18:51:00 GMT
0019 //
0020
0021 // system include files
0022 #include <string>
0023 #include <utility>
0024 #include <memory>
0025 #include "oneapi/tbb/global_control.h"
0026 #include "oneapi/tbb/task_arena.h"
0027 #include "oneapi/tbb/task_group.h"
0028
0029 // user include files
0030 #include "FWCore/Common/interface/FWCoreCommonFwd.h"
0031 #include "FWCore/Framework/interface/SharedResourcesAcquirer.h"
0032 #include "FWCore/Framework/interface/PrincipalCache.h"
0033 #include "FWCore/Framework/interface/SignallingProductRegistryFiller.h"
0034 #include "FWCore/Framework/interface/PreallocationConfiguration.h"
0035 #include "FWCore/Framework/interface/ModuleRegistry.h"
0036 #include "FWCore/Framework/interface/Schedule.h"
0037 #include "FWCore/Framework/interface/EventSetupRecordKey.h"
0038 #include "FWCore/Framework/interface/DataKey.h"
0039 #include "FWCore/Framework/interface/MergeableRunProductProcesses.h"
0040
0041 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0042 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
0043 #include "FWCore/ServiceRegistry/interface/ServiceLegacy.h"
0044 #include "FWCore/ServiceRegistry/interface/ServiceToken.h"
0045
0046 #include "DataFormats/Provenance/interface/RunLumiEventNumber.h"
0047 #include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h"
0048 #include "DataFormats/Common/interface/Wrapper.h"
0049
0050 #include "FWCore/TestProcessor/interface/Event.h"
0051 #include "FWCore/TestProcessor/interface/LuminosityBlock.h"
0052 #include "FWCore/TestProcessor/interface/ProcessBlock.h"
0053 #include "FWCore/TestProcessor/interface/Run.h"
0054 #include "FWCore/TestProcessor/interface/TestESProductResolver.h"
0055 #include "FWCore/TestProcessor/interface/ESPutTokenT.h"
0056 #include "FWCore/TestProcessor/interface/ESProduceEntry.h"
0057 #include "FWCore/TestProcessor/interface/EventSetupTestHelper.h"
0058
0059 #include "FWCore/Utilities/interface/EDPutToken.h"
0060
0061 // forward declarations
0062
0063 namespace edm {
0064   class ThinnedAssociationsHelper;
0065   class ExceptionToActionTable;
0066   class HistoryAppender;
0067   class ModuleTypeResolverMaker;
0068
0069   namespace eventsetup {
0070     class EventSetupProvider;
0071     class EventSetupsController;
0072   }  // namespace eventsetup
0073
0074   namespace test {
0075     class TestProcessorConfig;
0076     class EventSetupTestHelper;
0077
0078     class ProcessToken {
0079       friend TestProcessorConfig;
0080
0081     public:
0082       ProcessToken() : index_{undefinedIndex()} {}
0083
0084       int index() const { return index_; }
0085
0086       static int undefinedIndex() { return -1; }
0087
0088     private:
0089       explicit ProcessToken(int index) : index_{index} {}
0090
0091       int index_;
0092     };
0093
0094     class TestProcessorConfig {
0095     public:
0096       TestProcessorConfig(std::string const& iPythonConfiguration) : config_(iPythonConfiguration) {}
0097       std::string const& pythonConfiguration() const { return config_; }
0098       void setPythonConfiguration(std::string iConfig) { config_ = std::move(iConfig); }
0099
0100       /** add a Process name to the Process history. If multiple calls are made to addProcess,
0101      then the call order will be the order of the Processes in the history.*/
0102       ProcessToken addExtraProcess(std::string const& iProcessName) {
0103         extraProcesses_.emplace_back(iProcessName);
0104         return ProcessToken(extraProcesses_.size() - 1);
0105       }
0106
0107       std::vector<std::string> const& extraProcesses() const { return extraProcesses_; }
0108
0109       /** A blank iProcessName means it produces in the primary process.
0110      The use of any other name requires that it is first added via `addExtraProcess`.
0111      */
0112       template <typename T>
0113       edm::EDPutTokenT<T> produces(std::string iModuleLabel,
0114                                    std::string iProductInstanceLabel = std::string(),
0115                                    ProcessToken iToken = ProcessToken()) {
0116         produceEntries_.emplace_back(
0117             edm::TypeID(typeid(T)), std::move(iModuleLabel), std::move(iProductInstanceLabel), processName(iToken));
0118         return edm::EDPutTokenT<T>(produceEntries_.size() - 1);
0119       }
0120
0121       template <typename REC, typename T>
0122       edm::test::ESPutTokenT<T> esProduces(std::string iLabel = std::string()) {
0123         auto rk = eventsetup::EventSetupRecordKey::makeKey<REC>();
0124         eventsetup::DataKey dk(eventsetup::DataKey::makeTypeTag<T>(), iLabel.c_str());
0125         esProduceEntries_.emplace_back(rk, dk, std::make_shared<TestESProductResolver<T>>());
0126         return edm::test::ESPutTokenT<T>(esProduceEntries_.size() - 1);
0127       }
0128
0129       struct ProduceEntry {
0130         ProduceEntry(edm::TypeID const& iType,
0131                      std::string moduleLabel,
0132                      std::string instanceLabel,
0133                      std::string processName)
0134             : type_{iType},
0135               moduleLabel_{std::move(moduleLabel)},
0136               instanceLabel_{std::move(instanceLabel)},
0137               processName_{std::move(processName)} {}
0138         edm::TypeID type_;
0139         std::string moduleLabel_;
0140         std::string instanceLabel_;
0141         std::string processName_;
0142       };
0143
0144       std::vector<ProduceEntry> const& produceEntries() const { return produceEntries_; }
0145
0146       std::vector<ESProduceEntry> const& esProduceEntries() const { return esProduceEntries_; }
0147
0148     private:
0149       std::string config_;
0150       std::vector<std::string> extraProcesses_;
0151       std::vector<ProduceEntry> produceEntries_;
0152       std::vector<ESProduceEntry> esProduceEntries_;
0153
0154       std::string processName(ProcessToken iToken) {
0155         if (iToken.index() == ProcessToken::undefinedIndex()) {
0156           return std::string();
0157         }
0158         return extraProcesses_[iToken.index()];
0159       }
0160     };
0161
0162     class TestProcessor {
0163     public:
0164       using Config = TestProcessorConfig;
0165
0166       TestProcessor(Config const& iConfig, ServiceToken iToken = ServiceToken());
0167       TestProcessor(const TestProcessor&) = delete;
0168       const TestProcessor& operator=(const TestProcessor&) = delete;
0169       ~TestProcessor() noexcept(false);
0170
0171       /** Run the test. The function arguments are the data products to be added to the
0172      Event for this particular test.
0173      */
0174       template <typename... T>
0175       edm::test::Event test(T&&... iArgs) {
0176         return testImpl(std::forward<T>(iArgs)...);
0177       }
0178
0179       template <typename... T>
0180       edm::test::LuminosityBlock testBeginLuminosityBlock(edm::LuminosityBlockNumber_t iNum, T&&... iArgs) {
0181         return testBeginLuminosityBlockImpl(iNum, std::forward<T>(iArgs)...);
0182       }
0183       template <typename... T>
0184       edm::test::LuminosityBlock testEndLuminosityBlock(T&&... iArgs) {
0185         return testEndLuminosityBlockImpl(std::forward<T>(iArgs)...);
0186       }
0187
0188       template <typename... T>
0189       edm::test::Run testBeginRun(edm::RunNumber_t iNum, T&&... iArgs) {
0190         return testBeginRunImpl(iNum, std::forward<T>(iArgs)...);
0191       }
0192       template <typename... T>
0193       edm::test::Run testEndRun(T&&... iArgs) {
0194         return testEndRunImpl(std::forward<T>(iArgs)...);
0195       }
0196
0197       // It makes no sense to pass EventSetup products and at least
0198       // for now Runs, Lumis, and ProcessBlocks don't allow passing
0199       // in other products. So for now these don't need to be templates
0200       // for ProcessBlock.
0201       edm::test::ProcessBlock testBeginProcessBlock() { return testBeginProcessBlockImpl(); }
0202
0203       edm::test::ProcessBlock testEndProcessBlock() { return testEndProcessBlockImpl(); }
0204
0205       /** Run only beginJob and endJob. Once this is used, you should not attempt to run any further tests.
0206 This simulates a problem happening early in the job which causes processing not to proceed.
0207    */
0208       void testBeginAndEndJobOnly() {
0209         arena_.execute([this]() {
0210           beginJob();
0211           endJob();
0212         });
0213       }
0214
0215       void testWithNoRuns() {
0216         arena_.execute([this]() {
0217           beginJob();
0218           beginProcessBlock();
0219           endProcessBlock();
0220           endJob();
0221         });
0222       }
0223
0224       void testRunWithNoLuminosityBlocks() {
0225         arena_.execute([this]() {
0226           beginJob();
0227           beginProcessBlock();
0228           beginRun();
0229           endRun();
0230           endProcessBlock();
0231           endJob();
0232         });
0233       }
0234
0235       void testLuminosityBlockWithNoEvents() {
0236         arena_.execute([this]() {
0237           beginJob();
0238           beginProcessBlock();
0239           beginRun();
0240           beginLuminosityBlock();
0241           endLuminosityBlock();
0242           endRun();
0243           endProcessBlock();
0244           endJob();
0245         });
0246       }
0247       void setRunNumber(edm::RunNumber_t);
0248       void setLuminosityBlockNumber(edm::LuminosityBlockNumber_t);
0249       void setEventNumber(edm::EventNumber_t);
0250
0251       std::string const& labelOfTestModule() const { return labelOfTestModule_; }
0252
0253     private:
0254       template <typename T, typename... U>
0255       edm::test::Event testImpl(std::pair<edm::EDPutTokenT<T>, std::unique_ptr<T>>&& iPut, U&&... iArgs) {
0256         put(std::move(iPut));
0257         return testImpl(std::forward<U>(iArgs)...);
0258       }
0259
0260       template <typename T, typename... U>
0261       edm::test::Event testImpl(std::pair<edm::test::ESPutTokenT<T>, std::unique_ptr<T>>&& iPut, U&&... iArgs) {
0262         put(std::move(iPut));
0263         return testImpl(std::forward<U>(iArgs)...);
0264       }
0265
0266       template <typename T>
0267       void put(std::pair<edm::EDPutTokenT<T>, std::unique_ptr<T>>&& iPut) {
0268         put(iPut.first.index(), std::make_unique<edm::Wrapper<T>>(std::move(iPut.second)));
0269       }
0270
0271       template <typename T>
0272       void put(std::pair<edm::test::ESPutTokenT<T>, std::unique_ptr<T>>&& iPut) {
0273         dynamic_cast<TestESProductResolver<T>*>(esHelper_->getResolver(iPut.first.index()).get())
0274             ->setData(std::move(iPut.second));
0275       }
0276
0277       void put(unsigned int, std::unique_ptr<WrapperBase>);
0278
0279       edm::test::Event testImpl();
0280
0281       template <typename T, typename... U>
0282       edm::test::LuminosityBlock testBeginLuminosityBlockImpl(
0283           edm::LuminosityBlockNumber_t iNum,
0284           std::pair<edm::test::ESPutTokenT<T>, std::unique_ptr<T>>&& iPut,
0285           U&&... iArgs) {
0286         put(std::move(iPut));
0287         return testBeginLuminosityBlockImpl(iNum, std::forward<U>(iArgs)...);
0288       }
0289       edm::test::LuminosityBlock testBeginLuminosityBlockImpl(edm::LuminosityBlockNumber_t);
0290
0291       template <typename T, typename... U>
0292       edm::test::LuminosityBlock testEndLuminosityBlockImpl(
0293           std::pair<edm::test::ESPutTokenT<T>, std::unique_ptr<T>>&& iPut, U&&... iArgs) {
0294         put(std::move(iPut));
0295         return testEndLuminosityBlockImpl(std::forward<U>(iArgs)...);
0296       }
0297       edm::test::LuminosityBlock testEndLuminosityBlockImpl();
0298
0299       template <typename T, typename... U>
0300       edm::test::Run testBeginRunImpl(edm::RunNumber_t iNum,
0301                                       std::pair<edm::test::ESPutTokenT<T>, std::unique_ptr<T>>&& iPut,
0302                                       U&&... iArgs) {
0303         put(std::move(iPut));
0304         return testBeginRunImpl(iNum, std::forward<U>(iArgs)...);
0305       }
0306       edm::test::Run testBeginRunImpl(edm::RunNumber_t);
0307       template <typename T, typename... U>
0308       edm::test::LuminosityBlock testEndRunImpl(std::pair<edm::test::ESPutTokenT<T>, std::unique_ptr<T>>&& iPut,
0309                                                 U&&... iArgs) {
0310         put(std::move(iPut));
0311         return testEndRunImpl(std::forward<U>(iArgs)...);
0312       }
0313       edm::test::Run testEndRunImpl();
0314
0315       edm::test::ProcessBlock testBeginProcessBlockImpl();
0316       edm::test::ProcessBlock testEndProcessBlockImpl();
0317
0318       void setupProcessing();
0319       void teardownProcessing();
0320
0321       void beginJob();
0322       void respondToOpenInputFile();
0323       void openOutputFiles();
0324       void beginProcessBlock();
0325       void beginRun();
0326       void beginLuminosityBlock();
0327       void event();
0328       std::shared_ptr<LuminosityBlockPrincipal> endLuminosityBlock();
0329       std::shared_ptr<RunPrincipal> endRun();
0330       void respondToCloseInputFile();
0331       ProcessBlockPrincipal const* endProcessBlock();
0332       void closeOutputFiles();
0333       void endJob();
0334
0335       template <typename Traits>
0336       void processTransitionForAllStreams(typename Traits::TransitionInfoType& transitionInfo);
0337
0338       template <typename Traits>
0339       void processGlobalTransition(typename Traits::TransitionInfoType& transitionInfo);
0340
0341       // ---------- member data --------------------------------
0342       oneapi::tbb::global_control globalControl_;
0343       oneapi::tbb::task_group taskGroup_;
0344       oneapi::tbb::task_arena arena_;
0345       std::string labelOfTestModule_;
0346       std::shared_ptr<ActivityRegistry> actReg_;  // We do not use propagate_const because the registry itself is mutable.
0347       std::shared_ptr<ProductRegistry> preg_;
0348       std::shared_ptr<BranchIDListHelper> branchIDListHelper_;
0349       std::shared_ptr<ProcessBlockHelper> processBlockHelper_;
0350       std::shared_ptr<ThinnedAssociationsHelper> thinnedAssociationsHelper_;
0351       ServiceToken serviceToken_;
0352       std::unique_ptr<ModuleTypeResolverMaker const> moduleTypeResolverMaker_;
0353       std::unique_ptr<eventsetup::EventSetupsController> espController_;
0354       std::shared_ptr<eventsetup::EventSetupProvider> esp_;
0355       std::shared_ptr<EventSetupTestHelper> esHelper_;
0356
0357       std::unique_ptr<ExceptionToActionTable const> act_table_;
0358       std::shared_ptr<ProcessConfiguration const> processConfiguration_;
0359       ProcessContext processContext_;
0360
0361       MergeableRunProductProcesses mergeableRunProductProcesses_;
0362
0363       ProcessHistoryRegistry processHistoryRegistry_;
0364       ProcessHistory processHistory_;
0365       std::unique_ptr<HistoryAppender> historyAppender_;
0366
0367       PrincipalCache principalCache_;
0368       PreallocationConfiguration preallocations_;
0369
0370       std::shared_ptr<ModuleRegistry> moduleRegistry_;
0371       std::unique_ptr<Schedule> schedule_;
0372       std::shared_ptr<RunPrincipal> runPrincipal_;
0373       std::shared_ptr<LuminosityBlockPrincipal> lumiPrincipal_;
0374
0375       std::vector<std::pair<edm::ProductDescription, std::unique_ptr<WrapperBase>>> dataProducts_;
0376
0377       RunNumber_t runNumber_ = 1;
0378       LuminosityBlockNumber_t lumiNumber_ = 1;
0379       EventNumber_t eventNumber_ = 1;
0380       bool beginJobCalled_ = false;
0381       bool respondToOpenInputFileCalled_ = false;
0382       bool beginProcessBlockCalled_ = false;
0383       bool beginRunCalled_ = false;
0384       bool beginLumiCalled_ = false;
0385       bool openOutputFilesCalled_ = false;
0386     };
0387   }  // namespace test
0388 }  // namespace edm
0389
0390 #endif