File indexing completed on 2023-03-17 11:02:44
0001
0002
0003
0004
0005
0006
0007
0008 #include <iostream>
0009 #include <atomic>
0010 #include <vector>
0011 #include <map>
0012 #include <functional>
0013 #include "oneapi/tbb/global_control.h"
0014 #include "FWCore/Framework/interface/one/OutputModule.h"
0015 #include "FWCore/Framework/interface/OutputModuleCommunicatorT.h"
0016 #include "FWCore/Framework/interface/TransitionInfoTypes.h"
0017 #include "FWCore/Framework/interface/maker/WorkerT.h"
0018 #include "FWCore/Framework/interface/OccurrenceTraits.h"
0019 #include "DataFormats/Provenance/interface/ProductRegistry.h"
0020 #include "DataFormats/Provenance/interface/BranchIDListHelper.h"
0021 #include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h"
0022 #include "FWCore/Framework/interface/HistoryAppender.h"
0023 #include "FWCore/Utilities/interface/GlobalIdentifier.h"
0024 #include "FWCore/Framework/interface/TriggerNamesService.h"
0025 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0026 #include "FWCore/ServiceRegistry/interface/ParentContext.h"
0027 #include "FWCore/ServiceRegistry/interface/Service.h"
0028 #include "FWCore/ServiceRegistry/interface/ServiceRegistry.h"
0029 #include "FWCore/Framework/interface/FileBlock.h"
0030 #include "FWCore/Framework/interface/PreallocationConfiguration.h"
0031 #include "FWCore/Concurrency/interface/WaitingTaskHolder.h"
0032 #include "FWCore/Concurrency/interface/FinalWaitingTask.h"
0033
0034 #include "FWCore/Utilities/interface/Exception.h"
0035
0036 #include "cppunit/extensions/HelperMacros.h"
0037
0038 namespace edm {
0039 class ModuleCallingContext;
0040 }
0041
0042 class testOneOutputModule : public CppUnit::TestFixture {
0043 CPPUNIT_TEST_SUITE(testOneOutputModule);
0044
0045 CPPUNIT_TEST(basicTest);
0046 CPPUNIT_TEST(runTest);
0047 CPPUNIT_TEST(runCacheTest);
0048 CPPUNIT_TEST(lumiTest);
0049 CPPUNIT_TEST(lumiCacheTest);
0050 CPPUNIT_TEST(fileTest);
0051 CPPUNIT_TEST(resourceTest);
0052
0053 CPPUNIT_TEST_SUITE_END();
0054
0055 public:
0056 testOneOutputModule();
0057
0058 void setUp() {}
0059 void tearDown() {}
0060
0061 void basicTest();
0062 void runTest();
0063 void runCacheTest();
0064 void lumiTest();
0065 void lumiCacheTest();
0066 void fileTest();
0067 void resourceTest();
0068
0069 enum class Trans {
0070 kBeginJob,
0071 kGlobalOpenInputFile,
0072 kGlobalBeginRun,
0073 kGlobalBeginRunProduce,
0074 kGlobalBeginLuminosityBlock,
0075 kEvent,
0076 kGlobalEndLuminosityBlock,
0077 kGlobalEndRun,
0078 kGlobalCloseInputFile,
0079 kEndJob
0080 };
0081
0082 typedef std::vector<Trans> Expectations;
0083
0084 private:
0085 std::map<Trans, std::function<void(edm::Worker*, edm::OutputModuleCommunicator*)>> m_transToFunc;
0086
0087 edm::ProcessConfiguration m_procConfig;
0088 edm::PreallocationConfiguration m_preallocConfig;
0089 std::shared_ptr<edm::ProductRegistry> m_prodReg;
0090 std::shared_ptr<edm::BranchIDListHelper> m_idHelper;
0091 std::shared_ptr<edm::ThinnedAssociationsHelper> m_associationsHelper;
0092 std::unique_ptr<edm::EventPrincipal> m_ep;
0093 edm::HistoryAppender historyAppender_;
0094 std::shared_ptr<edm::LuminosityBlockPrincipal> m_lbp;
0095 std::shared_ptr<edm::RunPrincipal> m_rp;
0096 std::shared_ptr<edm::ActivityRegistry>
0097 m_actReg;
0098 edm::EventSetupImpl* m_es = nullptr;
0099 edm::ModuleDescription m_desc = {"Dummy", "dummy"};
0100 edm::WorkerParams m_params;
0101
0102 typedef edm::service::TriggerNamesService TNS;
0103 typedef edm::serviceregistry::ServiceWrapper<TNS> w_TNS;
0104 std::shared_ptr<w_TNS> tnsptr_;
0105 edm::ServiceToken serviceToken_;
0106
0107 template <typename T>
0108 void testTransitions(std::shared_ptr<T> iMod, Expectations const& iExpect);
0109
0110 template <typename Traits, typename Info>
0111 void doWork(edm::Worker* iBase, Info const& info, edm::StreamID id, edm::ParentContext const& iContext) {
0112 oneapi::tbb::task_group group;
0113 edm::FinalWaitingTask task{group};
0114 edm::ServiceToken token;
0115 iBase->doWorkAsync<Traits>(edm::WaitingTaskHolder(group, &task), info, token, id, iContext, nullptr);
0116 task.wait();
0117 }
0118
0119 class BasicOutputModule : public edm::one::OutputModule<> {
0120 public:
0121 using edm::one::OutputModuleBase::doPreallocate;
0122 BasicOutputModule(edm::ParameterSet const& iPSet)
0123 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<>(iPSet) {}
0124 unsigned int m_count = 0;
0125
0126 void write(edm::EventForOutput const&) override { ++m_count; }
0127 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0128 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0129 };
0130
0131 class RunOutputModule : public edm::one::OutputModule<edm::one::WatchRuns> {
0132 public:
0133 using edm::one::OutputModuleBase::doPreallocate;
0134 RunOutputModule(edm::ParameterSet const& iPSet)
0135 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::one::WatchRuns>(iPSet) {}
0136 unsigned int m_count = 0;
0137 void write(edm::EventForOutput const&) override { ++m_count; }
0138 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0139 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0140
0141 void beginRun(edm::RunForOutput const&) override { ++m_count; }
0142
0143 void endRun(edm::RunForOutput const&) override { ++m_count; }
0144 };
0145
0146 class LumiOutputModule : public edm::one::OutputModule<edm::one::WatchLuminosityBlocks> {
0147 public:
0148 using edm::one::OutputModuleBase::doPreallocate;
0149 LumiOutputModule(edm::ParameterSet const& iPSet)
0150 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::one::WatchLuminosityBlocks>(iPSet) {}
0151 unsigned int m_count = 0;
0152 void write(edm::EventForOutput const&) override { ++m_count; }
0153 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0154 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0155
0156 void beginLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0157
0158 void endLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0159 };
0160
0161 struct DummyCache {};
0162
0163 class RunCacheOutputModule : public edm::one::OutputModule<edm::RunCache<DummyCache>> {
0164 public:
0165 using edm::one::OutputModuleBase::doPreallocate;
0166 RunCacheOutputModule(edm::ParameterSet const& iPSet)
0167 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::RunCache<DummyCache>>(iPSet) {}
0168 mutable std::atomic<unsigned int> m_count = 0;
0169 void write(edm::EventForOutput const&) override { ++m_count; }
0170 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0171 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0172
0173 std::shared_ptr<DummyCache> globalBeginRun(edm::RunForOutput const&) const override {
0174 ++m_count;
0175 return std::shared_ptr<DummyCache>{};
0176 }
0177
0178 void globalEndRun(edm::RunForOutput const&) override { ++m_count; }
0179 };
0180
0181 class LumiCacheOutputModule : public edm::one::OutputModule<edm::LuminosityBlockCache<DummyCache>> {
0182 public:
0183 using edm::one::OutputModuleBase::doPreallocate;
0184 LumiCacheOutputModule(edm::ParameterSet const& iPSet)
0185 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::LuminosityBlockCache<DummyCache>>(iPSet) {}
0186 mutable std::atomic<unsigned int> m_count = 0;
0187 void write(edm::EventForOutput const&) override { ++m_count; }
0188 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0189 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0190
0191 std::shared_ptr<DummyCache> globalBeginLuminosityBlock(edm::LuminosityBlockForOutput const&) const override {
0192 ++m_count;
0193 return std::shared_ptr<DummyCache>{};
0194 }
0195
0196 void globalEndLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0197 };
0198
0199 class FileOutputModule : public edm::one::OutputModule<edm::WatchInputFiles> {
0200 public:
0201 using edm::one::OutputModuleBase::doPreallocate;
0202 FileOutputModule(edm::ParameterSet const& iPSet)
0203 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::WatchInputFiles>(iPSet) {}
0204 unsigned int m_count = 0;
0205 void write(edm::EventForOutput const&) override { ++m_count; }
0206 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0207 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0208
0209 void respondToOpenInputFile(edm::FileBlock const&) override { ++m_count; }
0210
0211 void respondToCloseInputFile(edm::FileBlock const&) override { ++m_count; }
0212 };
0213
0214 class ResourceOutputModule : public edm::one::OutputModule<edm::one::SharedResources> {
0215 public:
0216 using edm::one::OutputModuleBase::doPreallocate;
0217 ResourceOutputModule(edm::ParameterSet const& iPSet)
0218 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::one::SharedResources>(iPSet) {
0219 usesResource();
0220 }
0221 unsigned int m_count = 0;
0222
0223 void write(edm::EventForOutput const&) override { ++m_count; }
0224 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0225 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0226 };
0227 };
0228
0229 namespace {
0230
0231 edm::ActivityRegistry activityRegistry;
0232
0233 struct ShadowStreamID {
0234 constexpr ShadowStreamID() : value(0) {}
0235 unsigned int value;
0236 };
0237
0238 union IDUnion {
0239 IDUnion() : m_shadow() {}
0240 ShadowStreamID m_shadow;
0241 edm::StreamID m_id;
0242 };
0243 }
0244 static edm::StreamID makeID() {
0245 IDUnion u;
0246 assert(u.m_id.value() == 0);
0247 return u.m_id;
0248 }
0249 static const edm::StreamID s_streamID0 = makeID();
0250
0251
0252 CPPUNIT_TEST_SUITE_REGISTRATION(testOneOutputModule);
0253
0254 testOneOutputModule::testOneOutputModule()
0255 : m_prodReg(new edm::ProductRegistry{}),
0256 m_idHelper(new edm::BranchIDListHelper{}),
0257 m_associationsHelper(new edm::ThinnedAssociationsHelper{}),
0258 m_ep() {
0259
0260 m_prodReg->setFrozen();
0261 m_idHelper->updateFromRegistry(*m_prodReg);
0262 edm::EventID eventID = edm::EventID::firstValidEvent();
0263
0264 std::string uuid = edm::createGlobalIdentifier();
0265 edm::Timestamp now(1234567UL);
0266 m_rp.reset(new edm::RunPrincipal(m_prodReg, m_procConfig, &historyAppender_, 0));
0267 m_rp->setAux(edm::RunAuxiliary(eventID.run(), now, now));
0268 edm::LuminosityBlockAuxiliary lumiAux(m_rp->run(), 1, now, now);
0269 m_lbp.reset(new edm::LuminosityBlockPrincipal(m_prodReg, m_procConfig, &historyAppender_, 0));
0270 m_lbp->setAux(lumiAux);
0271 m_lbp->setRunPrincipal(m_rp);
0272 edm::EventAuxiliary eventAux(eventID, uuid, now, true);
0273
0274 m_ep.reset(new edm::EventPrincipal(m_prodReg, m_idHelper, m_associationsHelper, m_procConfig, nullptr));
0275 m_ep->fillEventPrincipal(eventAux, nullptr);
0276 m_ep->setLuminosityBlockPrincipal(m_lbp.get());
0277 m_actReg.reset(new edm::ActivityRegistry);
0278
0279
0280 m_transToFunc[Trans::kGlobalOpenInputFile] = [](edm::Worker* iBase, edm::OutputModuleCommunicator*) {
0281 edm::FileBlock fb;
0282 iBase->respondToOpenInputFile(fb);
0283 };
0284
0285 m_transToFunc[Trans::kGlobalBeginRun] = [this](edm::Worker* iBase, edm::OutputModuleCommunicator*) {
0286 typedef edm::OccurrenceTraits<edm::RunPrincipal, edm::BranchActionGlobalBegin> Traits;
0287 edm::GlobalContext gc(edm::GlobalContext::Transition::kBeginRun, nullptr);
0288 edm::ParentContext parentContext(&gc);
0289 iBase->setActivityRegistry(m_actReg);
0290 edm::RunTransitionInfo info(*m_rp, *m_es);
0291 doWork<Traits>(iBase, info, edm::StreamID::invalidStreamID(), parentContext);
0292 };
0293
0294 m_transToFunc[Trans::kGlobalBeginLuminosityBlock] = [this](edm::Worker* iBase, edm::OutputModuleCommunicator*) {
0295 typedef edm::OccurrenceTraits<edm::LuminosityBlockPrincipal, edm::BranchActionGlobalBegin> Traits;
0296 edm::GlobalContext gc(edm::GlobalContext::Transition::kBeginLuminosityBlock, nullptr);
0297 edm::ParentContext parentContext(&gc);
0298 iBase->setActivityRegistry(m_actReg);
0299 edm::LumiTransitionInfo info(*m_lbp, *m_es);
0300 doWork<Traits>(iBase, info, edm::StreamID::invalidStreamID(), parentContext);
0301 };
0302
0303 m_transToFunc[Trans::kEvent] = [this](edm::Worker* iBase, edm::OutputModuleCommunicator*) {
0304 typedef edm::OccurrenceTraits<edm::EventPrincipal, edm::BranchActionStreamBegin> Traits;
0305 edm::StreamContext streamContext(s_streamID0, nullptr);
0306 edm::ParentContext parentContext(&streamContext);
0307 iBase->setActivityRegistry(m_actReg);
0308 edm::EventTransitionInfo info(*m_ep, *m_es);
0309 doWork<Traits>(iBase, info, s_streamID0, parentContext);
0310 };
0311
0312 m_transToFunc[Trans::kGlobalEndLuminosityBlock] = [this](edm::Worker* iBase, edm::OutputModuleCommunicator* iComm) {
0313 typedef edm::OccurrenceTraits<edm::LuminosityBlockPrincipal, edm::BranchActionGlobalEnd> Traits;
0314 edm::GlobalContext gc(edm::GlobalContext::Transition::kEndLuminosityBlock, nullptr);
0315 edm::ParentContext parentContext(&gc);
0316 iBase->setActivityRegistry(m_actReg);
0317 edm::LumiTransitionInfo info(*m_lbp, *m_es);
0318 doWork<Traits>(iBase, info, edm::StreamID::invalidStreamID(), parentContext);
0319 oneapi::tbb::task_group group;
0320 edm::FinalWaitingTask task{group};
0321 iComm->writeLumiAsync(edm::WaitingTaskHolder(group, &task), *m_lbp, nullptr, &activityRegistry);
0322 task.wait();
0323 };
0324
0325 m_transToFunc[Trans::kGlobalEndRun] = [this](edm::Worker* iBase, edm::OutputModuleCommunicator* iComm) {
0326 typedef edm::OccurrenceTraits<edm::RunPrincipal, edm::BranchActionGlobalEnd> Traits;
0327 edm::GlobalContext gc(edm::GlobalContext::Transition::kEndRun, nullptr);
0328 edm::ParentContext parentContext(&gc);
0329 iBase->setActivityRegistry(m_actReg);
0330 edm::RunTransitionInfo info(*m_rp, *m_es);
0331 doWork<Traits>(iBase, info, edm::StreamID::invalidStreamID(), parentContext);
0332 oneapi::tbb::task_group group;
0333 edm::FinalWaitingTask task{group};
0334 iComm->writeRunAsync(edm::WaitingTaskHolder(group, &task), *m_rp, nullptr, &activityRegistry, nullptr);
0335 task.wait();
0336 };
0337
0338 m_transToFunc[Trans::kGlobalCloseInputFile] = [](edm::Worker* iBase, edm::OutputModuleCommunicator*) {
0339 edm::FileBlock fb;
0340 iBase->respondToCloseInputFile(fb);
0341 };
0342
0343
0344
0345
0346
0347 edm::ParameterSet proc_pset;
0348
0349 std::string processName("HLT");
0350 proc_pset.addParameter<std::string>("@process_name", processName);
0351
0352 std::vector<std::string> paths;
0353 edm::ParameterSet trigPaths;
0354 trigPaths.addParameter<std::vector<std::string>>("@trigger_paths", paths);
0355 proc_pset.addParameter<edm::ParameterSet>("@trigger_paths", trigPaths);
0356
0357 std::vector<std::string> endPaths;
0358 proc_pset.addParameter<std::vector<std::string>>("@end_paths", endPaths);
0359
0360
0361 tnsptr_.reset(new w_TNS(std::make_unique<TNS>(proc_pset)));
0362
0363 serviceToken_ = edm::ServiceRegistry::createContaining(tnsptr_);
0364 }
0365
0366 namespace {
0367 template <typename T>
0368 void testTransition(std::shared_ptr<T> iMod,
0369 edm::Worker* iWorker,
0370 edm::OutputModuleCommunicator* iComm,
0371 testOneOutputModule::Trans iTrans,
0372 testOneOutputModule::Expectations const& iExpect,
0373 std::function<void(edm::Worker*, edm::OutputModuleCommunicator*)> iFunc) {
0374 assert(0 == iMod->m_count);
0375 iFunc(iWorker, iComm);
0376 auto count = std::count(iExpect.begin(), iExpect.end(), iTrans);
0377 if (count != iMod->m_count) {
0378 std::cout << "For trans " << static_cast<std::underlying_type<testOneOutputModule::Trans>::type>(iTrans)
0379 << " expected " << count << " and got " << iMod->m_count << std::endl;
0380 }
0381 CPPUNIT_ASSERT(iMod->m_count == count);
0382 iMod->m_count = 0;
0383 iWorker->reset();
0384 }
0385 }
0386
0387 template <typename T>
0388 void testOneOutputModule::testTransitions(std::shared_ptr<T> iMod, Expectations const& iExpect) {
0389 oneapi::tbb::global_control control(oneapi::tbb::global_control::max_allowed_parallelism, 1);
0390
0391 iMod->doPreallocate(m_preallocConfig);
0392 edm::WorkerT<edm::one::OutputModuleBase> w{iMod, m_desc, m_params.actions_};
0393 w.beginJob();
0394 edm::OutputModuleCommunicatorT<edm::one::OutputModuleBase> comm(iMod.get());
0395 for (auto& keyVal : m_transToFunc) {
0396 testTransition(iMod, &w, &comm, keyVal.first, iExpect, keyVal.second);
0397 }
0398 }
0399
0400 void testOneOutputModule::basicTest() {
0401
0402 edm::ServiceRegistry::Operate operate(serviceToken_);
0403
0404 edm::ParameterSet pset;
0405 auto testProd = std::make_shared<BasicOutputModule>(pset);
0406
0407 CPPUNIT_ASSERT(0 == testProd->m_count);
0408 testTransitions(testProd, {Trans::kEvent, Trans::kGlobalEndLuminosityBlock, Trans::kGlobalEndRun});
0409 }
0410
0411 void testOneOutputModule::runTest() {
0412
0413 edm::ServiceRegistry::Operate operate(serviceToken_);
0414
0415 edm::ParameterSet pset;
0416 auto testProd = std::make_shared<RunOutputModule>(pset);
0417
0418 CPPUNIT_ASSERT(0 == testProd->m_count);
0419 testTransitions(testProd,
0420 {Trans::kGlobalBeginRun,
0421 Trans::kEvent,
0422 Trans::kGlobalEndLuminosityBlock,
0423 Trans::kGlobalEndRun,
0424 Trans::kGlobalEndRun});
0425 }
0426
0427 void testOneOutputModule::lumiTest() {
0428
0429 edm::ServiceRegistry::Operate operate(serviceToken_);
0430
0431 edm::ParameterSet pset;
0432 auto testProd = std::make_shared<LumiOutputModule>(pset);
0433
0434 CPPUNIT_ASSERT(0 == testProd->m_count);
0435 testTransitions(testProd,
0436 {Trans::kGlobalBeginLuminosityBlock,
0437 Trans::kEvent,
0438 Trans::kGlobalEndLuminosityBlock,
0439 Trans::kGlobalEndLuminosityBlock,
0440 Trans::kGlobalEndRun});
0441 }
0442
0443 void testOneOutputModule::runCacheTest() {
0444
0445 edm::ServiceRegistry::Operate operate(serviceToken_);
0446
0447 edm::ParameterSet pset;
0448 auto testProd = std::make_shared<RunCacheOutputModule>(pset);
0449
0450 CPPUNIT_ASSERT(0 == testProd->m_count);
0451 testTransitions(testProd,
0452 {Trans::kGlobalBeginRun,
0453 Trans::kEvent,
0454 Trans::kGlobalEndLuminosityBlock,
0455 Trans::kGlobalEndRun,
0456 Trans::kGlobalEndRun});
0457 }
0458
0459 void testOneOutputModule::lumiCacheTest() {
0460
0461 edm::ServiceRegistry::Operate operate(serviceToken_);
0462
0463 edm::ParameterSet pset;
0464 auto testProd = std::make_shared<LumiCacheOutputModule>(pset);
0465
0466 CPPUNIT_ASSERT(0 == testProd->m_count);
0467 testTransitions(testProd,
0468 {Trans::kGlobalBeginLuminosityBlock,
0469 Trans::kEvent,
0470 Trans::kGlobalEndLuminosityBlock,
0471 Trans::kGlobalEndLuminosityBlock,
0472 Trans::kGlobalEndRun});
0473 }
0474
0475 void testOneOutputModule::fileTest() {
0476
0477 edm::ServiceRegistry::Operate operate(serviceToken_);
0478
0479 edm::ParameterSet pset;
0480 auto testProd = std::make_shared<FileOutputModule>(pset);
0481
0482 CPPUNIT_ASSERT(0 == testProd->m_count);
0483 testTransitions(testProd,
0484 {Trans::kGlobalOpenInputFile,
0485 Trans::kEvent,
0486 Trans::kGlobalEndLuminosityBlock,
0487 Trans::kGlobalEndRun,
0488 Trans::kGlobalCloseInputFile});
0489 }
0490
0491 void testOneOutputModule::resourceTest() {
0492
0493 edm::ServiceRegistry::Operate operate(serviceToken_);
0494
0495 edm::ParameterSet pset;
0496 auto testProd = std::make_shared<ResourceOutputModule>(pset);
0497
0498 CPPUNIT_ASSERT(0 == testProd->m_count);
0499 testTransitions(testProd, {Trans::kEvent, Trans::kGlobalEndLuminosityBlock, Trans::kGlobalEndRun});
0500 }