File indexing completed on 2021-12-24 02:18:52
0001
0002
0003
0004
0005
0006
0007
0008 #include <iostream>
0009 #include <atomic>
0010 #include <vector>
0011 #include <map>
0012 #include <functional>
0013 #include "oneapi/tbb/global_control.h"
0014 #include "FWCore/Framework/interface/one/OutputModule.h"
0015 #include "FWCore/Framework/interface/OutputModuleCommunicatorT.h"
0016 #include "FWCore/Framework/interface/TransitionInfoTypes.h"
0017 #include "FWCore/Framework/interface/maker/WorkerT.h"
0018 #include "FWCore/Framework/interface/OccurrenceTraits.h"
0019 #include "DataFormats/Provenance/interface/ProductRegistry.h"
0020 #include "DataFormats/Provenance/interface/BranchIDListHelper.h"
0021 #include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h"
0022 #include "FWCore/Framework/interface/HistoryAppender.h"
0023 #include "FWCore/Utilities/interface/GlobalIdentifier.h"
0024 #include "FWCore/Framework/interface/TriggerNamesService.h"
0025 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0026 #include "FWCore/ServiceRegistry/interface/ParentContext.h"
0027 #include "FWCore/ServiceRegistry/interface/Service.h"
0028 #include "FWCore/ServiceRegistry/interface/ServiceRegistry.h"
0029 #include "FWCore/Framework/interface/FileBlock.h"
0030 #include "FWCore/Framework/interface/PreallocationConfiguration.h"
0031 #include "FWCore/Concurrency/interface/WaitingTaskHolder.h"
0032
0033 #include "FWCore/Utilities/interface/Exception.h"
0034
0035 #include "cppunit/extensions/HelperMacros.h"
0036
0037 namespace edm {
0038 class ModuleCallingContext;
0039 }
0040
0041 class testOneOutputModule : public CppUnit::TestFixture {
0042 CPPUNIT_TEST_SUITE(testOneOutputModule);
0043
0044 CPPUNIT_TEST(basicTest);
0045 CPPUNIT_TEST(runTest);
0046 CPPUNIT_TEST(runCacheTest);
0047 CPPUNIT_TEST(lumiTest);
0048 CPPUNIT_TEST(lumiCacheTest);
0049 CPPUNIT_TEST(fileTest);
0050 CPPUNIT_TEST(resourceTest);
0051
0052 CPPUNIT_TEST_SUITE_END();
0053
0054 public:
0055 testOneOutputModule();
0056
0057 void setUp() {}
0058 void tearDown() {}
0059
0060 void basicTest();
0061 void runTest();
0062 void runCacheTest();
0063 void lumiTest();
0064 void lumiCacheTest();
0065 void fileTest();
0066 void resourceTest();
0067
0068 enum class Trans {
0069 kBeginJob,
0070 kGlobalOpenInputFile,
0071 kGlobalBeginRun,
0072 kGlobalBeginRunProduce,
0073 kGlobalBeginLuminosityBlock,
0074 kEvent,
0075 kGlobalEndLuminosityBlock,
0076 kGlobalEndRun,
0077 kGlobalCloseInputFile,
0078 kEndJob
0079 };
0080
0081 typedef std::vector<Trans> Expectations;
0082
0083 private:
0084 std::map<Trans, std::function<void(edm::Worker*, edm::OutputModuleCommunicator*)>> m_transToFunc;
0085
0086 edm::ProcessConfiguration m_procConfig;
0087 edm::PreallocationConfiguration m_preallocConfig;
0088 std::shared_ptr<edm::ProductRegistry> m_prodReg;
0089 std::shared_ptr<edm::BranchIDListHelper> m_idHelper;
0090 std::shared_ptr<edm::ThinnedAssociationsHelper> m_associationsHelper;
0091 std::unique_ptr<edm::EventPrincipal> m_ep;
0092 edm::HistoryAppender historyAppender_;
0093 std::shared_ptr<edm::LuminosityBlockPrincipal> m_lbp;
0094 std::shared_ptr<edm::RunPrincipal> m_rp;
0095 std::shared_ptr<edm::ActivityRegistry>
0096 m_actReg;
0097 edm::EventSetupImpl* m_es = nullptr;
0098 edm::ModuleDescription m_desc = {"Dummy", "dummy"};
0099 edm::WorkerParams m_params;
0100
0101 typedef edm::service::TriggerNamesService TNS;
0102 typedef edm::serviceregistry::ServiceWrapper<TNS> w_TNS;
0103 std::shared_ptr<w_TNS> tnsptr_;
0104 edm::ServiceToken serviceToken_;
0105
0106 template <typename T>
0107 void testTransitions(std::shared_ptr<T> iMod, Expectations const& iExpect);
0108
0109 template <typename Traits, typename Info>
0110 void doWork(edm::Worker* iBase, Info const& info, edm::StreamID id, edm::ParentContext const& iContext) {
0111 edm::FinalWaitingTask task;
0112 oneapi::tbb::task_group group;
0113 edm::ServiceToken token;
0114 iBase->doWorkAsync<Traits>(edm::WaitingTaskHolder(group, &task), info, token, id, iContext, nullptr);
0115 do {
0116 group.wait();
0117 } while (not task.done());
0118 if (auto e = task.exceptionPtr()) {
0119 std::rethrow_exception(*e);
0120 }
0121 }
0122
0123 class BasicOutputModule : public edm::one::OutputModule<> {
0124 public:
0125 using edm::one::OutputModuleBase::doPreallocate;
0126 BasicOutputModule(edm::ParameterSet const& iPSet)
0127 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<>(iPSet) {}
0128 unsigned int m_count = 0;
0129
0130 void write(edm::EventForOutput const&) override { ++m_count; }
0131 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0132 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0133 };
0134
0135 class RunOutputModule : public edm::one::OutputModule<edm::one::WatchRuns> {
0136 public:
0137 using edm::one::OutputModuleBase::doPreallocate;
0138 RunOutputModule(edm::ParameterSet const& iPSet)
0139 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::one::WatchRuns>(iPSet) {}
0140 unsigned int m_count = 0;
0141 void write(edm::EventForOutput const&) override { ++m_count; }
0142 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0143 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0144
0145 void beginRun(edm::RunForOutput const&) override { ++m_count; }
0146
0147 void endRun(edm::RunForOutput const&) override { ++m_count; }
0148 };
0149
0150 class LumiOutputModule : public edm::one::OutputModule<edm::one::WatchLuminosityBlocks> {
0151 public:
0152 using edm::one::OutputModuleBase::doPreallocate;
0153 LumiOutputModule(edm::ParameterSet const& iPSet)
0154 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::one::WatchLuminosityBlocks>(iPSet) {}
0155 unsigned int m_count = 0;
0156 void write(edm::EventForOutput const&) override { ++m_count; }
0157 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0158 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0159
0160 void beginLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0161
0162 void endLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0163 };
0164
0165 struct DummyCache {};
0166
0167 class RunCacheOutputModule : public edm::one::OutputModule<edm::RunCache<DummyCache>> {
0168 public:
0169 using edm::one::OutputModuleBase::doPreallocate;
0170 RunCacheOutputModule(edm::ParameterSet const& iPSet)
0171 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::RunCache<DummyCache>>(iPSet) {}
0172 mutable std::atomic<unsigned int> m_count = 0;
0173 void write(edm::EventForOutput const&) override { ++m_count; }
0174 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0175 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0176
0177 std::shared_ptr<DummyCache> globalBeginRun(edm::RunForOutput const&) const override {
0178 ++m_count;
0179 return std::shared_ptr<DummyCache>{};
0180 }
0181
0182 void globalEndRun(edm::RunForOutput const&) override { ++m_count; }
0183 };
0184
0185 class LumiCacheOutputModule : public edm::one::OutputModule<edm::LuminosityBlockCache<DummyCache>> {
0186 public:
0187 using edm::one::OutputModuleBase::doPreallocate;
0188 LumiCacheOutputModule(edm::ParameterSet const& iPSet)
0189 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::LuminosityBlockCache<DummyCache>>(iPSet) {}
0190 mutable std::atomic<unsigned int> m_count = 0;
0191 void write(edm::EventForOutput const&) override { ++m_count; }
0192 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0193 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0194
0195 std::shared_ptr<DummyCache> globalBeginLuminosityBlock(edm::LuminosityBlockForOutput const&) const override {
0196 ++m_count;
0197 return std::shared_ptr<DummyCache>{};
0198 }
0199
0200 void globalEndLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0201 };
0202
0203 class FileOutputModule : public edm::one::OutputModule<edm::WatchInputFiles> {
0204 public:
0205 using edm::one::OutputModuleBase::doPreallocate;
0206 FileOutputModule(edm::ParameterSet const& iPSet)
0207 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::WatchInputFiles>(iPSet) {}
0208 unsigned int m_count = 0;
0209 void write(edm::EventForOutput const&) override { ++m_count; }
0210 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0211 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0212
0213 void respondToOpenInputFile(edm::FileBlock const&) override { ++m_count; }
0214
0215 void respondToCloseInputFile(edm::FileBlock const&) override { ++m_count; }
0216 };
0217
0218 class ResourceOutputModule : public edm::one::OutputModule<edm::one::SharedResources> {
0219 public:
0220 using edm::one::OutputModuleBase::doPreallocate;
0221 ResourceOutputModule(edm::ParameterSet const& iPSet)
0222 : edm::one::OutputModuleBase(iPSet), edm::one::OutputModule<edm::one::SharedResources>(iPSet) {
0223 usesResource();
0224 }
0225 unsigned int m_count = 0;
0226
0227 void write(edm::EventForOutput const&) override { ++m_count; }
0228 void writeRun(edm::RunForOutput const&) override { ++m_count; }
0229 void writeLuminosityBlock(edm::LuminosityBlockForOutput const&) override { ++m_count; }
0230 };
0231 };
0232
0233 namespace {
0234
0235 edm::ActivityRegistry activityRegistry;
0236
0237 struct ShadowStreamID {
0238 constexpr ShadowStreamID() : value(0) {}
0239 unsigned int value;
0240 };
0241
0242 union IDUnion {
0243 IDUnion() : m_shadow() {}
0244 ShadowStreamID m_shadow;
0245 edm::StreamID m_id;
0246 };
0247 }
0248 static edm::StreamID makeID() {
0249 IDUnion u;
0250 assert(u.m_id.value() == 0);
0251 return u.m_id;
0252 }
0253 static const edm::StreamID s_streamID0 = makeID();
0254
0255
0256 CPPUNIT_TEST_SUITE_REGISTRATION(testOneOutputModule);
0257
0258 testOneOutputModule::testOneOutputModule()
0259 : m_prodReg(new edm::ProductRegistry{}),
0260 m_idHelper(new edm::BranchIDListHelper{}),
0261 m_associationsHelper(new edm::ThinnedAssociationsHelper{}),
0262 m_ep() {
0263
0264 m_prodReg->setFrozen();
0265 m_idHelper->updateFromRegistry(*m_prodReg);
0266 edm::EventID eventID = edm::EventID::firstValidEvent();
0267
0268 std::string uuid = edm::createGlobalIdentifier();
0269 edm::Timestamp now(1234567UL);
0270 auto runAux = std::make_shared<edm::RunAuxiliary>(eventID.run(), now, now);
0271 m_rp.reset(new edm::RunPrincipal(runAux, m_prodReg, m_procConfig, &historyAppender_, 0));
0272 edm::LuminosityBlockAuxiliary lumiAux(m_rp->run(), 1, now, now);
0273 m_lbp.reset(new edm::LuminosityBlockPrincipal(m_prodReg, m_procConfig, &historyAppender_, 0));
0274 m_lbp->setAux(lumiAux);
0275 m_lbp->setRunPrincipal(m_rp);
0276 edm::EventAuxiliary eventAux(eventID, uuid, now, true);
0277
0278 m_ep.reset(new edm::EventPrincipal(m_prodReg, m_idHelper, m_associationsHelper, m_procConfig, nullptr));
0279 m_ep->fillEventPrincipal(eventAux, nullptr);
0280 m_ep->setLuminosityBlockPrincipal(m_lbp.get());
0281 m_actReg.reset(new edm::ActivityRegistry);
0282
0283
0284 m_transToFunc[Trans::kGlobalOpenInputFile] = [](edm::Worker* iBase, edm::OutputModuleCommunicator*) {
0285 edm::FileBlock fb;
0286 iBase->respondToOpenInputFile(fb);
0287 };
0288
0289 m_transToFunc[Trans::kGlobalBeginRun] = [this](edm::Worker* iBase, edm::OutputModuleCommunicator*) {
0290 typedef edm::OccurrenceTraits<edm::RunPrincipal, edm::BranchActionGlobalBegin> Traits;
0291 edm::ParentContext parentContext;
0292 edm::RunTransitionInfo info(*m_rp, *m_es);
0293 doWork<Traits>(iBase, info, edm::StreamID::invalidStreamID(), parentContext);
0294 };
0295
0296 m_transToFunc[Trans::kGlobalBeginLuminosityBlock] = [this](edm::Worker* iBase, edm::OutputModuleCommunicator*) {
0297 typedef edm::OccurrenceTraits<edm::LuminosityBlockPrincipal, edm::BranchActionGlobalBegin> Traits;
0298 edm::ParentContext parentContext;
0299 edm::LumiTransitionInfo info(*m_lbp, *m_es);
0300 doWork<Traits>(iBase, info, edm::StreamID::invalidStreamID(), parentContext);
0301 };
0302
0303 m_transToFunc[Trans::kEvent] = [this](edm::Worker* iBase, edm::OutputModuleCommunicator*) {
0304 typedef edm::OccurrenceTraits<edm::EventPrincipal, edm::BranchActionStreamBegin> Traits;
0305 edm::StreamContext streamContext(s_streamID0, nullptr);
0306 edm::ParentContext parentContext(&streamContext);
0307 iBase->setActivityRegistry(m_actReg);
0308 edm::EventTransitionInfo info(*m_ep, *m_es);
0309 doWork<Traits>(iBase, info, s_streamID0, parentContext);
0310 };
0311
0312 m_transToFunc[Trans::kGlobalEndLuminosityBlock] = [this](edm::Worker* iBase, edm::OutputModuleCommunicator* iComm) {
0313 typedef edm::OccurrenceTraits<edm::LuminosityBlockPrincipal, edm::BranchActionGlobalEnd> Traits;
0314 edm::ParentContext parentContext;
0315 edm::LumiTransitionInfo info(*m_lbp, *m_es);
0316 doWork<Traits>(iBase, info, edm::StreamID::invalidStreamID(), parentContext);
0317 edm::FinalWaitingTask task;
0318 oneapi::tbb::task_group group;
0319 iComm->writeLumiAsync(edm::WaitingTaskHolder(group, &task), *m_lbp, nullptr, &activityRegistry);
0320 do {
0321 group.wait();
0322 } while (not task.done());
0323 if (task.exceptionPtr() != nullptr) {
0324 std::rethrow_exception(*task.exceptionPtr());
0325 }
0326 };
0327
0328 m_transToFunc[Trans::kGlobalEndRun] = [this](edm::Worker* iBase, edm::OutputModuleCommunicator* iComm) {
0329 typedef edm::OccurrenceTraits<edm::RunPrincipal, edm::BranchActionGlobalEnd> Traits;
0330 edm::ParentContext parentContext;
0331 edm::RunTransitionInfo info(*m_rp, *m_es);
0332 doWork<Traits>(iBase, info, edm::StreamID::invalidStreamID(), parentContext);
0333 edm::FinalWaitingTask task;
0334 oneapi::tbb::task_group group;
0335 iComm->writeRunAsync(edm::WaitingTaskHolder(group, &task), *m_rp, nullptr, &activityRegistry, nullptr);
0336 do {
0337 group.wait();
0338 } while (not task.done());
0339 if (task.exceptionPtr() != nullptr) {
0340 std::rethrow_exception(*task.exceptionPtr());
0341 }
0342 };
0343
0344 m_transToFunc[Trans::kGlobalCloseInputFile] = [](edm::Worker* iBase, edm::OutputModuleCommunicator*) {
0345 edm::FileBlock fb;
0346 iBase->respondToCloseInputFile(fb);
0347 };
0348
0349
0350
0351
0352
0353 edm::ParameterSet proc_pset;
0354
0355 std::string processName("HLT");
0356 proc_pset.addParameter<std::string>("@process_name", processName);
0357
0358 std::vector<std::string> paths;
0359 edm::ParameterSet trigPaths;
0360 trigPaths.addParameter<std::vector<std::string>>("@trigger_paths", paths);
0361 proc_pset.addParameter<edm::ParameterSet>("@trigger_paths", trigPaths);
0362
0363 std::vector<std::string> endPaths;
0364 proc_pset.addParameter<std::vector<std::string>>("@end_paths", endPaths);
0365
0366
0367 tnsptr_.reset(new w_TNS(std::make_unique<TNS>(proc_pset)));
0368
0369 serviceToken_ = edm::ServiceRegistry::createContaining(tnsptr_);
0370 }
0371
0372 namespace {
0373 template <typename T>
0374 void testTransition(std::shared_ptr<T> iMod,
0375 edm::Worker* iWorker,
0376 edm::OutputModuleCommunicator* iComm,
0377 testOneOutputModule::Trans iTrans,
0378 testOneOutputModule::Expectations const& iExpect,
0379 std::function<void(edm::Worker*, edm::OutputModuleCommunicator*)> iFunc) {
0380 assert(0 == iMod->m_count);
0381 iFunc(iWorker, iComm);
0382 auto count = std::count(iExpect.begin(), iExpect.end(), iTrans);
0383 if (count != iMod->m_count) {
0384 std::cout << "For trans " << static_cast<std::underlying_type<testOneOutputModule::Trans>::type>(iTrans)
0385 << " expected " << count << " and got " << iMod->m_count << std::endl;
0386 }
0387 CPPUNIT_ASSERT(iMod->m_count == count);
0388 iMod->m_count = 0;
0389 iWorker->reset();
0390 }
0391 }
0392
0393 template <typename T>
0394 void testOneOutputModule::testTransitions(std::shared_ptr<T> iMod, Expectations const& iExpect) {
0395 oneapi::tbb::global_control control(oneapi::tbb::global_control::max_allowed_parallelism, 1);
0396
0397 iMod->doPreallocate(m_preallocConfig);
0398 edm::WorkerT<edm::one::OutputModuleBase> w{iMod, m_desc, m_params.actions_};
0399 w.beginJob();
0400 edm::OutputModuleCommunicatorT<edm::one::OutputModuleBase> comm(iMod.get());
0401 for (auto& keyVal : m_transToFunc) {
0402 testTransition(iMod, &w, &comm, keyVal.first, iExpect, keyVal.second);
0403 }
0404 }
0405
0406 void testOneOutputModule::basicTest() {
0407
0408 edm::ServiceRegistry::Operate operate(serviceToken_);
0409
0410 edm::ParameterSet pset;
0411 auto testProd = std::make_shared<BasicOutputModule>(pset);
0412
0413 CPPUNIT_ASSERT(0 == testProd->m_count);
0414 testTransitions(testProd, {Trans::kEvent, Trans::kGlobalEndLuminosityBlock, Trans::kGlobalEndRun});
0415 }
0416
0417 void testOneOutputModule::runTest() {
0418
0419 edm::ServiceRegistry::Operate operate(serviceToken_);
0420
0421 edm::ParameterSet pset;
0422 auto testProd = std::make_shared<RunOutputModule>(pset);
0423
0424 CPPUNIT_ASSERT(0 == testProd->m_count);
0425 testTransitions(testProd,
0426 {Trans::kGlobalBeginRun,
0427 Trans::kEvent,
0428 Trans::kGlobalEndLuminosityBlock,
0429 Trans::kGlobalEndRun,
0430 Trans::kGlobalEndRun});
0431 }
0432
0433 void testOneOutputModule::lumiTest() {
0434
0435 edm::ServiceRegistry::Operate operate(serviceToken_);
0436
0437 edm::ParameterSet pset;
0438 auto testProd = std::make_shared<LumiOutputModule>(pset);
0439
0440 CPPUNIT_ASSERT(0 == testProd->m_count);
0441 testTransitions(testProd,
0442 {Trans::kGlobalBeginLuminosityBlock,
0443 Trans::kEvent,
0444 Trans::kGlobalEndLuminosityBlock,
0445 Trans::kGlobalEndLuminosityBlock,
0446 Trans::kGlobalEndRun});
0447 }
0448
0449 void testOneOutputModule::runCacheTest() {
0450
0451 edm::ServiceRegistry::Operate operate(serviceToken_);
0452
0453 edm::ParameterSet pset;
0454 auto testProd = std::make_shared<RunCacheOutputModule>(pset);
0455
0456 CPPUNIT_ASSERT(0 == testProd->m_count);
0457 testTransitions(testProd,
0458 {Trans::kGlobalBeginRun,
0459 Trans::kEvent,
0460 Trans::kGlobalEndLuminosityBlock,
0461 Trans::kGlobalEndRun,
0462 Trans::kGlobalEndRun});
0463 }
0464
0465 void testOneOutputModule::lumiCacheTest() {
0466
0467 edm::ServiceRegistry::Operate operate(serviceToken_);
0468
0469 edm::ParameterSet pset;
0470 auto testProd = std::make_shared<LumiCacheOutputModule>(pset);
0471
0472 CPPUNIT_ASSERT(0 == testProd->m_count);
0473 testTransitions(testProd,
0474 {Trans::kGlobalBeginLuminosityBlock,
0475 Trans::kEvent,
0476 Trans::kGlobalEndLuminosityBlock,
0477 Trans::kGlobalEndLuminosityBlock,
0478 Trans::kGlobalEndRun});
0479 }
0480
0481 void testOneOutputModule::fileTest() {
0482
0483 edm::ServiceRegistry::Operate operate(serviceToken_);
0484
0485 edm::ParameterSet pset;
0486 auto testProd = std::make_shared<FileOutputModule>(pset);
0487
0488 CPPUNIT_ASSERT(0 == testProd->m_count);
0489 testTransitions(testProd,
0490 {Trans::kGlobalOpenInputFile,
0491 Trans::kEvent,
0492 Trans::kGlobalEndLuminosityBlock,
0493 Trans::kGlobalEndRun,
0494 Trans::kGlobalCloseInputFile});
0495 }
0496
0497 void testOneOutputModule::resourceTest() {
0498
0499 edm::ServiceRegistry::Operate operate(serviceToken_);
0500
0501 edm::ParameterSet pset;
0502 auto testProd = std::make_shared<ResourceOutputModule>(pset);
0503
0504 CPPUNIT_ASSERT(0 == testProd->m_count);
0505 testTransitions(testProd, {Trans::kEvent, Trans::kGlobalEndLuminosityBlock, Trans::kGlobalEndRun});
0506 }