Services/plugins/DependencyGraph.cc

0001 /*
0002  * Simple Service to make a GraphViz graph of the modules runtime dependencies:
0003  *   - draw hard dependencies according to the "consumes" dependencies;
0004  *   - draw soft dependencies to reflect the order of scheduled modue in each path;
0005  *   - draw SubProcesses in subgraphs.
0006  *
0007  * Use GraphViz dot to generate an SVG representation of the dependencies:
0008  *
0009  *   dot -v -Tsvg dependency.dot -o dependency.svg
0010  *
0011  */
0012
0013 #include <iostream>
0014 #include <vector>
0015 #include <string>
0016 #include <type_traits>
0017
0018 // boost optional (used by boost graph) results in some false positives with -Wmaybe-uninitialized
0019 #pragma GCC diagnostic push
0020 #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
0021 #include <boost/graph/adjacency_list.hpp>
0022 #include <boost/graph/graphviz.hpp>
0023 #include <boost/graph/lookup_edge.hpp>
0024 #pragma GCC diagnostic pop
0025
0026 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0027 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0028 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0029 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0030 #include "FWCore/ParameterSet/interface/Registry.h"
0031 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0032 #include "FWCore/ServiceRegistry/interface/ConsumesInfo.h"
0033 #include "FWCore/ServiceRegistry/interface/PathsAndConsumesOfModulesBase.h"
0034 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
0035 #include "FWCore/Utilities/interface/Exception.h"
0036 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0037
0038 using namespace edm;
0039 using namespace edm::service;
0040
0041 namespace {
0042   namespace {
0043
0044     template <typename T>
0045     std::unordered_set<T> make_unordered_set(std::vector<T> &&entries) {
0046       std::unordered_set<T> u;
0047       for (T &entry : entries)
0048         u.insert(std::move(entry));
0049       return u;
0050     }
0051
0052   }  // namespace
0053 }  // namespace
0054
0055 class DependencyGraph {
0056 public:
0057   DependencyGraph(const ParameterSet &, ActivityRegistry &);
0058
0059   static void fillDescriptions(edm::ConfigurationDescriptions &descriptions);
0060
0061   void preSourceConstruction(ModuleDescription const &);
0062   void preBeginJob(PathsAndConsumesOfModulesBase const &, ProcessContext const &);
0063   void postBeginJob();
0064
0065 private:
0066   bool highlighted(std::string const &module) { return (m_highlightModules.find(module) != m_highlightModules.end()); }
0067
0068   enum class EDMModuleType { Unknown, Source, ESSource, ESProducer, EDAnalyzer, EDProducer, EDFilter, OutputModule };
0069
0070   static constexpr const char *module_type_desc[]{
0071       "Unknown", "Source", "ESSource", "ESProducer", "EDAnalyzer", "EDProducer", "EDFilter", "OutputModule"};
0072
0073   static constexpr const char *shapes[]{
0074       "note",      // Unknown
0075       "oval",      // Source
0076       "cylinder",  // ESSource
0077       "cylinder",  // ESProducer
0078       "oval",      // EDAnalyzer
0079       "box",       // EDProducer
0080       "diamond",   // EDFilter
0081       "oval",      // OutputModule
0082   };
0083
0084   static EDMModuleType edmModuleTypeEnum(edm::ModuleDescription const &module);
0085
0086   static const char *edmModuleType(edm::ModuleDescription const &module);
0087
0088   struct node {
0089     std::string label;
0090     std::string class_;
0091     unsigned int id;
0092     EDMModuleType type;
0093     bool scheduled;
0094   };
0095
0096   using GraphvizAttributes = std::map<std::string, std::string>;
0097
0098   // directed graph, with `node` properties attached to each vertex
0099   boost::subgraph<boost::adjacency_list<
0100       // edge list
0101       boost::vecS,
0102       // vertex list
0103       boost::vecS,
0104       boost::directedS,
0105       // vertex properties
0106       boost::property<boost::vertex_attribute_t,
0107                       GraphvizAttributes,  // Graphviz vertex attributes
0108                       node>,
0109       // edge propoerties
0110       boost::property<boost::edge_index_t,
0111                       int,  // used internally by boost::subgraph
0112                       boost::property<boost::edge_attribute_t, GraphvizAttributes>>,  // Graphviz edge attributes
0113       // graph properties
0114       boost::property<
0115           boost::graph_name_t,
0116           std::string,  // name each boost::subgraph
0117           boost::property<boost::graph_graph_attribute_t,
0118                           GraphvizAttributes,  // Graphviz graph attributes
0119                           boost::property<boost::graph_vertex_attribute_t,
0120                                           GraphvizAttributes,
0121                                           boost::property<boost::graph_edge_attribute_t, GraphvizAttributes>>>>>>
0122       m_graph;
0123
0124   std::string m_filename;
0125   std::unordered_set<std::string> m_highlightModules;
0126
0127   bool m_showPathDependencies;
0128   bool m_initialized;
0129 };
0130
0131 constexpr const char *DependencyGraph::module_type_desc[];
0132
0133 constexpr const char *DependencyGraph::shapes[];
0134
0135 DependencyGraph::EDMModuleType DependencyGraph::edmModuleTypeEnum(edm::ModuleDescription const &module) {
0136   auto const &registry = *edm::pset::Registry::instance();
0137   auto const &pset = *registry.getMapped(module.parameterSetID());
0138
0139   if (not pset.existsAs<std::string>("@module_edm_type"))
0140     return EDMModuleType::Unknown;
0141
0142   std::string const &t = pset.getParameter<std::string>("@module_edm_type");
0143   for (EDMModuleType v : {EDMModuleType::Source,
0144                           EDMModuleType::ESSource,
0145                           EDMModuleType::ESProducer,
0146                           EDMModuleType::EDAnalyzer,
0147                           EDMModuleType::EDProducer,
0148                           EDMModuleType::EDFilter,
0149                           EDMModuleType::OutputModule}) {
0150     if (t == module_type_desc[static_cast<std::underlying_type_t<EDMModuleType>>(v)])
0151       return v;
0152   }
0153   return EDMModuleType::Unknown;
0154 }
0155
0156 const char *DependencyGraph::edmModuleType(edm::ModuleDescription const &module) {
0157   return module_type_desc[static_cast<std::underlying_type_t<EDMModuleType>>(edmModuleTypeEnum(module))];
0158 }
0159
0160 void DependencyGraph::fillDescriptions(edm::ConfigurationDescriptions &descriptions) {
0161   edm::ParameterSetDescription desc;
0162   desc.addUntracked<std::string>("fileName", "dependency.dot");
0163   desc.addUntracked<std::vector<std::string>>("highlightModules", {});
0164   desc.addUntracked<bool>("showPathDependencies", true);
0165   descriptions.add("DependencyGraph", desc);
0166 }
0167
0168 DependencyGraph::DependencyGraph(ParameterSet const &config, ActivityRegistry &registry)
0169     : m_filename(config.getUntrackedParameter<std::string>("fileName")),
0170       m_highlightModules(
0171           make_unordered_set(config.getUntrackedParameter<std::vector<std::string>>("highlightModules"))),
0172       m_showPathDependencies(config.getUntrackedParameter<bool>("showPathDependencies")),
0173       m_initialized(false) {
0174   registry.watchPreSourceConstruction(this, &DependencyGraph::preSourceConstruction);
0175   registry.watchPreBeginJob(this, &DependencyGraph::preBeginJob);
0176   registry.watchPostBeginJob(this, &DependencyGraph::postBeginJob);
0177 }
0178
0179 // adaptor to use range-based for loops with boost::graph edges(...) and vertices(...) functions
0180 template <typename I>
0181 struct iterator_pair_as_a_range : std::pair<I, I> {
0182 public:
0183   using std::pair<I, I>::pair;
0184
0185   I begin() { return this->first; }
0186   I end() { return this->second; }
0187 };
0188
0189 template <typename I>
0190 iterator_pair_as_a_range<I> make_range(std::pair<I, I> p) {
0191   return iterator_pair_as_a_range<I>(p);
0192 }
0193
0194 void DependencyGraph::preSourceConstruction(ModuleDescription const &module) {
0195   // create graph vertex for the source module and fill its attributes
0196   boost::add_vertex(m_graph);
0197   m_graph.m_graph[module.id()] =
0198       node{module.moduleLabel(), module.moduleName(), module.id(), EDMModuleType::Source, true};
0199   auto &attributes = boost::get(boost::get(boost::vertex_attribute, m_graph), 0);
0200   attributes["label"] = module.moduleLabel();
0201   attributes["tooltip"] = module.moduleName();
0202   attributes["shape"] = shapes[static_cast<std::underlying_type_t<EDMModuleType>>(EDMModuleType::Source)];
0203   attributes["style"] = "filled";
0204   attributes["color"] = "black";
0205   attributes["fillcolor"] = highlighted(module.moduleLabel()) ? "lightgreen" : "white";
0206 }
0207
0208 void DependencyGraph::preBeginJob(PathsAndConsumesOfModulesBase const &pathsAndConsumes,
0209                                   ProcessContext const &context) {
0210   // if the Service is not in the main Process do not do anything
0211   if (context.isSubProcess() and not m_initialized) {
0212     edm::LogError("DependencyGraph") << "You have requested an instance of the DependencyGraph Service in the \""
0213                                      << context.processName()
0214                                      << "\" SubProcess, which is not supported.\nPlease move it to the main process.";
0215     return;
0216   }
0217
0218   if (not context.isSubProcess()) {
0219     // set the graph name property to the process name
0220     boost::get_property(m_graph, boost::graph_name) = context.processName();
0221     boost::get_property(m_graph, boost::graph_graph_attribute)["label"] = "process " + context.processName();
0222     boost::get_property(m_graph, boost::graph_graph_attribute)["labelloc"] = "top";
0223
0224     // create graph vertices associated to all modules in the process
0225     auto size = pathsAndConsumes.largestModuleID() - boost::num_vertices(m_graph) + 1;
0226     for (size_t i = 0; i < size; ++i)
0227       boost::add_vertex(m_graph);
0228
0229     m_initialized = true;
0230   } else {
0231     // create a subgraph to match the subprocess
0232     auto &graph = m_graph.create_subgraph();
0233
0234     // set the subgraph name property to the subprocess name
0235     boost::get_property(graph, boost::graph_name) = "cluster" + context.processName();
0236     boost::get_property(graph, boost::graph_graph_attribute)["label"] = "subprocess " + context.processName();
0237     boost::get_property(graph, boost::graph_graph_attribute)["labelloc"] = "top";
0238
0239     // create graph vertices associated to all modules in the subprocess
0240     auto size = pathsAndConsumes.largestModuleID() - boost::num_vertices(m_graph) + 1;
0241     for (size_t i = 0; i < size; ++i)
0242       boost::add_vertex(graph);
0243   }
0244
0245   // set the vertices properties (use the module id as the global index into the graph)
0246   for (edm::ModuleDescription const *module : pathsAndConsumes.allModules()) {
0247     m_graph.m_graph[module->id()] = {
0248         module->moduleLabel(), module->moduleName(), module->id(), edmModuleTypeEnum(*module), false};
0249
0250     auto &attributes = boost::get(boost::get(boost::vertex_attribute, m_graph), module->id());
0251     attributes["label"] = module->moduleLabel();
0252     attributes["tooltip"] = module->moduleName();
0253     attributes["shape"] = shapes[static_cast<std::underlying_type_t<EDMModuleType>>(edmModuleTypeEnum(*module))];
0254     attributes["style"] = "filled";
0255     attributes["color"] = "black";
0256     attributes["fillcolor"] = highlighted(module->moduleLabel()) ? "green" : "lightgrey";
0257   }
0258
0259   // paths and endpaths
0260   auto const &paths = pathsAndConsumes.paths();
0261   auto const &endps = pathsAndConsumes.endPaths();
0262
0263   // add graph edges associated to module dependencies
0264   for (edm::ModuleDescription const *consumer : pathsAndConsumes.allModules()) {
0265     for (edm::ModuleDescription const *module : pathsAndConsumes.modulesWhoseProductsAreConsumedBy(consumer->id())) {
0266       edm::LogInfo("DependencyGraph") << "module " << consumer->moduleLabel() << " depends on module "
0267                                       << module->moduleLabel();
0268       auto edge_status = boost::add_edge(consumer->id(), module->id(), m_graph);
0269       // highlight the arrow between highlighted nodes
0270       if (highlighted(module->moduleLabel()) and highlighted(consumer->moduleLabel())) {
0271         auto const &edge = edge_status.first;
0272         auto &attributes = boost::get(boost::get(boost::edge_attribute, m_graph), edge);
0273         attributes["color"] = "darkgreen";
0274       }
0275     }
0276   }
0277
0278   // marke the modules in the paths as scheduled, and add a soft dependency to reflect the order of modules along each path
0279   edm::ModuleDescription const *previous;
0280   for (unsigned int i = 0; i < paths.size(); ++i) {
0281     previous = nullptr;
0282     for (edm::ModuleDescription const *module : pathsAndConsumes.modulesOnPath(i)) {
0283       m_graph.m_graph[module->id()].scheduled = true;
0284       auto &attributes = boost::get(boost::get(boost::vertex_attribute, m_graph), module->id());
0285       attributes["fillcolor"] = highlighted(module->moduleLabel()) ? "lightgreen" : "white";
0286       if (previous and m_showPathDependencies) {
0287         edm::LogInfo("DependencyGraph") << "module " << module->moduleLabel() << " follows module "
0288                                         << previous->moduleLabel() << " in Path " << i;
0289         auto edge_status = boost::lookup_edge(module->id(), previous->id(), m_graph);
0290         bool found = edge_status.second;
0291         if (not found) {
0292           edge_status = boost::add_edge(module->id(), previous->id(), m_graph);
0293           auto const &edge = edge_status.first;
0294           auto &edgeAttributes = boost::get(boost::get(boost::edge_attribute, m_graph), edge);
0295           edgeAttributes["style"] = "dashed";
0296           // highlight the arrow between highlighted nodes
0297           if (highlighted(module->moduleLabel()) and highlighted(previous->moduleLabel()))
0298             edgeAttributes["color"] = "darkgreen";
0299         }
0300       }
0301       previous = module;
0302     }
0303   }
0304   for (unsigned int i = 0; i < endps.size(); ++i) {
0305     previous = nullptr;
0306     for (edm::ModuleDescription const *module : pathsAndConsumes.modulesOnEndPath(i)) {
0307       m_graph.m_graph[module->id()].scheduled = true;
0308       auto &attributes = boost::get(boost::get(boost::vertex_attribute, m_graph), module->id());
0309       attributes["fillcolor"] = highlighted(module->moduleLabel()) ? "lightgreen" : "white";
0310       if (previous and m_showPathDependencies) {
0311         edm::LogInfo("DependencyGraph") << "module " << module->moduleLabel() << " follows module "
0312                                         << previous->moduleLabel() << " in EndPath " << i;
0313         auto edge_status = boost::lookup_edge(module->id(), previous->id(), m_graph);
0314         bool found = edge_status.second;
0315         if (not found) {
0316           edge_status = boost::add_edge(module->id(), previous->id(), m_graph);
0317           auto const &edge = edge_status.first;
0318           auto &edgeAttributes = boost::get(boost::get(boost::edge_attribute, m_graph), edge);
0319           edgeAttributes["style"] = "dashed";
0320           // highlight the arrow between highlighted nodes
0321           if (highlighted(module->moduleLabel()) and highlighted(previous->moduleLabel()))
0322             edgeAttributes["color"] = "darkgreen";
0323         }
0324       }
0325       previous = module;
0326     }
0327   }
0328 }
0329
0330 void DependencyGraph::postBeginJob() {
0331   if (not m_initialized)
0332     return;
0333
0334   // draw the dependency graph
0335   std::ofstream out(m_filename);
0336   boost::write_graphviz(out, m_graph);
0337   out.close();
0338 }
0339
0340 namespace edm {
0341   namespace service {
0342
0343     inline bool isProcessWideService(DependencyGraph const *) { return true; }
0344
0345   }  // namespace service
0346 }  // namespace edm
0347
0348 // define as a framework servie
0349 #include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
0350 DEFINE_FWK_SERVICE(DependencyGraph);