Services/plugins/DependencyGraph.cc

0001 /*
0002  * Simple Service to make a GraphViz graph of the modules runtime dependencies:
0003  *   - draw hard dependencies according to the "consumes" dependencies;
0004  *   - draw soft dependencies to reflect the order of scheduled modue in each path;
0005  *   - draw SubProcesses in subgraphs.
0006  *
0007  * Use GraphViz dot to generate an SVG representation of the dependencies:
0008  *
0009  *   dot -v -Tsvg dependency.dot -o dependency.svg
0010  *
0011  */
0012
0013 #include <iostream>
0014 #include <vector>
0015 #include <string>
0016 #include <type_traits>
0017
0018 // boost optional (used by boost graph) results in some false positives with -Wmaybe-uninitialized
0019 #pragma GCC diagnostic push
0020 #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
0021 #include <boost/graph/adjacency_list.hpp>
0022 #include <boost/graph/graphviz.hpp>
0023 #include <boost/graph/lookup_edge.hpp>
0024 #pragma GCC diagnostic pop
0025
0026 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0027 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0028 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0029 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0030 #include "FWCore/ParameterSet/interface/Registry.h"
0031 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0032 #include "FWCore/ServiceRegistry/interface/ConsumesInfo.h"
0033 #include "FWCore/ServiceRegistry/interface/PathsAndConsumesOfModulesBase.h"
0034 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
0035 #include "FWCore/Utilities/interface/Exception.h"
0036 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0037
0038 using namespace edm;
0039 using namespace edm::service;
0040
0041 namespace {
0042   namespace {
0043
0044     template <typename T>
0045     std::unordered_set<T> make_unordered_set(std::vector<T> &&entries) {
0046       std::unordered_set<T> u;
0047       for (T &entry : entries)
0048         u.insert(std::move(entry));
0049       return u;
0050     }
0051
0052   }  // namespace
0053 }  // namespace
0054
0055 class DependencyGraph {
0056 public:
0057   DependencyGraph(const ParameterSet &, ActivityRegistry &);
0058
0059   static void fillDescriptions(edm::ConfigurationDescriptions &descriptions);
0060
0061   void preSourceConstruction(ModuleDescription const &);
0062   void preBeginJob(PathsAndConsumesOfModulesBase const &, ProcessContext const &);
0063   void postBeginJob();
0064
0065 private:
0066   bool highlighted(std::string const &module) { return (m_highlightModules.find(module) != m_highlightModules.end()); }
0067
0068   enum class EDMModuleType { Unknown, Source, ESSource, ESProducer, EDAnalyzer, EDProducer, EDFilter, OutputModule };
0069
0070   static constexpr const char *module_type_desc[]{
0071       "Unknown", "Source", "ESSource", "ESProducer", "EDAnalyzer", "EDProducer", "EDFilter", "OutputModule"};
0072
0073   static constexpr const char *shapes[]{
0074       "note",      // Unknown
0075       "oval",      // Source
0076       "cylinder",  // ESSource
0077       "cylinder",  // ESProducer
0078       "oval",      // EDAnalyzer
0079       "box",       // EDProducer
0080       "diamond",   // EDFilter
0081       "oval",      // OutputModule
0082   };
0083
0084   static EDMModuleType edmModuleTypeEnum(edm::ModuleDescription const &module);
0085
0086   static const char *edmModuleType(edm::ModuleDescription const &module);
0087
0088   struct node {
0089     std::string label;
0090     std::string class_;
0091     unsigned int id;
0092     EDMModuleType type;
0093     bool scheduled;
0094   };
0095
0096   using GraphvizAttributes = std::map<std::string, std::string>;
0097
0098   // directed graph, with `node` properties attached to each vertex
0099   using GraphType = boost::subgraph<boost::adjacency_list<
0100       // edge list
0101       boost::vecS,
0102       // vertex list
0103       boost::vecS,
0104       boost::directedS,
0105       // vertex properties
0106       boost::property<boost::vertex_attribute_t,
0107                       GraphvizAttributes,  // Graphviz vertex attributes
0108                       node>,
0109       // edge propoerties
0110       boost::property<boost::edge_index_t,
0111                       int,  // used internally by boost::subgraph
0112                       boost::property<boost::edge_attribute_t, GraphvizAttributes>>,  // Graphviz edge attributes
0113       // graph properties
0114       boost::property<
0115           boost::graph_name_t,
0116           std::string,  // name each boost::subgraph
0117           boost::property<boost::graph_graph_attribute_t,
0118                           GraphvizAttributes,  // Graphviz graph attributes
0119                           boost::property<boost::graph_vertex_attribute_t,
0120                                           GraphvizAttributes,
0121                                           boost::property<boost::graph_edge_attribute_t, GraphvizAttributes>>>>>>;
0122   GraphType m_graph;
0123
0124   std::string m_filename;
0125   std::unordered_set<std::string> m_highlightModules;
0126
0127   bool m_showPathDependencies;
0128   bool m_initialized;
0129 };
0130
0131 constexpr const char *DependencyGraph::module_type_desc[];
0132
0133 constexpr const char *DependencyGraph::shapes[];
0134
0135 DependencyGraph::EDMModuleType DependencyGraph::edmModuleTypeEnum(edm::ModuleDescription const &module) {
0136   auto const &registry = *edm::pset::Registry::instance();
0137   auto const &pset = *registry.getMapped(module.parameterSetID());
0138
0139   if (not pset.existsAs<std::string>("@module_edm_type"))
0140     return EDMModuleType::Unknown;
0141
0142   std::string const &t = pset.getParameter<std::string>("@module_edm_type");
0143   for (EDMModuleType v : {EDMModuleType::Source,
0144                           EDMModuleType::ESSource,
0145                           EDMModuleType::ESProducer,
0146                           EDMModuleType::EDAnalyzer,
0147                           EDMModuleType::EDProducer,
0148                           EDMModuleType::EDFilter,
0149                           EDMModuleType::OutputModule}) {
0150     if (t == module_type_desc[static_cast<std::underlying_type_t<EDMModuleType>>(v)])
0151       return v;
0152   }
0153   return EDMModuleType::Unknown;
0154 }
0155
0156 const char *DependencyGraph::edmModuleType(edm::ModuleDescription const &module) {
0157   return module_type_desc[static_cast<std::underlying_type_t<EDMModuleType>>(edmModuleTypeEnum(module))];
0158 }
0159
0160 void DependencyGraph::fillDescriptions(edm::ConfigurationDescriptions &descriptions) {
0161   edm::ParameterSetDescription desc;
0162   desc.addUntracked<std::string>("fileName", "dependency.dot");
0163   desc.addUntracked<std::vector<std::string>>("highlightModules", {});
0164   desc.addUntracked<bool>("showPathDependencies", true);
0165   descriptions.add("DependencyGraph", desc);
0166 }
0167
0168 DependencyGraph::DependencyGraph(ParameterSet const &config, ActivityRegistry &registry)
0169     : m_filename(config.getUntrackedParameter<std::string>("fileName")),
0170       m_highlightModules(
0171           make_unordered_set(config.getUntrackedParameter<std::vector<std::string>>("highlightModules"))),
0172       m_showPathDependencies(config.getUntrackedParameter<bool>("showPathDependencies")),
0173       m_initialized(false) {
0174   registry.watchPreSourceConstruction(this, &DependencyGraph::preSourceConstruction);
0175   registry.watchPreBeginJob(this, &DependencyGraph::preBeginJob);
0176   registry.watchPostBeginJob(this, &DependencyGraph::postBeginJob);
0177 }
0178
0179 // adaptor to use range-based for loops with boost::graph edges(...) and vertices(...) functions
0180 template <typename I>
0181 struct iterator_pair_as_a_range : std::pair<I, I> {
0182 public:
0183   using std::pair<I, I>::pair;
0184
0185   I begin() { return this->first; }
0186   I end() { return this->second; }
0187 };
0188
0189 template <typename I>
0190 iterator_pair_as_a_range<I> make_range(std::pair<I, I> p) {
0191   return iterator_pair_as_a_range<I>(p);
0192 }
0193
0194 void DependencyGraph::preSourceConstruction(ModuleDescription const &module) {
0195   // create graph vertex for the source module and fill its attributes
0196   boost::add_vertex(m_graph);
0197   m_graph.m_graph[module.id()] =
0198       node{module.moduleLabel(), module.moduleName(), module.id(), EDMModuleType::Source, true};
0199   auto &attributes = boost::get(boost::get(boost::vertex_attribute, m_graph), 0);
0200   attributes["label"] = module.moduleLabel();
0201   attributes["tooltip"] = module.moduleName();
0202   attributes["shape"] = shapes[static_cast<std::underlying_type_t<EDMModuleType>>(EDMModuleType::Source)];
0203   attributes["style"] = "filled";
0204   attributes["color"] = "black";
0205   attributes["fillcolor"] = highlighted(module.moduleLabel()) ? "lightgreen" : "white";
0206 }
0207
0208 void DependencyGraph::preBeginJob(PathsAndConsumesOfModulesBase const &pathsAndConsumes,
0209                                   ProcessContext const &context) {
0210   // if the Service is not in the main Process do not do anything
0211   if (context.isSubProcess() and not m_initialized) {
0212     edm::LogError("DependencyGraph") << "You have requested an instance of the DependencyGraph Service in the \""
0213                                      << context.processName()
0214                                      << "\" SubProcess, which is not supported.\nPlease move it to the main process.";
0215     return;
0216   }
0217
0218   if (not context.isSubProcess()) {
0219     // set the graph name property to the process name
0220     boost::get_property(m_graph, boost::graph_name) = context.processName();
0221     boost::get_property(m_graph, boost::graph_graph_attribute)["label"] = "process " + context.processName();
0222     boost::get_property(m_graph, boost::graph_graph_attribute)["labelloc"] = "top";
0223
0224     // create graph vertices associated to all modules in the process
0225     auto size = pathsAndConsumes.largestModuleID() - boost::num_vertices(m_graph) + 1;
0226     for (size_t i = 0; i < size; ++i)
0227       boost::add_vertex(m_graph);
0228
0229     m_initialized = true;
0230   } else {
0231     // create a subgraph to match the subprocess
0232     auto &graph = m_graph.create_subgraph();
0233
0234     // set the subgraph name property to the subprocess name
0235     boost::get_property(graph, boost::graph_name) = "cluster" + context.processName();
0236     boost::get_property(graph, boost::graph_graph_attribute)["label"] = "subprocess " + context.processName();
0237     boost::get_property(graph, boost::graph_graph_attribute)["labelloc"] = "top";
0238
0239     // create graph vertices associated to all modules in the subprocess
0240     auto size = pathsAndConsumes.largestModuleID() - boost::num_vertices(m_graph) + 1;
0241     for (size_t i = 0; i < size; ++i)
0242       boost::add_vertex(graph);
0243   }
0244
0245   // set the vertices properties (use the module id as the global index into the graph)
0246   for (edm::ModuleDescription const *module : pathsAndConsumes.allModules()) {
0247     m_graph.m_graph[module->id()] = {
0248         module->moduleLabel(), module->moduleName(), module->id(), edmModuleTypeEnum(*module), false};
0249
0250     auto &attributes = boost::get(boost::get(boost::vertex_attribute, m_graph), module->id());
0251     attributes["label"] = module->moduleLabel();
0252     attributes["tooltip"] = module->moduleName();
0253     attributes["shape"] = shapes[static_cast<std::underlying_type_t<EDMModuleType>>(edmModuleTypeEnum(*module))];
0254     attributes["style"] = "filled";
0255     attributes["color"] = "black";
0256     attributes["fillcolor"] = highlighted(module->moduleLabel()) ? "green" : "lightgrey";
0257   }
0258
0259   // paths and endpaths
0260   auto const &paths = pathsAndConsumes.paths();
0261   auto const &endps = pathsAndConsumes.endPaths();
0262
0263   // add graph edges associated to module dependencies
0264   for (edm::ModuleDescription const *consumer : pathsAndConsumes.allModules()) {
0265     for (edm::ModuleDescription const *module : pathsAndConsumes.modulesWhoseProductsAreConsumedBy(consumer->id())) {
0266       edm::LogInfo("DependencyGraph") << "module " << consumer->moduleLabel() << " depends on module "
0267                                       << module->moduleLabel();
0268       auto edge_status = boost::add_edge(consumer->id(), module->id(), m_graph);
0269       // highlight the edge between highlighted nodes
0270       if (highlighted(module->moduleLabel()) and highlighted(consumer->moduleLabel())) {
0271         auto const &edge = edge_status.first;
0272         auto &attributes = boost::get(boost::get(boost::edge_attribute, m_graph), edge);
0273         attributes["color"] = "darkgreen";
0274       }
0275     }
0276   }
0277
0278   // save each Path and EndPath as a Graphviz subgraph
0279   for (unsigned int i = 0; i < paths.size(); ++i) {
0280     // create a subgraph to match the Path
0281     auto &graph = m_graph.create_subgraph();
0282
0283     // set the subgraph name property to the Path name
0284     boost::get_property(graph, boost::graph_name) = paths[i];
0285     boost::get_property(graph, boost::graph_graph_attribute)["label"] = "Path " + paths[i];
0286     boost::get_property(graph, boost::graph_graph_attribute)["labelloc"] = "bottom";
0287
0288     // add to the subgraph the node corresponding to the scheduled modules on the Path
0289     for (edm::ModuleDescription const *module : pathsAndConsumes.modulesOnPath(i)) {
0290       boost::add_vertex(module->id(), graph);
0291     }
0292   }
0293   for (unsigned int i = 0; i < endps.size(); ++i) {
0294     // create a subgraph to match the EndPath
0295     auto &graph = m_graph.create_subgraph();
0296
0297     // set the subgraph name property to the EndPath name
0298     boost::get_property(graph, boost::graph_name) = paths[i];
0299     boost::get_property(graph, boost::graph_graph_attribute)["label"] = "EndPath " + paths[i];
0300     boost::get_property(graph, boost::graph_graph_attribute)["labelloc"] = "bottom";
0301
0302     // add to the subgraph the node corresponding to the scheduled modules on the EndPath
0303     for (edm::ModuleDescription const *module : pathsAndConsumes.modulesOnEndPath(i)) {
0304       boost::add_vertex(module->id(), graph);
0305     }
0306   }
0307
0308   // optionally, add a dependency of the TriggerResults module on the PathStatusInserter modules
0309   const int size = boost::num_vertices(m_graph);
0310   int triggerResults = -1;
0311   bool highlightTriggerResults = false;
0312   for (int i = 0; i < size; ++i) {
0313     if (m_graph.m_graph[i].label == "TriggerResults") {
0314       triggerResults = i;
0315       highlightTriggerResults = highlighted("TriggerResults");
0316       break;
0317     }
0318   }
0319
0320   // mark the modules in the paths as scheduled, and add a soft dependency to reflect the order of modules along each path
0321   edm::ModuleDescription const *previous;
0322   for (unsigned int i = 0; i < paths.size(); ++i) {
0323     previous = nullptr;
0324     for (edm::ModuleDescription const *module : pathsAndConsumes.modulesOnPath(i)) {
0325       m_graph.m_graph[module->id()].scheduled = true;
0326       auto &attributes = boost::get(boost::get(boost::vertex_attribute, m_graph), module->id());
0327       attributes["fillcolor"] = highlighted(module->moduleLabel()) ? "lightgreen" : "white";
0328       if (previous and m_showPathDependencies) {
0329         edm::LogInfo("DependencyGraph") << "module " << module->moduleLabel() << " follows module "
0330                                         << previous->moduleLabel() << " in Path " << paths[i];
0331         auto edge_status = boost::lookup_edge(module->id(), previous->id(), m_graph);
0332         bool found = edge_status.second;
0333         if (not found) {
0334           edge_status = boost::add_edge(module->id(), previous->id(), m_graph);
0335           auto const &edge = edge_status.first;
0336           auto &edgeAttributes = boost::get(boost::get(boost::edge_attribute, m_graph), edge);
0337           edgeAttributes["style"] = "dashed";
0338           // highlight the edge between highlighted nodes
0339           if (highlighted(module->moduleLabel()) and highlighted(previous->moduleLabel()))
0340             edgeAttributes["color"] = "darkgreen";
0341         }
0342       }
0343       previous = module;
0344     }
0345     // previous points to the last scheduled module on the path
0346     if (previous and m_showPathDependencies) {
0347       // look for the PathStatusInserter module corresponding to this path
0348       for (int j = 0; j < size; ++j) {
0349         if (m_graph.m_graph[j].label == paths[i]) {
0350           edm::LogInfo("DependencyGraph") << "module " << paths[i] << " implicitly follows module "
0351                                           << previous->moduleLabel() << " in Path " << paths[i];
0352           // add an edge from the PathStatusInserter module to the last module scheduled on the path
0353           auto edge_status = boost::add_edge(j, previous->id(), m_graph);
0354           auto const &edge = edge_status.first;
0355           auto &edgeAttributes = boost::get(boost::get(boost::edge_attribute, m_graph), edge);
0356           edgeAttributes["style"] = "dashed";
0357           // highlight the edge between highlighted nodes
0358           bool highlightedPath = highlighted(paths[i]);
0359           if (highlightedPath and highlighted(previous->moduleLabel()))
0360             edgeAttributes["color"] = "darkgreen";
0361           if (triggerResults > 0) {
0362             // add an edge from the TriggerResults module to the PathStatusInserter module
0363             auto edge_status = boost::add_edge(triggerResults, j, m_graph);
0364             auto const &edge = edge_status.first;
0365             auto &edgeAttributes = boost::get(boost::get(boost::edge_attribute, m_graph), edge);
0366             edgeAttributes["style"] = "dashed";
0367             // highlight the edge between highlighted nodes
0368             if (highlightedPath and highlightTriggerResults)
0369               edgeAttributes["color"] = "darkgreen";
0370           }
0371           break;
0372         }
0373       }
0374     }
0375   }
0376
0377   // mark the modules in the endpaths as scheduled, and add a soft dependency to reflect the order of modules along each endpath
0378   for (unsigned int i = 0; i < endps.size(); ++i) {
0379     previous = nullptr;
0380     for (edm::ModuleDescription const *module : pathsAndConsumes.modulesOnEndPath(i)) {
0381       m_graph.m_graph[module->id()].scheduled = true;
0382       auto &attributes = boost::get(boost::get(boost::vertex_attribute, m_graph), module->id());
0383       attributes["fillcolor"] = highlighted(module->moduleLabel()) ? "lightgreen" : "white";
0384       if (previous and m_showPathDependencies) {
0385         edm::LogInfo("DependencyGraph") << "module " << module->moduleLabel() << " follows module "
0386                                         << previous->moduleLabel() << " in EndPath " << i;
0387         auto edge_status = boost::lookup_edge(module->id(), previous->id(), m_graph);
0388         bool found = edge_status.second;
0389         if (not found) {
0390           edge_status = boost::add_edge(module->id(), previous->id(), m_graph);
0391           auto const &edge = edge_status.first;
0392           auto &edgeAttributes = boost::get(boost::get(boost::edge_attribute, m_graph), edge);
0393           edgeAttributes["style"] = "dashed";
0394           // highlight the edge between highlighted nodes
0395           if (highlighted(module->moduleLabel()) and highlighted(previous->moduleLabel()))
0396             edgeAttributes["color"] = "darkgreen";
0397         }
0398       }
0399       previous = module;
0400     }
0401   }
0402 }
0403
0404 void DependencyGraph::postBeginJob() {
0405   if (not m_initialized)
0406     return;
0407
0408   // remove the nodes corresponding to the modules that have been removed from the process
0409   for (int i = boost::num_vertices(m_graph) - 1; i > 1; --i) {
0410     if (m_graph.m_graph[i].label.empty())
0411       boost::remove_vertex(i, m_graph.m_graph);
0412   }
0413
0414   // draw the dependency graph
0415   std::ofstream out(m_filename);
0416   boost::write_graphviz(out, m_graph);
0417   out.close();
0418 }
0419
0420 namespace edm {
0421   namespace service {
0422
0423     inline bool isProcessWideService(DependencyGraph const *) { return true; }
0424
0425   }  // namespace service
0426 }  // namespace edm
0427
0428 // define as a framework servie
0429 #include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
0430 DEFINE_FWK_SERVICE(DependencyGraph);