StaticAnalyzers/scripts/callgraph.py

0001 #! /usr/bin/env python3
0002 import networkx as nx
0003 import re
0004 import json
0005 from yaml import load, dump, FullLoader
0006 try:
0007     from yaml import CLoader as Loader, CDumper as Dumper
0008 except ImportError:
0009     from yaml import Loader, Dumper
0010
0011 topfunc = r"::(dqmBeginRun|dqmEndRun|bookHistograms|accumulate|acquire|startingNewLoop|duringLoop|endOfLoop|beginOfJob|endOfJob|produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream|streamBeginRun|streamBeginLuminosityBlock|streamEndRun|streamEndLuminosityBlock|globalBeginRun|globalEndRun|globalBeginLuminosityBlock|globalEndLuminosityBlock|endRun|endRunProduce|endLuminosityBlock)\("
0012 topfuncre = re.compile(topfunc)
0013
0014 baseclass = r"\b(edm::)?(one::|stream::|global::)?((DQM)?(Global|One)?ED(Producer|Filter|Analyzer|(IterateNTimes|NavigateEvents)?Looper)(Base)?|impl::(ExternalWork|Accumulator|RunWatcher|RunCacheHolder)|FromFiles|ProducerSourceBase|OutputModuleBase|InputSource|ProducerSourceFromFiles|ProducerBase|PuttableSourceBase|OutputModule|RawOutput|RawInputSource|impl::RunWatcher<edm::one::EDProducerBase>|impl::EndRunProducer<edm::one::EDProducerBase>|DQMEDHarvester|AlignmentProducerBase|BMixingModule|TrackProducerBase|cms::CkfTrackCandidateMakerBase|CallEndRunProduceImpl|CallGlobalImpl|impl::makeGlobal|impl::makeStreamModule|impl::EndRunProducer)\b"
0015 baseclassre = re.compile(baseclass)
0016 assert(baseclassre.match('edm::one::impl::EndRunProducer'))
0017 assert(baseclassre.match('DQMOneEDAnalyzer'))
0018 assert(baseclassre.match('edm::EDAnalyzer'))
0019 assert(baseclassre.match('edm::stream::EDProducerBase'))
0020 assert(baseclassre.match('DQMEDHarvester'))
0021 assert(baseclassre.match('edm::one::impl::RunWatcher<edm::one::EDProducerBase>'))
0022 assert(baseclassre.match('edm::global::EDFilter::filter() virtual'))
0023 assert(baseclassre.search('edm::stream::EDProducerBase::produce'))
0024 assert(topfuncre.search('edm::global::EDFilterBase::filter(&) const virtual'))
0025 assert(not baseclassre.match('edm::BaseFlatGunProducer'))
0026 assert(not baseclassre.match('edm::FlatRandomEGunProducer'))
0027 assert(baseclassre.match('edm::ProducerSourceBase'))
0028 assert(baseclassre.match('edm::one::OutputModuleBase'))
0029 farg = re.compile(r"\(.*?\)")
0030 tmpl = re.compile(r'<(.*)>')
0031 toplevelfuncs = set()
0032
0033 getfuncre = re.compile(r"edm::eventsetup::EventSetupRecord::get<")
0034 epfuncre = re.compile(r"edm::eventsetup::EventSetupRecord::deprecated_get<")
0035 f = 'edm::eventsetup::EventSetupRecord::get<class edm::ESHandle<class CaloTopology>>(const std::string &, class edm::ESHandle<class CaloTopology> &) const'
0036 g = 'edm::eventsetup::EventSetupRecord::deprecated_get<class edm::ESHandle<class HcalDDDSimConstants>>(const char *, class edm::ESHandle<class HcalDDDSimConstants> &) const'
0037 g2 = 'edm::eventsetup::EventSetupRecord::deprecated_get<class edm::ESHandle<class HcalDDDSimConstants>>(class edm::ESHandle<class HcalDDDSimConstants> &) const'
0038 h = 'edm::eventsetup::EventSetupRecord::get<class edm::ESHandle<class CaloTPGTranscoder>>(class edm::ESHandle<class CaloTPGTranscoder> &) const'
0039 assert(getfuncre.search(f))
0040 assert(epfuncre.search(g))
0041 assert(getfuncre.search(h))
0042 assert(epfuncre.search(g2))
0043 epf='edm::eventsetup::EventSetupRecord::get<class edm::ESHandle<class CaloTPGTranscoder>>(edm::ESHandle<CaloTPGTranscoder> &) const'
0044 assert(getfuncre.search(epf))
0045
0046 skipfunc = re.compile(r"TGraph::IsA\(.*\)")
0047 epfuncs = set()
0048
0049 function='(anonymous namespace in /src/RecoTracker/TkHitPairs/plugins/HitPairEDProducer.cc)::Impl<(anonymous namespace)::ImplSeedingHitSets, (anonymous namespace)::DoNothing, (anonymous namespace)::RegionsLayersSeparate>::produce(const _Bool, edm::Event &, const edm::EventSetup &);'
0050 function2='HitPairEDProducer::produce(edm::Event &, const edm::EventSetup &);'
0051 value='HitPairEDProducer'
0052 valuere=r'\b%s\b' % value
0053 vre=re.compile(valuere)
0054 m=vre.search(function)
0055 assert(m)
0056 m2=vre.search(function2)
0057 assert(m2)
0058
0059
0060 module2package = dict()
0061 with open('module_to_package.yaml') as m:
0062     module2package=load(m, Loader=FullLoader)
0063
0064 with open('packages.json', 'r') as j:
0065     p = json.load(j)
0066
0067 for k,v in p.items():
0068     module=k.split('|')[0]
0069     dirs=v.split('|')
0070     package=dirs[0]+'/'+dirs[1]
0071     module2package.setdefault(package, []).append(module)
0072
0073 for k in module2package.keys():
0074     module2package[k]=sorted(set(module2package[k]))
0075
0076 G = nx.DiGraph()
0077 with open('function-calls-db.txt') as f:
0078     for line in f:
0079         fields = line.split("'")
0080         if len(fields) < 3:
0081             continue
0082         if fields[2] == ' calls function ':
0083             G.add_edge(fields[1], fields[3], kind=fields[2])
0084             if epfuncre.search(fields[3]) or getfuncre.search(fields[3]):
0085                 epfuncs.add(fields[3])
0086         if fields[2] == ' overrides function ':
0087             if baseclassre.match(fields[3]):
0088                 G.add_edge(fields[1], fields[3], kind=' overrides function ')
0089                 if topfuncre.search(fields[1]) :
0090                     toplevelfuncs.add(fields[1])
0091             else:
0092                 G.add_edge(fields[3], fields[1], kind=' overriden function calls function ')
0093             if epfuncre.search(fields[3]) or getfuncre.search(fields[3]):
0094                 epfuncs.add(fields[3])
0095
0096 callstacks = set()
0097 for tfunc in toplevelfuncs:
0098     for epfunc in epfuncs:
0099         if G.has_node(tfunc) and G.has_node(epfunc) and nx.has_path(G, tfunc, epfunc):
0100             for path in nx.all_shortest_paths(G, tfunc, epfunc):
0101                 cs = str("")
0102                 previous = str("")
0103                 for p in path:
0104                     if getfuncre.search(p) or epfuncre.search(p):
0105                         break
0106                     #stripped=re.sub(farg, "()", p)
0107                     stripped=p
0108                     if previous != stripped:
0109                         cs += stripped + '; '
0110                         previous = stripped
0111                 if cs not in callstacks:
0112                     callstacks.add(cs)
0113
0114 report = dict()
0115 csset = set(callstacks)
0116 for callstack in sorted(list(callstacks)):
0117     for key in module2package.keys():
0118         for value in module2package.get(key):
0119             regex_str = r'\b%s\b'%value
0120             vre=re.compile(regex_str)
0121             if vre.search(callstack):
0122                 report.setdefault(str(key), {}).setdefault(str(value), []).append(str(callstack))
0123                 csset.discard(callstack)
0124
0125 report['no-package']=dict()
0126 for cs in sorted(list(csset)):
0127     key=tmpl.split(topfuncre.split(cs)[0])[0]
0128     report['no-package'].setdefault(key,[]).append(str(cs))
0129
0130 r = open('eventsetuprecord-get.yaml', 'w')
0131 dump(report, r, width=float("inf"))