StaticAnalyzers/scripts/callgraph.py

0001 #! /usr/bin/env python3
0002 from __future__ import print_function
0003 import networkx as nx
0004 import re
0005 import json
0006 from yaml import load, dump, FullLoader
0007 try:
0008     from yaml import CLoader as Loader, CDumper as Dumper
0009 except ImportError:
0010     from yaml import Loader, Dumper
0011
0012 topfunc = r"::(dqmBeginRun|dqmEndRun|bookHistograms|accumulate|acquire|startingNewLoop|duringLoop|endOfLoop|beginOfJob|endOfJob|produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream|streamBeginRun|streamBeginLuminosityBlock|streamEndRun|streamEndLuminosityBlock|globalBeginRun|globalEndRun|globalBeginLuminosityBlock|globalEndLuminosityBlock|endRun|endRunProduce|endLuminosityBlock)\("
0013 topfuncre = re.compile(topfunc)
0014
0015 baseclass = r"\b(edm::)?(one::|stream::|global::)?((DQM)?(Global|One)?ED(Producer|Filter|Analyzer|(IterateNTimes|NavigateEvents)?Looper)(Base)?|impl::(ExternalWork|Accumulator|RunWatcher|RunCacheHolder)|FromFiles|ProducerSourceBase|OutputModuleBase|InputSource|ProducerSourceFromFiles|ProducerBase|PuttableSourceBase|OutputModule|RawOutput|RawInputSource|impl::RunWatcher<edm::one::EDProducerBase>|impl::EndRunProducer<edm::one::EDProducerBase>|DQMEDHarvester|AlignmentProducerBase|BMixingModule|TrackProducerBase|cms::CkfTrackCandidateMakerBase|CallEndRunProduceImpl|CallGlobalImpl|impl::makeGlobal|impl::makeStreamModule|impl::EndRunProducer)\b"
0016 baseclassre = re.compile(baseclass)
0017 assert(baseclassre.match('edm::one::impl::EndRunProducer'))
0018 assert(baseclassre.match('DQMOneEDAnalyzer'))
0019 assert(baseclassre.match('edm::EDAnalyzer'))
0020 assert(baseclassre.match('edm::stream::EDProducerBase'))
0021 assert(baseclassre.match('DQMEDHarvester'))
0022 assert(baseclassre.match('edm::one::impl::RunWatcher<edm::one::EDProducerBase>'))
0023 assert(baseclassre.match('edm::global::EDFilter::filter() virtual'))
0024 assert(baseclassre.search('edm::stream::EDProducerBase::produce'))
0025 assert(topfuncre.search('edm::global::EDFilterBase::filter(&) const virtual'))
0026 assert(not baseclassre.match('edm::BaseFlatGunProducer'))
0027 assert(not baseclassre.match('edm::FlatRandomEGunProducer'))
0028 assert(baseclassre.match('edm::ProducerSourceBase'))
0029 assert(baseclassre.match('edm::one::OutputModuleBase'))
0030 farg = re.compile(r"\(.*?\)")
0031 tmpl = re.compile(r'<(.*)>')
0032 toplevelfuncs = set()
0033
0034 getfuncre = re.compile(r"edm::eventsetup::EventSetupRecord::get<")
0035 epfuncre = re.compile(r"edm::eventsetup::EventSetupRecord::deprecated_get<")
0036 f = 'edm::eventsetup::EventSetupRecord::get<class edm::ESHandle<class CaloTopology>>(const std::string &, class edm::ESHandle<class CaloTopology> &) const'
0037 g = 'edm::eventsetup::EventSetupRecord::deprecated_get<class edm::ESHandle<class HcalDDDSimConstants>>(const char *, class edm::ESHandle<class HcalDDDSimConstants> &) const'
0038 g2 = 'edm::eventsetup::EventSetupRecord::deprecated_get<class edm::ESHandle<class HcalDDDSimConstants>>(class edm::ESHandle<class HcalDDDSimConstants> &) const'
0039 h = 'edm::eventsetup::EventSetupRecord::get<class edm::ESHandle<class CaloTPGTranscoder>>(class edm::ESHandle<class CaloTPGTranscoder> &) const'
0040 assert(getfuncre.search(f))
0041 assert(epfuncre.search(g))
0042 assert(getfuncre.search(h))
0043 assert(epfuncre.search(g2))
0044 epf='edm::eventsetup::EventSetupRecord::get<class edm::ESHandle<class CaloTPGTranscoder>>(edm::ESHandle<CaloTPGTranscoder> &) const'
0045 assert(getfuncre.search(epf))
0046
0047 skipfunc = re.compile(r"TGraph::IsA\(.*\)")
0048 epfuncs = set()
0049
0050 function='(anonymous namespace in /src/RecoTracker/TkHitPairs/plugins/HitPairEDProducer.cc)::Impl<(anonymous namespace)::ImplSeedingHitSets, (anonymous namespace)::DoNothing, (anonymous namespace)::RegionsLayersSeparate>::produce(const _Bool, edm::Event &, const edm::EventSetup &);'
0051 function2='HitPairEDProducer::produce(edm::Event &, const edm::EventSetup &);'
0052 value='HitPairEDProducer'
0053 valuere=r'\b%s\b' % value
0054 vre=re.compile(valuere)
0055 m=vre.search(function)
0056 assert(m)
0057 m2=vre.search(function2)
0058 assert(m2)
0059
0060
0061 module2package = dict()
0062 with open('module_to_package.yaml') as m:
0063     module2package=load(m, Loader=FullLoader)
0064
0065 with open('packages.json', 'r') as j:
0066     p = json.load(j)
0067
0068 for k,v in p.items():
0069     module=k.split('|')[0]
0070     dirs=v.split('|')
0071     package=dirs[0]+'/'+dirs[1]
0072     module2package.setdefault(package, []).append(module)
0073
0074 for k in module2package.keys():
0075     module2package[k]=sorted(set(module2package[k]))
0076
0077 G = nx.DiGraph()
0078 with open('function-calls-db.txt') as f:
0079     for line in f:
0080         fields = line.split("'")
0081         if len(fields) < 3:
0082             continue
0083         if fields[2] == ' calls function ':
0084             G.add_edge(fields[1], fields[3], kind=fields[2])
0085             if epfuncre.search(fields[3]) or getfuncre.search(fields[3]):
0086                 epfuncs.add(fields[3])
0087         if fields[2] == ' overrides function ':
0088             if baseclassre.match(fields[3]):
0089                 G.add_edge(fields[1], fields[3], kind=' overrides function ')
0090                 if topfuncre.search(fields[1]) :
0091                     toplevelfuncs.add(fields[1])
0092             else:
0093                 G.add_edge(fields[3], fields[1], kind=' overriden function calls function ')
0094             if epfuncre.search(fields[3]) or getfuncre.search(fields[3]):
0095                 epfuncs.add(fields[3])
0096
0097 callstacks = set()
0098 for tfunc in toplevelfuncs:
0099     for epfunc in epfuncs:
0100         if G.has_node(tfunc) and G.has_node(epfunc) and nx.has_path(G, tfunc, epfunc):
0101             for path in nx.all_shortest_paths(G, tfunc, epfunc):
0102                 cs = str("")
0103                 previous = str("")
0104                 for p in path:
0105                     if getfuncre.search(p) or epfuncre.search(p):
0106                         break
0107                     #stripped=re.sub(farg, "()", p)
0108                     stripped=p
0109                     if previous != stripped:
0110                         cs += stripped + '; '
0111                         previous = stripped
0112                 if cs not in callstacks:
0113                     callstacks.add(cs)
0114
0115 report = dict()
0116 csset = set(callstacks)
0117 for callstack in sorted(list(callstacks)):
0118     for key in module2package.keys():
0119         for value in module2package.get(key):
0120             regex_str = r'\b%s\b'%value
0121             vre=re.compile(regex_str)
0122             if vre.search(callstack):
0123                 report.setdefault(str(key), {}).setdefault(str(value), []).append(str(callstack))
0124                 csset.discard(callstack)
0125
0126 report['no-package']=dict()
0127 for cs in sorted(list(csset)):
0128     key=tmpl.split(topfuncre.split(cs)[0])[0]
0129     report['no-package'].setdefault(key,[]).append(str(cs))
0130
0131 r = open('eventsetuprecord-get.yaml', 'w')
0132 dump(report, r, width=float("inf"))