ReleaseScripts/scripts/duplicateReflexLibrarySearch.py

0001 #! /usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import optparse
0005 import os
0006 import re
0007 import sys
0008 import pprint
0009 import subprocess
0010 from XML2Python import xml2obj
0011 from subprocess import getoutput
0012 # These aren't all typedefs, but can sometimes make the output more
0013 # readable
0014 typedefsDict = \
0015              {
0016     # What we want <=  What we have
0017     'unsigned int' : ['unsignedint', 'UInt32_t', 'uint32_t'],
0018     'unsigned long': ['unsignedlong'],
0019     'int'          : ['Int32_t'],
0020     'float'        : ['Float_t'],
0021     'double'       : ['Double_t'],
0022     'char'         : ['Char_t'],
0023     '< '           : ['<', '&lt;'],
0024     ' >'           : ['>', '&gt;'],
0025     ', '           : [','],
0026     }
0027
0028
0029 # Equivalent names for packages - lets script know that, for example,
0030 # 'TrackReco' package should have objects 'reco::Track'.
0031 #Ordered List to search for matched packages
0032 equivDict = \
0033      [
0034          {'SelectorUtils': ['VersionedSelector']},
0035          {'Associations': ['TTTrackTruthPair', 'edm::Wrapper.+edm::AssociationMap.+TrackingParticle', 'MtdSimLayerCluster.+TrackingParticle', 'TrackingParticle.+MtdSimLayerCluster']},
0036          {'TrajectoryState'         : ['TrajectoryStateOnSurface']},
0037          {'TrackTriggerAssociation' : ['(TTClusterAssociationMap|TTStubAssociationMap|TTTrackAssociationMap|TrackingParticle).*Phase2TrackerDigi',
0038                                        '(TTStub|TTCluster|TTTrack).*Phase2TrackerDigi.*TrackingParticle']},
0039          {'L1TrackTrigger'        : ['(TTStub|TTCluster|TTTrack).*Phase2TrackerDigi']},
0040          {'L1TCalorimeterPhase2'  : ['l1tp2::CaloTower.*']},
0041          {'L1TCalorimeter'        : ['l1t::CaloTower.*']},
0042          {'VertexFinder'          : ['l1tVertexFinder::Vertex']},
0043          {'GsfTracking'           : ['reco::GsfTrack(Collection|).*(MomentumConstraint|VertexConstraint)', 'Trajectory.*reco::GsfTrack']},
0044          {'PatCandidates'         : ['pat::PATObject','pat::Lepton', 'reco::RecoCandidate','pat::[A-Za-z]+Ref(Vector|)', 'pat::UserHolder']},
0045          {'BTauReco'              : ['reco::.*SoftLeptonTagInfo', 'reco::SoftLeptonProperties','reco::SecondaryVertexTagInfo','reco::IPTagInfo','reco::TemplatedSecondaryVertexTagInfo', 'reco::CATopJetProperties','reco::HTTTopJetProperties']},
0046          {'CastorReco'            : ['reco::CastorJet']},
0047          {'JetMatching'           : ['reco::JetFlavourInfo', 'reco::JetFlavour','reco::MatchedPartons']},
0048          {'RecoCandidate'         : ['reco::Candidate']},
0049          {'TrackingAnalysis'      : ['TrackingParticle']},
0050          {'Egamma'                : ['reco::ElectronID']},
0051          {'TopObjects'            : ['reco::CATopJetProperties']},
0052          {'TauReco'               : ['reco::L2TauIsolationInfo','reco::RecoTauPiZero','reco::BaseTau']},
0053          {'ValidationFormats'     : ['PGlobalDigi::.+','PGlobalRecHit::.+']},
0054          {'TrajectorySeed'        : ['TrajectorySeed']},
0055          {'TrackCandidate'        : ['TrackCandidate']},
0056          {'PatternTools'          : ['MomentumConstraint','VertexConstraint','Trajectory']},
0057          {'TrackerRecHit2D'       : ['SiStrip(Matched|)RecHit[12]D','SiTrackerGSRecHit[12]D','SiPixelRecHit']},
0058          {'MuonReco'              : ['reco::Muon(Ref|)(Vector|)']},
0059          {'MuonSeed'              : ['L3MuonTrajectorySeed']},
0060          {'HepMCCandidate'        : ['reco::GenParticle.*']},
0061          {'L1Trigger'             : ['l1extra::L1.+Particle', 'l1t::Vertex']},
0062          {'TrackInfo'             : ['reco::TrackingRecHitInfo']},
0063          {'EgammaCandidates'      : ['reco::GsfElectron.*','reco::Photon.*']},
0064          {'HcalIsolatedTrack'     : ['reco::IsolatedPixelTrackCandidate', 'reco::EcalIsolatedParticleCandidate', 'reco::HcalIsolatedTrackCandidate']},
0065          {'HcalRecHit'            : ['HFRecHit','HORecHit','ZDCRecHit','HBHERecHit','HcalRecHitSoA']},
0066          {'PFRootEvent'           : ['EventColin::']},
0067          {'CaloTowers'            : ['CaloTower.*']},
0068          {'GsfTrackReco'          : ['GsfTrack.*']},
0069          {'METReco'               : ['reco::(Calo|PF|Gen|)MET','reco::PFClusterMET']},
0070          {'ParticleFlowReco'      : ['reco::RecoPFClusterRefCandidateRef.*']},
0071          {'ParticleFlowCandidate' : ['reco::PFCandidateRef','reco::PFCandidateFwdRef','reco::PFCandidate']},
0072          {'PhysicsToolsObjects'   : ['PhysicsTools::Calibration']},
0073          {'TrackReco'             : ['reco::Track','reco::TrackRef']},
0074          {'VertexReco'            : ['reco::Vertex']},
0075          {'TFWLiteSelectorTest'   : ['tfwliteselectortest']},
0076          {'TauReco'               : ['reco::PFJetRef']},
0077          {'JetReco'               : ['reco::.*Jet','reco::.*Jet(Collection|Ref)']},
0078          {'HGCDigi'               : ['HGCSample']},
0079          {'HGCRecHit'             : ['constHGCRecHit','HGCRecHit']},
0080          {'SiPixelObjects'        : ['SiPixelQuality.*']},
0081      ]
0082
0083 ignoreEdmDP = {
0084   'LCGReflex/__gnu_cxx::__normal_iterator<std::basic_string<char>*,std::vector<std::basic_string<char>%>%>' : 1,
0085   '' : 1
0086 }
0087
0088 def searchClassDefXml ():
0089     """ Searches through the requested directory looking at
0090     'classes_def.xml' files looking for duplicate Reflex definitions."""
0091     # compile necessary RE statements
0092     classNameRE    = re.compile (r'class\s+name\s*=\s*"([^"]*)"')
0093     spacesRE       = re.compile (r'\s+')
0094     stdRE          = re.compile (r'std::')
0095     srcClassNameRE = re.compile (r'(\w+)/src/classes_def.*[.]xml')
0096     ignoreSrcRE    = re.compile (r'.*/FWCore/Skeletons/scripts/mkTemplates/.+')
0097     braketRE       = re.compile (r'<.+>')
0098     print("Searching for 'classes_def.xml' in '%s'." % os.path.join(os.environ.get('CMSSW_BASE'),'src'))
0099     xmlFiles = []
0100     for srcDir in [os.environ.get('CMSSW_BASE'),os.environ.get('CMSSW_RELEASE_BASE')]:
0101         if not len(srcDir): continue
0102         for xml in getoutput ('cd '+os.path.join(srcDir,'src')+'; find . -name "*classes_def*.xml" -follow -print').split ('\n'):
0103             if xml and (not xml in xmlFiles):
0104                 xmlFiles.append(xml)
0105     if options.showXMLs:
0106         pprint.pprint (xmlFiles)
0107     # try and figure out the names of the packages
0108     xmlPackages = []
0109     packagesREs = {}
0110     equivREs    = {}
0111     explicitREs = []
0112     for item in equivDict:
0113         for pack in item:
0114             for equiv in item[pack]:
0115                 explicitREs.append( (re.compile(r'\b' + equiv + r'\b'),pack))
0116     if options.lostDefs:
0117         for filename in xmlFiles:
0118             if (not filename) or (ignoreSrcRE.match(filename)): continue
0119             match = srcClassNameRE.search (filename)
0120             if not match: continue
0121             packageName = match.group(1)
0122             xmlPackages.append (packageName)
0123             matchString = r'\b' + packageName + r'\b'
0124             packagesREs[packageName] = re.compile (matchString)
0125             equivList = equivREs.setdefault (packageName, [])
0126             for item in equivDict:
0127                 for equiv in item.get (packageName, []):
0128                     matchString = re.compile(r'\b' + equiv + r'\b')
0129                     equivList.append( (matchString, equiv) )
0130             equivList.append( (packagesREs[packageName], packageName) )
0131     classDict = {}
0132     ncdict = {'class' : 'className', 'function' : 'functionName'}
0133     for filename in xmlFiles:
0134         if (not filename) or (ignoreSrcRE.match(filename)): continue
0135         dupProblems     = ''
0136         exceptName      = ''
0137         regexList       = []
0138         localObjects    = []
0139         simpleObjectREs = []
0140         if options.lostDefs:
0141             lostMatch = srcClassNameRE.search (filename)
0142             if lostMatch:
0143                 exceptName = lostMatch.group (1)
0144                 regexList = equivREs[exceptName]
0145                 xcount = len(regexList)-1
0146                 if not regexList[xcount][0].search (exceptName):
0147                     print('%s not found in' % exceptName, end=' ')
0148                     print(regexList[xcount][0])
0149                     sys.exit()
0150             else: continue
0151         if options.verbose:
0152             print("filename", filename)
0153         try:
0154             filepath = os.path.join(os.environ.get('CMSSW_BASE'),'src',filename)
0155             if not os.path.exists(filepath):
0156                 filepath = os.path.join(os.environ.get('CMSSW_RELEASE_BASE'),'src',filename)
0157             xmlObj = xml2obj (filename = filepath,
0158                               filtering = True,
0159                               nameChangeDict = ncdict)
0160         except Exception as detail:
0161             print("File %s is malformed XML.  Please fix." % filename)
0162             print("  ", detail)
0163             continue
0164         try:
0165             classList = xmlObj.selection.className
0166         except:
0167             try:
0168                 classList = xmlObj.className
0169             except:
0170                 # this isn't a real classes_def.xml file.  Skip it
0171                 print("**** SKIPPING '%s' - Doesn't seem to have proper information." % filename)
0172                 continue
0173         if not classList:
0174             classList = xmlObj.functionName
0175             if not classList:
0176                 print("**** SKIPPING '%s' - Dosen't seem to have proper information(not class/function)." % filename)
0177                 continue
0178         for piece in classList:
0179             try:
0180                 className = spacesRE.sub ('', piece.name)
0181             except:
0182                 # must be one of these class pattern things.  Skip it
0183                 #print "     skipping %s" % filename, piece.__repr__()
0184                 continue
0185             className = stdRE.sub    ('', className)
0186             # print "  ", className
0187             # Now get rid of any typedefs
0188             for typedef, tdList in typedefsDict.items():
0189                 for alias in tdList:
0190                     className = re.sub (alias, typedef, className)
0191             classDict.setdefault (className, set()).add (filename)
0192             # should we check for lost definitions?
0193             if not options.lostDefs:
0194                 continue
0195             localObjects.append (className)
0196             if options.lazyLostDefs and not braketRE.search (className):
0197                 #print "  ", className
0198                 matchString = r'\b' + className + r'\b'
0199                 simpleObjectREs.append( (re.compile (matchString), className ) )
0200         for className in localObjects:
0201             # if we see our name (or equivalent) here, then let's
0202             # skip complaining about this
0203             foundEquiv = False
0204             for equivRE in regexList:
0205                 #print("searching %s for %s" % (equivRE[1], className))
0206                 if equivRE[0].search (className):
0207                     foundEquiv = True
0208                     break
0209             for simpleRE in simpleObjectREs:
0210                 if simpleRE[0].search (className):
0211                     foundEquiv = True
0212                     if options.verbose and simpleRE[1] != className:
0213                         print("    Using %s to ignore %s" \
0214                               % (simpleRE[1], className))
0215                     break
0216             if foundEquiv: continue
0217             for exRes in explicitREs:
0218                 if exRes[0].search(className):
0219                     dupProblems += "  %s : %s\n" % (exRes[1], className)
0220                     foundEquiv = True
0221                     break
0222             if foundEquiv: continue
0223             for packageName in xmlPackages:
0224                 # don't bother looking for the name of this
0225                 # package in this package
0226                 if packagesREs[packageName].search (className):
0227                     dupProblems += "  %s : %s\n" % (packageName, className)
0228                     break
0229         # for piece
0230         if dupProblems:
0231             print('\n%s\n%s\n' % (filename, dupProblems))
0232     # for filename
0233     if options.dups:
0234         for name, fileSet in sorted( classDict.items() ):
0235             if len (fileSet) < 2:
0236                 continue
0237             print(name)
0238             fileList = sorted (fileSet)
0239             for filename in fileList:
0240                 print("  ", filename)
0241             print()
0242         # for name, fileSet
0243     # if not noDups
0244     #pprint.pprint (classDict)
0245
0246
0247 def searchDuplicatePlugins ():
0248     """ Searches the edmpluginFile to find any duplicate
0249     plugins."""
0250     edmpluginFile = ''
0251     libenv = 'LD_LIBRARY_PATH'
0252     if os.environ.get('SCRAM_ARCH').startswith('osx'): libenv = 'DYLD_FALLBACK_LIBRARY_PATH'
0253     biglib = '/biglib/'+os.environ.get('SCRAM_ARCH')
0254     for libdir in os.environ.get(libenv).split(':'):
0255         if libdir.endswith(biglib): continue
0256         if os.path.exists(libdir+'/.edmplugincache'): edmpluginFile = edmpluginFile + ' ' + libdir+'/.edmplugincache'
0257     if edmpluginFile == '': edmpluginFile = os.path.join(os.environ.get('CMSSW_BASE'),'lib',os.environ.get('SCRAM_ARCH'),'.edmplugincache')
0258     cmd = "cat %s | awk '{print $2\"?\"$3\" \"$1}' | sort | uniq | awk '{print $1}' | sort | uniq -c | grep '2 ' | tr \"?\" \" \" | awk '{print $2}'" % edmpluginFile
0259     output = getoutput (cmd).split('\n')
0260     for line in output:
0261         if line in ignoreEdmDP: continue
0262         line = line.replace("*","\*")
0263         cmd = "cat %s | grep ' %s ' | awk '{print $1}' | sort | uniq " % (edmpluginFile,line)
0264         out1 = getoutput (cmd).split('\n')
0265         print(line)
0266         for plugin in out1:
0267             if plugin:
0268                 print("   **"+plugin+"**")
0269         print()
0270
0271 if __name__ == "__main__":
0272     # setup options parser
0273     parser = optparse.OptionParser ("Usage: %prog [options]\n"\
0274                                     "Searches classes_def.xml for wrong/duplicate "\
0275                                     "definitions")
0276     xmlGroup  = optparse.OptionGroup (parser, "ClassDef XML options")
0277     dumpGroup = optparse.OptionGroup (parser, "EdmPluginDump options")
0278     xmlGroup.add_option ('--dups', dest='dups', action='store_true',
0279                          default=False,
0280                          help="Search for duplicate definitions")
0281     xmlGroup.add_option ('--lostDefs', dest='lostDefs', action='store_true',
0282                          default=False,
0283                          help="Looks for definitions in the wrong libraries")
0284     xmlGroup.add_option ('--lazyLostDefs', dest='lazyLostDefs',
0285                          action='store_true',
0286                          default=False,
0287                          help="Will try to ignore as many lost defs as reasonable")
0288     xmlGroup.add_option ('--verbose', dest='verbose',
0289                          action='store_true',
0290                          default=False,
0291                          help="Prints out a lot of information")
0292     xmlGroup.add_option ('--showXMLs', dest='showXMLs', action='store_true',
0293                          default=False,
0294                          help="Shows all 'classes_def.xml' files")
0295     xmlGroup.add_option ('--dir', dest='srcdir', type='string', default='',
0296                          help="Obsolete")
0297     dumpGroup.add_option ('--edmPD', dest='edmPD', action='store_true',
0298                           default=False,
0299                           help="Searches EDM Plugin Dump for duplicates")
0300     dumpGroup.add_option ('--edmFile', dest='edmFile', type='string',
0301                           default='',
0302                           help="Obsolete")
0303     parser.add_option_group (xmlGroup)
0304     parser.add_option_group (dumpGroup)
0305     (options, args) = parser.parse_args()
0306
0307     # Let's go:
0308     if options.lazyLostDefs:
0309         options.lostDefs = True
0310     if options.showXMLs or options.lostDefs or options.dups:
0311         searchClassDefXml ()
0312     if options.edmPD:
0313         searchDuplicatePlugins ()