Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2025-01-09 23:34:06

0001 #! /usr/bin/env python3
0002 
0003 import optparse
0004 import os
0005 import re
0006 import sys
0007 import pprint
0008 import subprocess
0009 from XML2Python import xml2obj
0010 from subprocess import getoutput
0011 # These aren't all typedefs, but can sometimes make the output more
0012 # readable
0013 typedefsDict = \
0014              {
0015     # What we want <=  What we have
0016     'unsigned int' : ['unsignedint', 'UInt32_t', 'uint32_t'],
0017     'unsigned long': ['unsignedlong'],
0018     'int'          : ['Int32_t'],
0019     'float'        : ['Float_t'],
0020     'double'       : ['Double_t'],
0021     'char'         : ['Char_t'],
0022     '< '           : ['<', '&lt;'],
0023     ' >'           : ['>', '&gt;'],
0024     ', '           : [','],
0025     }
0026 
0027 
0028 # Equivalent names for packages - lets script know that, for example,
0029 # 'TrackReco' package should have objects 'reco::Track'.
0030 #Ordered List to search for matched packages
0031 equivDict = \
0032      [
0033          {'SelectorUtils': ['VersionedSelector']},
0034          {'Associations': ['TTTrackTruthPair', 'edm::Wrapper.+edm::AssociationMap.+TrackingParticle', 'MtdSimLayerCluster.+TrackingParticle', 'TrackingParticle.+MtdSimLayerCluster',
0035                            '(TTClusterAssociationMap|TTStubAssociationMap|TTTrackAssociationMap|TrackingParticle).*Phase2TrackerDigi',
0036                            '(TTStub|TTCluster|TTTrack).*Phase2TrackerDigi.*TrackingParticle']},
0037          {'TrajectoryState'         : ['TrajectoryStateOnSurface']},
0038          {'L1TrackTrigger'        : ['(TTStub|TTCluster|TTTrack).*Phase2TrackerDigi']},
0039          {'L1TCalorimeterPhase2'  : ['l1tp2::CaloTower.*']},
0040          {'L1TCalorimeter'        : ['l1t::CaloTower.*']},
0041          {'VertexFinder'          : ['l1tVertexFinder::Vertex']},
0042          {'GsfTracking'           : ['reco::GsfTrack(Collection|).*(MomentumConstraint|VertexConstraint)', 'Trajectory.*reco::GsfTrack']},
0043          {'PatCandidates'         : ['pat::PATObject','pat::Lepton', 'reco::RecoCandidate','pat::[A-Za-z]+Ref(Vector|)', 'pat::UserHolder']},
0044          {'BTauReco'              : ['reco::.*SoftLeptonTagInfo', 'reco::SoftLeptonProperties','reco::SecondaryVertexTagInfo','reco::IPTagInfo','reco::TemplatedSecondaryVertexTagInfo', 'reco::CATopJetProperties','reco::HTTTopJetProperties']},
0045          {'CastorReco'            : ['reco::CastorJet']},
0046          {'JetMatching'           : ['reco::JetFlavourInfo', 'reco::JetFlavour','reco::MatchedPartons']},
0047          {'RecoCandidate'         : ['reco::Candidate']},
0048          {'TrackingAnalysis'      : ['TrackingParticle']},
0049          {'Egamma'                : ['reco::ElectronID']},
0050          {'TopObjects'            : ['reco::CATopJetProperties']},
0051          {'TauReco'               : ['reco::L2TauIsolationInfo','reco::RecoTauPiZero','reco::BaseTau']},
0052          {'ValidationFormats'     : ['PGlobalDigi::.+','PGlobalRecHit::.+']},
0053          {'TrajectorySeed'        : ['TrajectorySeed']},
0054          {'TrackCandidate'        : ['TrackCandidate']},
0055          {'PatternTools'          : ['MomentumConstraint','VertexConstraint','Trajectory']},
0056          {'TrackerRecHit2D'       : ['SiStrip(Matched|)RecHit[12]D','SiTrackerGSRecHit[12]D','SiPixelRecHit']},
0057          {'MuonReco'              : ['reco::Muon(Ref|)(Vector|)']},
0058          {'MuonSeed'              : ['L3MuonTrajectorySeed']},
0059          {'HepMCCandidate'        : ['reco::GenParticle.*']},
0060          {'L1Trigger'             : ['l1extra::L1.+Particle', 'l1t::Vertex']},
0061          {'TrackInfo'             : ['reco::TrackingRecHitInfo']},
0062          {'EgammaCandidates'      : ['reco::GsfElectron.*','reco::Photon.*']},
0063          {'HcalIsolatedTrack'     : ['reco::IsolatedPixelTrackCandidate', 'reco::EcalIsolatedParticleCandidate', 'reco::HcalIsolatedTrackCandidate']},
0064          {'HcalRecHit'            : ['HFRecHit','HORecHit','ZDCRecHit','HBHERecHit','HcalRecHitSoA']},
0065          {'PFRootEvent'           : ['EventColin::']},
0066          {'CaloTowers'            : ['CaloTower.*']},
0067          {'GsfTrackReco'          : ['GsfTrack.*']},
0068          {'METReco'               : ['reco::(Calo|PF|Gen|)MET','reco::PFClusterMET']},
0069          {'ParticleFlowReco'      : ['reco::RecoPFClusterRefCandidateRef.*']},
0070          {'ParticleFlowCandidate' : ['reco::PFCandidateRef','reco::PFCandidateFwdRef','reco::PFCandidate']},
0071          {'PhysicsToolsObjects'   : ['PhysicsTools::Calibration']},
0072          {'TrackReco'             : ['reco::Track','reco::TrackRef']},
0073          {'VertexReco'            : ['reco::Vertex']},
0074          {'TauReco'               : ['reco::PFJetRef']},
0075          {'JetReco'               : ['reco::.*Jet','reco::.*Jet(Collection|Ref)']},
0076          {'HGCDigi'               : ['HGCSample']},
0077          {'HGCRecHit'             : ['constHGCRecHit','HGCRecHit']},
0078          {'SiPixelObjects'        : ['SiPixelQuality.*']},
0079          {'EcalRecHit'            : ['EcalRecHitSoA.*']},
0080      ]
0081 
0082 ignoreEdmDP = {
0083   'LCGReflex/__gnu_cxx::__normal_iterator<std::basic_string<char>*,std::vector<std::basic_string<char>%>%>' : 1,
0084   '' : 1
0085 }
0086 
0087 def searchClassDefXml ():
0088     """ Searches through the requested directory looking at
0089     'classes_def.xml' files looking for duplicate Reflex definitions."""
0090     # compile necessary RE statements
0091     classNameRE    = re.compile (r'class\s+name\s*=\s*"([^"]*)"')
0092     spacesRE       = re.compile (r'\s+')
0093     stdRE          = re.compile (r'std::')
0094     srcClassNameRE = re.compile (r'(\w+)/src/classes_def.*[.]xml')
0095     ignoreSrcRE    = re.compile (r'.*/FWCore/Skeletons/scripts/mkTemplates/.+')
0096     braketRE       = re.compile (r'<.+>')
0097     print("Searching for 'classes_def.xml' in '%s'." % os.path.join(os.environ.get('CMSSW_BASE'),'src'))
0098     xmlFiles = []
0099     for srcDir in [os.environ.get('CMSSW_BASE'),os.environ.get('CMSSW_RELEASE_BASE')]:
0100         if not len(srcDir): continue
0101         for xml in getoutput ('cd '+os.path.join(srcDir,'src')+'; find . -name "*classes_def*.xml" -follow -print').split ('\n'):
0102             if xml and (not xml in xmlFiles):
0103                 xmlFiles.append(xml)
0104     if options.showXMLs:
0105         pprint.pprint (xmlFiles)
0106     # try and figure out the names of the packages
0107     xmlPackages = []
0108     packagesREs = {}
0109     equivREs    = {}
0110     explicitREs = []
0111     for item in equivDict:
0112         for pack in item:
0113             for equiv in item[pack]:
0114                 explicitREs.append( (re.compile(r'\b' + equiv + r'\b'),pack))
0115     if options.lostDefs:
0116         for filename in xmlFiles:
0117             if (not filename) or (ignoreSrcRE.match(filename)): continue
0118             match = srcClassNameRE.search (filename)
0119             if not match: continue
0120             packageName = match.group(1)
0121             xmlPackages.append (packageName)
0122             matchString = r'\b' + packageName + r'\b'
0123             packagesREs[packageName] = re.compile (matchString)
0124             equivList = equivREs.setdefault (packageName, [])
0125             for item in equivDict:
0126                 for equiv in item.get (packageName, []):
0127                     matchString = re.compile(r'\b' + equiv + r'\b')
0128                     equivList.append( (matchString, equiv) )
0129             equivList.append( (packagesREs[packageName], packageName) )
0130     classDict = {}
0131     ncdict = {'class' : 'className', 'function' : 'functionName'}
0132     for filename in xmlFiles:
0133         if (not filename) or (ignoreSrcRE.match(filename)): continue
0134         lostProblems    = ''
0135         exceptName      = ''
0136         regexList       = []
0137         localObjects    = []
0138         simpleObjectREs = []
0139         if options.lostDefs:
0140             lostMatch = srcClassNameRE.search (filename)
0141             if lostMatch:
0142                 exceptName = lostMatch.group (1)
0143                 regexList = equivREs[exceptName]
0144                 xcount = len(regexList)-1
0145                 if not regexList[xcount][0].search (exceptName):
0146                     print('%s not found in' % exceptName, end=' ')
0147                     print(regexList[xcount][0])
0148                     sys.exit()
0149             else: continue
0150         if options.verbose:
0151             print("filename", filename)
0152         try:
0153             filepath = os.path.join(os.environ.get('CMSSW_BASE'),'src',filename)
0154             if not os.path.exists(filepath):
0155                 filepath = os.path.join(os.environ.get('CMSSW_RELEASE_BASE'),'src',filename)
0156             xmlObj = xml2obj (filename = filepath,
0157                               filtering = True,
0158                               nameChangeDict = ncdict)
0159         except Exception as detail:
0160             print("File %s is malformed XML.  Please fix." % filename)
0161             print("  ", detail)
0162             continue
0163         try:
0164             classList = xmlObj.selection.className
0165         except:
0166             try:
0167                 classList = xmlObj.className
0168             except:
0169                 # this isn't a real classes_def.xml file.  Skip it
0170                 print("**** SKIPPING '%s' - Doesn't seem to have proper information." % filename)
0171                 continue
0172         if not classList:
0173             classList = xmlObj.functionName
0174             if not classList:
0175                 print("**** SKIPPING '%s' - Dosen't seem to have proper information(not class/function)." % filename)
0176                 continue
0177         for piece in classList:
0178             try:
0179                 className = spacesRE.sub ('', piece.name)
0180             except:
0181                 # must be one of these class pattern things.  Skip it
0182                 #print "     skipping %s" % filename, piece.__repr__()
0183                 continue
0184             className = stdRE.sub    ('', className)
0185             # print "  ", className
0186             # Now get rid of any typedefs
0187             for typedef, tdList in typedefsDict.items():
0188                 for alias in tdList:
0189                     className = re.sub (alias, typedef, className)
0190             classDict.setdefault (className, set()).add (filename)
0191             # should we check for lost definitions?
0192             if not options.lostDefs:
0193                 continue
0194             localObjects.append (className)
0195             if options.lazyLostDefs and not braketRE.search (className):
0196                 #print "  ", className
0197                 matchString = r'\b' + className + r'\b'
0198                 simpleObjectREs.append( (re.compile (matchString), className ) )
0199         for className in localObjects:
0200             # if we see our name (or equivalent) here, then let's
0201             # skip complaining about this
0202             foundEquiv = False
0203             for equivRE in regexList:
0204                 #print("searching %s for %s" % (equivRE[1], className))
0205                 if equivRE[0].search (className):
0206                     foundEquiv = True
0207                     break
0208             for simpleRE in simpleObjectREs:
0209                 if simpleRE[0].search (className):
0210                     foundEquiv = True
0211                     if options.verbose and simpleRE[1] != className:
0212                         print("    Using %s to ignore %s" \
0213                               % (simpleRE[1], className))                    
0214                     break
0215             if foundEquiv: continue
0216             for exRes in explicitREs:
0217                 if exRes[0].search(className):
0218                     lostProblems += "  %s : %s\n" % (exRes[1], className)
0219                     foundEquiv = True
0220                     break
0221             if foundEquiv: continue
0222             for packageName in xmlPackages:
0223                 # don't bother looking for the name of this
0224                 # package in this package
0225                 if packagesREs[packageName].search (className):
0226                     lostProblems += "  %s : %s\n" % (packageName, className)
0227                     break
0228         # for piece
0229         if lostProblems:
0230             print(f'\n{filename} defines the following dictionaries that should be defined in another package\n{lostProblems}\n')
0231     # for filename
0232     if options.dups:
0233         for name, fileSet in sorted( classDict.items() ):
0234             if len (fileSet) < 2:
0235                 continue
0236             print(f"{name} is defined in more than one package")
0237             fileList = sorted (fileSet)
0238             for filename in fileList:
0239                 print("  ", filename)
0240             print()
0241         # for name, fileSet
0242     # if not noDups
0243     #pprint.pprint (classDict)
0244 
0245 
0246 def searchDuplicatePlugins ():
0247     """ Searches the edmpluginFile to find any duplicate
0248     plugins."""
0249     edmpluginFile = ''
0250     libenv = 'LD_LIBRARY_PATH'
0251     if os.environ.get('SCRAM_ARCH').startswith('osx'): libenv = 'DYLD_FALLBACK_LIBRARY_PATH'
0252     biglib = '/biglib/'+os.environ.get('SCRAM_ARCH')
0253     for libdir in os.environ.get(libenv).split(':'):
0254         if libdir.endswith(biglib): continue
0255         if os.path.exists(libdir+'/.edmplugincache'): edmpluginFile = edmpluginFile + ' ' + libdir+'/.edmplugincache'
0256     if edmpluginFile == '': edmpluginFile = os.path.join(os.environ.get('CMSSW_BASE'),'lib',os.environ.get('SCRAM_ARCH'),'.edmplugincache')
0257     cmd = "cat %s | awk '{print $2\"?\"$3\" \"$1}' | sort | uniq | awk '{print $1}' | sort | uniq -c | grep '2 ' | tr \"?\" \" \" | awk '{print $2}'" % edmpluginFile
0258     output = getoutput (cmd).split('\n')
0259     for line in output:
0260         if line in ignoreEdmDP: continue
0261         line = line.replace("*","\*")
0262         cmd = "cat %s | grep ' %s ' | awk '{print $1}' | sort | uniq " % (edmpluginFile,line)
0263         out1 = getoutput (cmd).split('\n')
0264         print(line)
0265         for plugin in out1:
0266             if plugin:
0267                 print("   **"+plugin+"**")
0268         print()
0269 
0270 if __name__ == "__main__":
0271     # setup options parser
0272     parser = optparse.OptionParser ("Usage: %prog [options]\n"\
0273                                     "Searches classes_def.xml for wrong/duplicate "\
0274                                     "definitions")
0275     xmlGroup  = optparse.OptionGroup (parser, "ClassDef XML options")
0276     dumpGroup = optparse.OptionGroup (parser, "EdmPluginDump options")
0277     xmlGroup.add_option ('--dups', dest='dups', action='store_true',
0278                          default=False,
0279                          help="Search for duplicate definitions")
0280     xmlGroup.add_option ('--lostDefs', dest='lostDefs', action='store_true',
0281                          default=False,
0282                          help="Looks for definitions in the wrong libraries")
0283     xmlGroup.add_option ('--lazyLostDefs', dest='lazyLostDefs',
0284                          action='store_true',
0285                          default=False,
0286                          help="Will try to ignore as many lost defs as reasonable")
0287     xmlGroup.add_option ('--verbose', dest='verbose',
0288                          action='store_true',
0289                          default=False,
0290                          help="Prints out a lot of information")
0291     xmlGroup.add_option ('--showXMLs', dest='showXMLs', action='store_true',
0292                          default=False,
0293                          help="Shows all 'classes_def.xml' files")
0294     xmlGroup.add_option ('--dir', dest='srcdir', type='string', default='',
0295                          help="Obsolete")
0296     dumpGroup.add_option ('--edmPD', dest='edmPD', action='store_true',
0297                           default=False,
0298                           help="Searches EDM Plugin Dump for duplicates")
0299     dumpGroup.add_option ('--edmFile', dest='edmFile', type='string',
0300                           default='',
0301                           help="Obsolete")
0302     parser.add_option_group (xmlGroup)
0303     parser.add_option_group (dumpGroup)
0304     (options, args) = parser.parse_args()
0305 
0306     # Let's go:
0307     if options.lazyLostDefs:
0308         options.lostDefs = True
0309     if options.showXMLs or options.lostDefs or options.dups:
0310         searchClassDefXml ()
0311     if options.edmPD:
0312         searchDuplicatePlugins ()