File indexing completed on 2025-01-09 23:34:06
0001
0002
0003 import optparse
0004 import os
0005 import re
0006 import sys
0007 import pprint
0008 import subprocess
0009 from XML2Python import xml2obj
0010 from subprocess import getoutput
0011
0012
0013 typedefsDict = \
0014 {
0015
0016 'unsigned int' : ['unsignedint', 'UInt32_t', 'uint32_t'],
0017 'unsigned long': ['unsignedlong'],
0018 'int' : ['Int32_t'],
0019 'float' : ['Float_t'],
0020 'double' : ['Double_t'],
0021 'char' : ['Char_t'],
0022 '< ' : ['<', '<'],
0023 ' >' : ['>', '>'],
0024 ', ' : [','],
0025 }
0026
0027
0028
0029
0030
0031 equivDict = \
0032 [
0033 {'SelectorUtils': ['VersionedSelector']},
0034 {'Associations': ['TTTrackTruthPair', 'edm::Wrapper.+edm::AssociationMap.+TrackingParticle', 'MtdSimLayerCluster.+TrackingParticle', 'TrackingParticle.+MtdSimLayerCluster',
0035 '(TTClusterAssociationMap|TTStubAssociationMap|TTTrackAssociationMap|TrackingParticle).*Phase2TrackerDigi',
0036 '(TTStub|TTCluster|TTTrack).*Phase2TrackerDigi.*TrackingParticle']},
0037 {'TrajectoryState' : ['TrajectoryStateOnSurface']},
0038 {'L1TrackTrigger' : ['(TTStub|TTCluster|TTTrack).*Phase2TrackerDigi']},
0039 {'L1TCalorimeterPhase2' : ['l1tp2::CaloTower.*']},
0040 {'L1TCalorimeter' : ['l1t::CaloTower.*']},
0041 {'VertexFinder' : ['l1tVertexFinder::Vertex']},
0042 {'GsfTracking' : ['reco::GsfTrack(Collection|).*(MomentumConstraint|VertexConstraint)', 'Trajectory.*reco::GsfTrack']},
0043 {'PatCandidates' : ['pat::PATObject','pat::Lepton', 'reco::RecoCandidate','pat::[A-Za-z]+Ref(Vector|)', 'pat::UserHolder']},
0044 {'BTauReco' : ['reco::.*SoftLeptonTagInfo', 'reco::SoftLeptonProperties','reco::SecondaryVertexTagInfo','reco::IPTagInfo','reco::TemplatedSecondaryVertexTagInfo', 'reco::CATopJetProperties','reco::HTTTopJetProperties']},
0045 {'CastorReco' : ['reco::CastorJet']},
0046 {'JetMatching' : ['reco::JetFlavourInfo', 'reco::JetFlavour','reco::MatchedPartons']},
0047 {'RecoCandidate' : ['reco::Candidate']},
0048 {'TrackingAnalysis' : ['TrackingParticle']},
0049 {'Egamma' : ['reco::ElectronID']},
0050 {'TopObjects' : ['reco::CATopJetProperties']},
0051 {'TauReco' : ['reco::L2TauIsolationInfo','reco::RecoTauPiZero','reco::BaseTau']},
0052 {'ValidationFormats' : ['PGlobalDigi::.+','PGlobalRecHit::.+']},
0053 {'TrajectorySeed' : ['TrajectorySeed']},
0054 {'TrackCandidate' : ['TrackCandidate']},
0055 {'PatternTools' : ['MomentumConstraint','VertexConstraint','Trajectory']},
0056 {'TrackerRecHit2D' : ['SiStrip(Matched|)RecHit[12]D','SiTrackerGSRecHit[12]D','SiPixelRecHit']},
0057 {'MuonReco' : ['reco::Muon(Ref|)(Vector|)']},
0058 {'MuonSeed' : ['L3MuonTrajectorySeed']},
0059 {'HepMCCandidate' : ['reco::GenParticle.*']},
0060 {'L1Trigger' : ['l1extra::L1.+Particle', 'l1t::Vertex']},
0061 {'TrackInfo' : ['reco::TrackingRecHitInfo']},
0062 {'EgammaCandidates' : ['reco::GsfElectron.*','reco::Photon.*']},
0063 {'HcalIsolatedTrack' : ['reco::IsolatedPixelTrackCandidate', 'reco::EcalIsolatedParticleCandidate', 'reco::HcalIsolatedTrackCandidate']},
0064 {'HcalRecHit' : ['HFRecHit','HORecHit','ZDCRecHit','HBHERecHit','HcalRecHitSoA']},
0065 {'PFRootEvent' : ['EventColin::']},
0066 {'CaloTowers' : ['CaloTower.*']},
0067 {'GsfTrackReco' : ['GsfTrack.*']},
0068 {'METReco' : ['reco::(Calo|PF|Gen|)MET','reco::PFClusterMET']},
0069 {'ParticleFlowReco' : ['reco::RecoPFClusterRefCandidateRef.*']},
0070 {'ParticleFlowCandidate' : ['reco::PFCandidateRef','reco::PFCandidateFwdRef','reco::PFCandidate']},
0071 {'PhysicsToolsObjects' : ['PhysicsTools::Calibration']},
0072 {'TrackReco' : ['reco::Track','reco::TrackRef']},
0073 {'VertexReco' : ['reco::Vertex']},
0074 {'TauReco' : ['reco::PFJetRef']},
0075 {'JetReco' : ['reco::.*Jet','reco::.*Jet(Collection|Ref)']},
0076 {'HGCDigi' : ['HGCSample']},
0077 {'HGCRecHit' : ['constHGCRecHit','HGCRecHit']},
0078 {'SiPixelObjects' : ['SiPixelQuality.*']},
0079 {'EcalRecHit' : ['EcalRecHitSoA.*']},
0080 ]
0081
0082 ignoreEdmDP = {
0083 'LCGReflex/__gnu_cxx::__normal_iterator<std::basic_string<char>*,std::vector<std::basic_string<char>%>%>' : 1,
0084 '' : 1
0085 }
0086
0087 def searchClassDefXml ():
0088 """ Searches through the requested directory looking at
0089 'classes_def.xml' files looking for duplicate Reflex definitions."""
0090
0091 classNameRE = re.compile (r'class\s+name\s*=\s*"([^"]*)"')
0092 spacesRE = re.compile (r'\s+')
0093 stdRE = re.compile (r'std::')
0094 srcClassNameRE = re.compile (r'(\w+)/src/classes_def.*[.]xml')
0095 ignoreSrcRE = re.compile (r'.*/FWCore/Skeletons/scripts/mkTemplates/.+')
0096 braketRE = re.compile (r'<.+>')
0097 print("Searching for 'classes_def.xml' in '%s'." % os.path.join(os.environ.get('CMSSW_BASE'),'src'))
0098 xmlFiles = []
0099 for srcDir in [os.environ.get('CMSSW_BASE'),os.environ.get('CMSSW_RELEASE_BASE')]:
0100 if not len(srcDir): continue
0101 for xml in getoutput ('cd '+os.path.join(srcDir,'src')+'; find . -name "*classes_def*.xml" -follow -print').split ('\n'):
0102 if xml and (not xml in xmlFiles):
0103 xmlFiles.append(xml)
0104 if options.showXMLs:
0105 pprint.pprint (xmlFiles)
0106
0107 xmlPackages = []
0108 packagesREs = {}
0109 equivREs = {}
0110 explicitREs = []
0111 for item in equivDict:
0112 for pack in item:
0113 for equiv in item[pack]:
0114 explicitREs.append( (re.compile(r'\b' + equiv + r'\b'),pack))
0115 if options.lostDefs:
0116 for filename in xmlFiles:
0117 if (not filename) or (ignoreSrcRE.match(filename)): continue
0118 match = srcClassNameRE.search (filename)
0119 if not match: continue
0120 packageName = match.group(1)
0121 xmlPackages.append (packageName)
0122 matchString = r'\b' + packageName + r'\b'
0123 packagesREs[packageName] = re.compile (matchString)
0124 equivList = equivREs.setdefault (packageName, [])
0125 for item in equivDict:
0126 for equiv in item.get (packageName, []):
0127 matchString = re.compile(r'\b' + equiv + r'\b')
0128 equivList.append( (matchString, equiv) )
0129 equivList.append( (packagesREs[packageName], packageName) )
0130 classDict = {}
0131 ncdict = {'class' : 'className', 'function' : 'functionName'}
0132 for filename in xmlFiles:
0133 if (not filename) or (ignoreSrcRE.match(filename)): continue
0134 lostProblems = ''
0135 exceptName = ''
0136 regexList = []
0137 localObjects = []
0138 simpleObjectREs = []
0139 if options.lostDefs:
0140 lostMatch = srcClassNameRE.search (filename)
0141 if lostMatch:
0142 exceptName = lostMatch.group (1)
0143 regexList = equivREs[exceptName]
0144 xcount = len(regexList)-1
0145 if not regexList[xcount][0].search (exceptName):
0146 print('%s not found in' % exceptName, end=' ')
0147 print(regexList[xcount][0])
0148 sys.exit()
0149 else: continue
0150 if options.verbose:
0151 print("filename", filename)
0152 try:
0153 filepath = os.path.join(os.environ.get('CMSSW_BASE'),'src',filename)
0154 if not os.path.exists(filepath):
0155 filepath = os.path.join(os.environ.get('CMSSW_RELEASE_BASE'),'src',filename)
0156 xmlObj = xml2obj (filename = filepath,
0157 filtering = True,
0158 nameChangeDict = ncdict)
0159 except Exception as detail:
0160 print("File %s is malformed XML. Please fix." % filename)
0161 print(" ", detail)
0162 continue
0163 try:
0164 classList = xmlObj.selection.className
0165 except:
0166 try:
0167 classList = xmlObj.className
0168 except:
0169
0170 print("**** SKIPPING '%s' - Doesn't seem to have proper information." % filename)
0171 continue
0172 if not classList:
0173 classList = xmlObj.functionName
0174 if not classList:
0175 print("**** SKIPPING '%s' - Dosen't seem to have proper information(not class/function)." % filename)
0176 continue
0177 for piece in classList:
0178 try:
0179 className = spacesRE.sub ('', piece.name)
0180 except:
0181
0182
0183 continue
0184 className = stdRE.sub ('', className)
0185
0186
0187 for typedef, tdList in typedefsDict.items():
0188 for alias in tdList:
0189 className = re.sub (alias, typedef, className)
0190 classDict.setdefault (className, set()).add (filename)
0191
0192 if not options.lostDefs:
0193 continue
0194 localObjects.append (className)
0195 if options.lazyLostDefs and not braketRE.search (className):
0196
0197 matchString = r'\b' + className + r'\b'
0198 simpleObjectREs.append( (re.compile (matchString), className ) )
0199 for className in localObjects:
0200
0201
0202 foundEquiv = False
0203 for equivRE in regexList:
0204
0205 if equivRE[0].search (className):
0206 foundEquiv = True
0207 break
0208 for simpleRE in simpleObjectREs:
0209 if simpleRE[0].search (className):
0210 foundEquiv = True
0211 if options.verbose and simpleRE[1] != className:
0212 print(" Using %s to ignore %s" \
0213 % (simpleRE[1], className))
0214 break
0215 if foundEquiv: continue
0216 for exRes in explicitREs:
0217 if exRes[0].search(className):
0218 lostProblems += " %s : %s\n" % (exRes[1], className)
0219 foundEquiv = True
0220 break
0221 if foundEquiv: continue
0222 for packageName in xmlPackages:
0223
0224
0225 if packagesREs[packageName].search (className):
0226 lostProblems += " %s : %s\n" % (packageName, className)
0227 break
0228
0229 if lostProblems:
0230 print(f'\n{filename} defines the following dictionaries that should be defined in another package\n{lostProblems}\n')
0231
0232 if options.dups:
0233 for name, fileSet in sorted( classDict.items() ):
0234 if len (fileSet) < 2:
0235 continue
0236 print(f"{name} is defined in more than one package")
0237 fileList = sorted (fileSet)
0238 for filename in fileList:
0239 print(" ", filename)
0240 print()
0241
0242
0243
0244
0245
0246 def searchDuplicatePlugins ():
0247 """ Searches the edmpluginFile to find any duplicate
0248 plugins."""
0249 edmpluginFile = ''
0250 libenv = 'LD_LIBRARY_PATH'
0251 if os.environ.get('SCRAM_ARCH').startswith('osx'): libenv = 'DYLD_FALLBACK_LIBRARY_PATH'
0252 biglib = '/biglib/'+os.environ.get('SCRAM_ARCH')
0253 for libdir in os.environ.get(libenv).split(':'):
0254 if libdir.endswith(biglib): continue
0255 if os.path.exists(libdir+'/.edmplugincache'): edmpluginFile = edmpluginFile + ' ' + libdir+'/.edmplugincache'
0256 if edmpluginFile == '': edmpluginFile = os.path.join(os.environ.get('CMSSW_BASE'),'lib',os.environ.get('SCRAM_ARCH'),'.edmplugincache')
0257 cmd = "cat %s | awk '{print $2\"?\"$3\" \"$1}' | sort | uniq | awk '{print $1}' | sort | uniq -c | grep '2 ' | tr \"?\" \" \" | awk '{print $2}'" % edmpluginFile
0258 output = getoutput (cmd).split('\n')
0259 for line in output:
0260 if line in ignoreEdmDP: continue
0261 line = line.replace("*","\*")
0262 cmd = "cat %s | grep ' %s ' | awk '{print $1}' | sort | uniq " % (edmpluginFile,line)
0263 out1 = getoutput (cmd).split('\n')
0264 print(line)
0265 for plugin in out1:
0266 if plugin:
0267 print(" **"+plugin+"**")
0268 print()
0269
0270 if __name__ == "__main__":
0271
0272 parser = optparse.OptionParser ("Usage: %prog [options]\n"\
0273 "Searches classes_def.xml for wrong/duplicate "\
0274 "definitions")
0275 xmlGroup = optparse.OptionGroup (parser, "ClassDef XML options")
0276 dumpGroup = optparse.OptionGroup (parser, "EdmPluginDump options")
0277 xmlGroup.add_option ('--dups', dest='dups', action='store_true',
0278 default=False,
0279 help="Search for duplicate definitions")
0280 xmlGroup.add_option ('--lostDefs', dest='lostDefs', action='store_true',
0281 default=False,
0282 help="Looks for definitions in the wrong libraries")
0283 xmlGroup.add_option ('--lazyLostDefs', dest='lazyLostDefs',
0284 action='store_true',
0285 default=False,
0286 help="Will try to ignore as many lost defs as reasonable")
0287 xmlGroup.add_option ('--verbose', dest='verbose',
0288 action='store_true',
0289 default=False,
0290 help="Prints out a lot of information")
0291 xmlGroup.add_option ('--showXMLs', dest='showXMLs', action='store_true',
0292 default=False,
0293 help="Shows all 'classes_def.xml' files")
0294 xmlGroup.add_option ('--dir', dest='srcdir', type='string', default='',
0295 help="Obsolete")
0296 dumpGroup.add_option ('--edmPD', dest='edmPD', action='store_true',
0297 default=False,
0298 help="Searches EDM Plugin Dump for duplicates")
0299 dumpGroup.add_option ('--edmFile', dest='edmFile', type='string',
0300 default='',
0301 help="Obsolete")
0302 parser.add_option_group (xmlGroup)
0303 parser.add_option_group (dumpGroup)
0304 (options, args) = parser.parse_args()
0305
0306
0307 if options.lazyLostDefs:
0308 options.lostDefs = True
0309 if options.showXMLs or options.lostDefs or options.dups:
0310 searchClassDefXml ()
0311 if options.edmPD:
0312 searchDuplicatePlugins ()