Warning, /DQM/Integration/scripts/XMLcfgfiles/pscram is written in an unsupported language. File is not indexed.
0001 #!/usr/bin/env python3
0002 import os,subprocess,sys,re,pprint,glob,getopt
0003
0004 from psClasses import *
0005 ### Global Lists and Directories
0006 SRCDIR ="%s/src/" % os.getenv("CMSSW_BASE")
0007 ServerDict = {}
0008 BuilFilesList = []
0009 ModuleTable = {}
0010 ModuleList = []
0011 buildTree = BuildTreeNodeList()
0012 serverQueues = queueList()
0013
0014
0015 ### Functions
0016
0017 def getBuildFiles():
0018 for cDir,subDirs,files in os.walk(SRCDIR):
0019 for f in files:
0020 if re.match(r"BuildFile.xml",f):
0021 BuilFilesList.append("%s/%s" %(cDir,f) )
0022 break
0023 if re.match(r"BuildFile",f):
0024 BuilFilesList.append("%s/%s" %(cDir,f) )
0025 break
0026
0027 def fillModuleRecord(bFile):
0028 bNameRegEx=r"file=[\"']*(?P<binNames>[a-zA-Z0-9\._\-\*,;\s]+)[\"']*"
0029 dirInfo=os.path.dirname(bFile).replace(SRCDIR,"")
0030 module=dirInfo.split("/")[0]
0031 subModule="%s/%s" % (module,dirInfo.split("/")[1])
0032 stdLibName=dirInfo.replace("/","").replace("plugin","Plugin")
0033 f=open(bFile,"r")
0034 deps=[]
0035 line=f.readline()
0036 libNames=[]
0037 binNames=[]
0038 buildFiles=[]
0039 while line:
0040 if re.search(r"<use.*",line):
0041 depName=re.search(r"name=[\"']*(?P<depName>([a-zA-Z0-9]+/[a-zA-Z0-9]+|[a-zA-z0-9._\-]+))[\"']*",line).group("depName")
0042 os.path.exists("%s/%s" % (SRCDIR,depName)) and deps.append(depName)
0043 elif re.search(r"\s+<lib ",line):
0044 libName=re.search(r"name=[\"']*(?P<libName>[a-zA-Z0-9]*)[\"']*",line).group("libName")
0045 libNames.append(libName == "1" and stdLibName or libName)
0046 elif re.search(r"<library",line):
0047 try:
0048 libName=re.search(r"name=[\"']*(?P<libName>[a-zA-Z0-9]*)[\"']*",line).group("libName")
0049 except:
0050 libName=re.search(r"file=[\"']*(?P<libName>[a-zA-Z0-9_]*).*",line).group("libName")
0051 libNames.append(libName)
0052 buildFiles.append(re.search(bNameRegEx,line).group("binNames"))
0053 elif re.search(r"<bin",line):
0054 try:
0055 binNames.append(re.search(r"name=[\"']*(?P<binName>[a-zA-Z0-9_\-]+)[\"']*",line).group("binName"))
0056 except:
0057 binNames.append(re.search(r"file=[\"']*(?P<binName>[a-zA-Z0-9_\-]+).*[\"']*",line).group("binName"))
0058 buildFiles.append(re.search(bNameRegEx,line).group("binNames"))
0059 line=f.readline()
0060 f.close()
0061 dirInfoBFiles= len(dirInfo.split("/"))==2 and "%s/src" % dirInfo or dirInfo
0062 ## Separating lilst of files to have a complete list of individual source files
0063 for sep in " \t,;":
0064 bf2=[]
0065 for bF1 in [bF.split(sep) for bF in buildFiles]:
0066 bf2+=bF1
0067 buildFiles=bf2
0068 ## Expanding * to file names
0069 bf2=[]
0070 for bF1 in [glob.glob("%s/%s" % (dirInfoBFiles,bF)) for bF in buildFiles if os.path.exists("%s/%s" % (dirInfoBFiles,bF)) or "*" in bF ]:
0071 bf2+=bF1
0072 bf2=dict.fromkeys(bf2)
0073 buildFiles=bf2.keys()
0074 libNames+=binNames
0075 return [module,subModule,deps,libNames,buildFiles,len(deps),len(buildFiles)/len(libNames)]
0076
0077 def fillModuleTable():
0078 for f in BuilFilesList:
0079 record=fillModuleRecord(f)
0080 ModuleTable.setdefault(record[0],{}).setdefault(record[1],[]).append(record[2:]+[0])
0081 ModuleList.append(record)
0082
0083
0084 def fillBuildTree():
0085 global serverQueues
0086 finished=False
0087 Count=0
0088 while not finished and Count < 7:
0089 finished=True
0090 for item in ModuleList:
0091 if len(item[2]) == 0:
0092 for lib in item[3]:
0093 if not buildTree.findLib(lib):
0094 buildTree.append(BuildTreeNode(libname=lib,module=item[0],submodule=item[1],weight=item[6],srvqueue=serverQueues))
0095 else:
0096 deps=item[2]
0097 libs=item[3]
0098 for lib in libs:
0099 pNode=buildTree.findLib(lib)
0100 if not pNode:
0101 node=BuildTreeNode(libname=lib,module=item[0],submodule=item[1],weight=item[6],srvqueue=serverQueues)
0102 else:
0103 continue
0104 for dep in deps:
0105 dpNode=buildTree.findDep(dep)
0106 if dpNode is not None:
0107 if dpNode not in node.DependsOn:
0108 node.DependsOn.append(dpNode)
0109 if node not in dpNode.AreDependent:
0110 dpNode.AreDependent.append(node)
0111 elif len(glob.glob("%s%s/BuildFile*" %(SRCDIR,dep))) == 0 :
0112 continue
0113 else:
0114 finished=False
0115 if Count==6:
0116 print "WARNING: Traversed The tree %d times and still there are some missing dependencies and no buildFile found for them: %s" % (Count,dep)
0117 Count += 1
0118
0119 def checkBuildOrder():
0120 concatenatedLog=[]
0121 for srv in serverQueues.keys():
0122 for i in serverQueues[srv].ThreadLog:
0123 concatenatedLog.append(i[:2])
0124 concatenatedLog.sort(cmp = lambda x,y: cmp(x[0],y[0]))
0125 seenLibs=[]
0126 Result=True
0127 for lib in concatenatedLog:
0128 node=buildTree.findLib(lib[1])
0129 seenLibs.append(lib[1])
0130 failedDeps=[]
0131 for depNode in node.DependsOn:
0132 if depNode.LibName not in seenLibs:
0133 failedDeps.append(lib)
0134 Result=False
0135 return (Result,failedDeps)
0136
0137 ####
0138 # Main Program
0139 ####
0140
0141 if not os.getenv("CMSSW_BASE"):
0142 print "No CMSSW Environment detected"
0143 sys.exit(os.EX_CONFIG)
0144
0145 if __name__ == "__main__":
0146
0147 """
0148 hosts
0149 """
0150 try:
0151 opts, args = getopt.getopt(sys.argv[1:], "j:n:")
0152 except getopt.GetoptError, err:
0153 # print help information and exit:
0154 print str(err) # will print something like "option -a not recognized"
0155 sys.exit(2)
0156 cores = -1
0157 jay =-1
0158 hosts=[arg.replace(","," ") for arg in args]
0159 for o, a in opts:
0160 if o == "-j":
0161 jay = int(a)
0162 elif o == "-n":
0163 cores = int(a)
0164 else:
0165 assert False, "unhandled option"
0166
0167 if cores == -1:
0168 print "Assuming 4 cores servers"
0169 cores = 4
0170
0171 if jay == -1:
0172 print "Assuming 2 threads per job"
0173 jay = 2
0174
0175 if len(hosts) == 0:
0176 print "No servers listed"
0177 sys.exit(1)
0178
0179 for srv in hosts:
0180 retcode=call(["ping","-c 1",srv],stdout=open("/dev/null"),stderr=open("/dev/null"))
0181 if retcode is not 0:
0182 print "Server %s is unreachable" % srv
0183 sys.exit(retcode)
0184
0185
0186 serverQueues = queueList(hosts,cores,jay)
0187 getBuildFiles()
0188 fillModuleTable()
0189 fillBuildTree()
0190 #print buildTree[1]
0191 #buildTree.BThread.start()
0192 #print enumerate()
0193 print "Setting up caches"
0194 nullFile=open("/dev/null")
0195 call(['scram','build','-r','-n'],stdout=nullFile,stderr=nullFile,cwd=os.path.realpath(os.path.curdir))
0196 nullFile.close()
0197 print "Starting threads"
0198 buildTree.startThreads()
0199 a=len(enumerate())
0200 while a > 1:
0201 #print "Running Threads = %d" % a
0202 a=len(enumerate())
0203 time.sleep(1)
0204 for srv in serverQueues.keys():
0205 for i in serverQueues[srv].ThreadLog:
0206 print i[2]
0207 r,failedLibs = checkBuildOrder()
0208 print "DepBuild Check %s, failed libs: \n...%s" % (r and "Succeded" or "Failed","\n...".join(failedLibs))
