Warning, /Utilities/ReleaseScripts/scripts/MakeBuildSet is written in an unsupported language. File is not indexed.
0001 #! /usr/bin/env python3
0002 #pylint: disable-msg=W0403
0003 """
0004 MakeBuildSet - utilizes dependency discovery mechanism for Partial Releases.
0005
0006 1 July 2008 - with -n option (test mode) does not reuse or save cached data
0007 - products dependencies are now taken into consideration
0008 """
0009
0010 __revision__ = "$Id: MakeBuildSet,v 1.8 2009/07/02 16:39:24 muzaffar Exp $"
0011 __version__ = "$Revision: 1.8 $"
0012
0013 import os
0014 import sys
0015 import getopt
0016 import re
0017
0018 __version__ = __version__.replace("$","").replace("Revision: ","")
0019 programPath = os.path.dirname( os.path.realpath(sys.argv[0] ) )
0020 programName = os.path.basename( sys.argv[0] )
0021
0022 packageListInFile = ""
0023 dependencyAnalysisDir = ""
0024 outputFormats = ["packages", "lcg", "tools"]
0025 outputFormat = "packages"
0026 dependencyTypes = ["binary", "source", "combined"]
0027 dependencyType = "binary"
0028 dropPluginDeps = 0
0029 dropAllPluginDeps = 0
0030 dropTests = 1
0031
0032 ##############################################################
0033 # Internal utility functions:
0034
0035 def usage():
0036 """
0037 Prints out usage help
0038 """
0039 # Require minimum arguments and use options only to
0040 # override the defaults
0041 print ("""
0042 USAGE:
0043 """ + programName + """ -d <dir> [OPTIONS] [<package>]+
0044 OPTIONS:
0045 -d <dir> - Required: get dependency information from directory <dir>,
0046 There should be at least dependencies.txt and products.txt
0047 files exist under this directory
0048 -f <file> - Optional: read list of packages from <file>.
0049 -o <format> - Optional: <format> = [ """ + " | ".join(outputFormats) + """ | all ]
0050 Default value is '""" + outputFormats[0] + """'.
0051 -D <dependency>
0052 - Optional: <dependency> = [ """ + " | ".join(dependencyTypes) + """ ]
0053 Default value is '""" + dependencyTypes[0] + """'
0054 -p - Optional: drop products generated from plugins directories
0055 -P - Optional: drop all plugins even generated from
0056 <subsystem>/<package>/[src|bin|test] directories
0057 -T - Optional: do not drop products generated from test directories.
0058 NOTE: if -P is used then plugins generated from test area
0059 will be excluded
0060 -h - Optional: print this help and exit.
0061 -v - Optional: display version number.
0062 """)
0063
0064 def usageError ( message, programName) :
0065 """
0066 usageError:
0067 Call it to quit in case of usage error
0068 """
0069 print ("Usage error: ", message, \
0070 "\nTry \'" + programName + " -h\' for help")
0071 sys.exit ( 2 )
0072
0073
0074 ##############################################################
0075 # Interface functions:
0076 def readPackageFromFile(pkfile):
0077 cache = {}
0078 f = open(pkfile)
0079 blankLine = re.compile('^\s*(#.*|)$')
0080 for line in f.readlines():
0081 line = line.strip()
0082 if blankLine.match(line):
0083 continue
0084 else:
0085 cache[line]=1
0086 f.close()
0087 return cache
0088
0089 def readProductsInfo(productFile):
0090 cache = {}
0091 prods=open(productFile)
0092 comment= re.compile("^\s*(#.*|)$")
0093 parts = re.compile("^([^:]+):([^:]+):([^:]+):([^:]+)$")
0094 for line in prods.readlines():
0095 line = line.strip()
0096 if comment.match(line):
0097 continue
0098 result = parts.match(line)
0099 if result:
0100 prodFrom = result.group(1)
0101 typeName = result.group(2)
0102 dirPath = result.group(3)
0103 prodName = result.group(4)
0104 cache[prodName]={}
0105 cache[prodName]["FROM"] = prodFrom
0106 cache[prodName]["TYPE"] = typeName
0107 cache[prodName]["PATH"] = dirPath
0108 prods.close()
0109 return cache
0110
0111 def excludeProducts(lookup,match,cache,maps):
0112 for p in cache.keys():
0113 if cache[p][lookup] == match:
0114 maps[p]=1
0115
0116 def addDependencies(packages, dependencies):
0117 cache = {}
0118 for t in outputFormats:
0119 cache[t] = {}
0120 for p in packages.keys():
0121 addDependency(p,"packages",dependencies,cache)
0122 return cache
0123
0124 def addDependency(p,scope,dependencies,cache):
0125 if p in cache[scope]:
0126 return
0127 cache[scope][p]=1
0128 if p in dependencies[scope]:
0129 for d in dependencies[scope][p].keys():
0130 if d in dependencies["tools"]:
0131 cache["tools"][d]=1
0132 continue
0133 xscope = "lcg"
0134 if not d in dependencies[xscope]:
0135 xscope = "packages"
0136 addDependency(d,xscope,dependencies,cache)
0137
0138 def readIgnominyDependencyDB(igFile,droppedProducts,packages):
0139 dependencies = {}
0140 for t in outputFormats:
0141 dependencies[t] = {}
0142 inBlock = 0
0143 inPackage = 0
0144 skipProduct = 0
0145 pname = re.compile("^(.*?)_(.*)")
0146 beginBlock = re.compile("^# Direct\s+" + dependencyType + "\s+dependencies")
0147 endBlock = re.compile('^############################*')
0148 packageEntry = re.compile('^(\S+)(\s+.+|):')
0149 requiresEntry = re.compile('^\s+(\S+)\s*$')
0150 tools = re.compile('^(tools|System)/(.+)\s*$',re.IGNORECASE)
0151 lcgProject = re.compile('^((CORAL|SEAL|POOL)/([^/]+))(/src|)',re.IGNORECASE)
0152 extraEntry = re.compile('^(.+)/(plugins|test|bin)\s*')
0153 f = open(igFile)
0154 for line in f.readlines():
0155 if inBlock:
0156 if endBlock.match(line):
0157 inBlock = 0
0158 return dependencies
0159 else:
0160 if inPackage:
0161 result=requiresEntry.match(line)
0162 if result:
0163 if not skipProduct:
0164 requires=result.group(0).strip()
0165 if pname.match(requires):
0166 continue
0167 if dependencyType != "binary" and extraEntry.match(requires):
0168 continue
0169 lcg = ""
0170 res = lcgProject.match (requires)
0171 if res:
0172 lcg = res.group(2).lower()
0173 requires = res.group(1)
0174 dependencies["tools"][lcg]=1
0175 else:
0176 res = tools.match (requires)
0177 if res:
0178 requires = res.group(2).lower()
0179 dependencies["tools"][requires]=1
0180 if currentPackage in dependencies["packages"]:
0181 dependencies["packages"][currentPackage][requires]=1
0182 if lcg:
0183 dependencies["packages"][currentPackage][lcg]=1
0184 else:
0185 dependencies["lcg"][currentPackage][requires]=1
0186 else:
0187 inPackage=0
0188 else:
0189 result = packageEntry.match(line)
0190 if result:
0191 inPackage = 1
0192 skipProduct = 0
0193 currentPackage = result.group(1)
0194 if currentPackage == "(UNKNOWN)":
0195 skipProduct = 1
0196 continue
0197 res = lcgProject.match(currentPackage)
0198 if res:
0199 dependencies["tools"][res.group(2).lower()]=1
0200 currentPackage = res.group(1)
0201 if not currentPackage in dependencies["lcg"]:
0202 dependencies["lcg"][currentPackage]={}
0203 continue
0204 res = tools.match (currentPackage)
0205 if res:
0206 currentPackage = res.group(2).lower()
0207 dependencies["tools"][currentPackage]=1
0208 skipProduct = 1
0209 continue
0210 prodname=currentPackage.replace("/","")
0211 pkg = 1
0212 res = pname.match(currentPackage)
0213 if res:
0214 pkg = 0
0215 currentPackage = res.group(1)
0216 prodname = res.group(2)
0217 elif dependencyType != "binary":
0218 res = extraEntry.match(currentPackage)
0219 if res:
0220 if res.group(2) == "test" and dropTests:
0221 skipProduct = 1
0222 continue
0223 elif res.group(2) == "plugins" and dropPluginDeps:
0224 skipProduct = 1
0225 continue
0226 else:
0227 pkg = 0
0228 currentPackage = res.group(1)
0229 prodname = currentPackage.replace("/","")
0230 if (not pkg) or (not currentPackage in packages):
0231 if prodname in droppedProducts:
0232 skipProduct = 1
0233 continue
0234 if not currentPackage in dependencies["packages"]:
0235 dependencies["packages"][currentPackage]={}
0236 else:
0237 if beginBlock.match(line):
0238 inBlock = 1
0239 f.close()
0240 return dependencies
0241
0242 ###############################################################
0243 # Here real action starts:
0244
0245 # Handle options and arguments:
0246 try:
0247 opts, args = getopt.getopt( sys.argv[1:], "hpPTvf:D:d:o:" )
0248
0249 except getopt.error:
0250 usageError( sys.exc_info()[1], programName)
0251
0252 # process verbosity options separately before anything else.
0253
0254 for o, a in opts:
0255 if o == "-v":
0256 print (__version__)
0257 sys.exit()
0258 if o == "-h":
0259 usage()
0260 sys.exit()
0261 if o == "-f":
0262 packageListInFile = a
0263 if not os.path.isfile(a):
0264 usageError( "file '" + a + "' does not exist!", programName)
0265 if o == "-o":
0266 outputFormat = a
0267 if a != "all" and not a in outputFormats:
0268 usageError( "unknown output format: '" + a + "'.", programName)
0269 if o == "-D":
0270 dependencyType = a
0271 if not a in dependencyTypes:
0272 usageError( "unknown ignominy generated dependency type: '" + a + "'.", programName)
0273 if o == "-d":
0274 if not os.path.isdir(a):
0275 usageError( "directory '" + a + "' does not exist!", programName)
0276 dependencyAnalysisDir = a
0277 if o == "-p":
0278 dropPluginDeps = 1
0279 if o == "-P":
0280 dropAllPluginDeps = 1
0281 dropPluginDeps = 1
0282 if o == "-T":
0283 dropTests = 0
0284
0285 if dependencyAnalysisDir == "":
0286 usageError( "missing ignominy analysis directory which should contain at least products.txt and dependencies.txt files.", programName)
0287
0288 # Add packages from file:
0289 packages = {}
0290 if (packageListInFile):
0291 packages = readPackageFromFile(packageListInFile)
0292 # Add extra packages passed via command-line
0293 for p in args[0:]:
0294 packages[p]=1
0295
0296 if not len(packages.keys()):
0297 print ("""
0298 ERROR: You should use either '-f <file>' command-line option
0299 to provide the list of packages to look for OR provide
0300 a <space> separated list of package at the end of command-line
0301 """)
0302 sys.exit(2)
0303
0304 # Read Product information dumped by ignominy in products.txt file
0305 productCache = readProductsInfo(os.path.join(dependencyAnalysisDir, "products.txt"))
0306
0307 # Exclude test and plugins products if requested via command-line
0308 droppedProducts={}
0309 if dropTests:
0310 excludeProducts("FROM","TEST",productCache,droppedProducts)
0311 if dropAllPluginDeps:
0312 excludeProducts("FROM","PLUGINS",productCache,droppedProducts)
0313 excludeProducts("TYPE","PLUGINS",productCache,droppedProducts)
0314 elif dropPluginDeps:
0315 excludeProducts("FROM","PLUGINS",productCache,droppedProducts)
0316
0317 # Get dependency info:
0318 dependencies = readIgnominyDependencyDB(os.path.join(dependencyAnalysisDir,"dependencies.txt"),droppedProducts,packages)
0319
0320 # Cleanup, this information is not needed after this
0321 droppedProducts={}
0322
0323 myPackages = addDependencies(packages, dependencies)
0324
0325 # Cleanup, no more needed dependencies information
0326 dependencies = {}
0327
0328 # Print output and finish:
0329 pitems = []
0330 if outputFormat != "all":
0331 outputFormats = [outputFormat]
0332 for item in outputFormats:
0333 pkg = sorted(myPackages[item].keys())
0334 for p in pkg:
0335 print (p)
0336
0337 sys.exit()
