Warning, /Utilities/ReleaseScripts/scripts/MakeBuildSet is written in an unsupported language. File is not indexed.
0001 #! /usr/bin/env python3
0002 from __future__ import print_function
0003 #pylint: disable-msg=W0403
0004 """
0005 MakeBuildSet - utilizes dependency discovery mechanism for Partial Releases.
0006
0007 1 July 2008 - with -n option (test mode) does not reuse or save cached data
0008 - products dependencies are now taken into consideration
0009 """
0010
0011 __revision__ = "$Id: MakeBuildSet,v 1.8 2009/07/02 16:39:24 muzaffar Exp $"
0012 __version__ = "$Revision: 1.8 $"
0013
0014 import os
0015 import sys
0016 import getopt
0017 import re
0018
0019 __version__ = __version__.replace("$","").replace("Revision: ","")
0020 programPath = os.path.dirname( os.path.realpath(sys.argv[0] ) )
0021 programName = os.path.basename( sys.argv[0] )
0022
0023 packageListInFile = ""
0024 dependencyAnalysisDir = ""
0025 outputFormats = ["packages", "lcg", "tools"]
0026 outputFormat = "packages"
0027 dependencyTypes = ["binary", "source", "combined"]
0028 dependencyType = "binary"
0029 dropPluginDeps = 0
0030 dropAllPluginDeps = 0
0031 dropTests = 1
0032
0033 ##############################################################
0034 # Internal utility functions:
0035
0036 def usage():
0037 """
0038 Prints out usage help
0039 """
0040 # Require minimum arguments and use options only to
0041 # override the defaults
0042 print ("""
0043 USAGE:
0044 """ + programName + """ -d <dir> [OPTIONS] [<package>]+
0045 OPTIONS:
0046 -d <dir> - Required: get dependency information from directory <dir>,
0047 There should be at least dependencies.txt and products.txt
0048 files exist under this directory
0049 -f <file> - Optional: read list of packages from <file>.
0050 -o <format> - Optional: <format> = [ """ + " | ".join(outputFormats) + """ | all ]
0051 Default value is '""" + outputFormats[0] + """'.
0052 -D <dependency>
0053 - Optional: <dependency> = [ """ + " | ".join(dependencyTypes) + """ ]
0054 Default value is '""" + dependencyTypes[0] + """'
0055 -p - Optional: drop products generated from plugins directories
0056 -P - Optional: drop all plugins even generated from
0057 <subsystem>/<package>/[src|bin|test] directories
0058 -T - Optional: do not drop products generated from test directories.
0059 NOTE: if -P is used then plugins generated from test area
0060 will be excluded
0061 -h - Optional: print this help and exit.
0062 -v - Optional: display version number.
0063 """)
0064
0065 def usageError ( message, programName) :
0066 """
0067 usageError:
0068 Call it to quit in case of usage error
0069 """
0070 print ("Usage error: ", message, \
0071 "\nTry \'" + programName + " -h\' for help")
0072 sys.exit ( 2 )
0073
0074
0075 ##############################################################
0076 # Interface functions:
0077 def readPackageFromFile(pkfile):
0078 cache = {}
0079 f = open(pkfile)
0080 blankLine = re.compile('^\s*(#.*|)$')
0081 for line in f.readlines():
0082 line = line.strip()
0083 if blankLine.match(line):
0084 continue
0085 else:
0086 cache[line]=1
0087 f.close()
0088 return cache
0089
0090 def readProductsInfo(productFile):
0091 cache = {}
0092 prods=open(productFile)
0093 comment= re.compile("^\s*(#.*|)$")
0094 parts = re.compile("^([^:]+):([^:]+):([^:]+):([^:]+)$")
0095 for line in prods.readlines():
0096 line = line.strip()
0097 if comment.match(line):
0098 continue
0099 result = parts.match(line)
0100 if result:
0101 prodFrom = result.group(1)
0102 typeName = result.group(2)
0103 dirPath = result.group(3)
0104 prodName = result.group(4)
0105 cache[prodName]={}
0106 cache[prodName]["FROM"] = prodFrom
0107 cache[prodName]["TYPE"] = typeName
0108 cache[prodName]["PATH"] = dirPath
0109 prods.close()
0110 return cache
0111
0112 def excludeProducts(lookup,match,cache,maps):
0113 for p in cache.keys():
0114 if cache[p][lookup] == match:
0115 maps[p]=1
0116
0117 def addDependencies(packages, dependencies):
0118 cache = {}
0119 for t in outputFormats:
0120 cache[t] = {}
0121 for p in packages.keys():
0122 addDependency(p,"packages",dependencies,cache)
0123 return cache
0124
0125 def addDependency(p,scope,dependencies,cache):
0126 if p in cache[scope]:
0127 return
0128 cache[scope][p]=1
0129 if p in dependencies[scope]:
0130 for d in dependencies[scope][p].keys():
0131 if d in dependencies["tools"]:
0132 cache["tools"][d]=1
0133 continue
0134 xscope = "lcg"
0135 if not d in dependencies[xscope]:
0136 xscope = "packages"
0137 addDependency(d,xscope,dependencies,cache)
0138
0139 def readIgnominyDependencyDB(igFile,droppedProducts,packages):
0140 dependencies = {}
0141 for t in outputFormats:
0142 dependencies[t] = {}
0143 inBlock = 0
0144 inPackage = 0
0145 skipProduct = 0
0146 pname = re.compile("^(.*?)_(.*)")
0147 beginBlock = re.compile("^# Direct\s+" + dependencyType + "\s+dependencies")
0148 endBlock = re.compile('^############################*')
0149 packageEntry = re.compile('^(\S+)(\s+.+|):')
0150 requiresEntry = re.compile('^\s+(\S+)\s*$')
0151 tools = re.compile('^(tools|System)/(.+)\s*$',re.IGNORECASE)
0152 lcgProject = re.compile('^((CORAL|SEAL|POOL)/([^/]+))(/src|)',re.IGNORECASE)
0153 extraEntry = re.compile('^(.+)/(plugins|test|bin)\s*')
0154 f = open(igFile)
0155 for line in f.readlines():
0156 if inBlock:
0157 if endBlock.match(line):
0158 inBlock = 0
0159 return dependencies
0160 else:
0161 if inPackage:
0162 result=requiresEntry.match(line)
0163 if result:
0164 if not skipProduct:
0165 requires=result.group(0).strip()
0166 if pname.match(requires):
0167 continue
0168 if dependencyType != "binary" and extraEntry.match(requires):
0169 continue
0170 lcg = ""
0171 res = lcgProject.match (requires)
0172 if res:
0173 lcg = res.group(2).lower()
0174 requires = res.group(1)
0175 dependencies["tools"][lcg]=1
0176 else:
0177 res = tools.match (requires)
0178 if res:
0179 requires = res.group(2).lower()
0180 dependencies["tools"][requires]=1
0181 if currentPackage in dependencies["packages"]:
0182 dependencies["packages"][currentPackage][requires]=1
0183 if lcg:
0184 dependencies["packages"][currentPackage][lcg]=1
0185 else:
0186 dependencies["lcg"][currentPackage][requires]=1
0187 else:
0188 inPackage=0
0189 else:
0190 result = packageEntry.match(line)
0191 if result:
0192 inPackage = 1
0193 skipProduct = 0
0194 currentPackage = result.group(1)
0195 if currentPackage == "(UNKNOWN)":
0196 skipProduct = 1
0197 continue
0198 res = lcgProject.match(currentPackage)
0199 if res:
0200 dependencies["tools"][res.group(2).lower()]=1
0201 currentPackage = res.group(1)
0202 if not currentPackage in dependencies["lcg"]:
0203 dependencies["lcg"][currentPackage]={}
0204 continue
0205 res = tools.match (currentPackage)
0206 if res:
0207 currentPackage = res.group(2).lower()
0208 dependencies["tools"][currentPackage]=1
0209 skipProduct = 1
0210 continue
0211 prodname=currentPackage.replace("/","")
0212 pkg = 1
0213 res = pname.match(currentPackage)
0214 if res:
0215 pkg = 0
0216 currentPackage = res.group(1)
0217 prodname = res.group(2)
0218 elif dependencyType != "binary":
0219 res = extraEntry.match(currentPackage)
0220 if res:
0221 if res.group(2) == "test" and dropTests:
0222 skipProduct = 1
0223 continue
0224 elif res.group(2) == "plugins" and dropPluginDeps:
0225 skipProduct = 1
0226 continue
0227 else:
0228 pkg = 0
0229 currentPackage = res.group(1)
0230 prodname = currentPackage.replace("/","")
0231 if (not pkg) or (not currentPackage in packages):
0232 if prodname in droppedProducts:
0233 skipProduct = 1
0234 continue
0235 if not currentPackage in dependencies["packages"]:
0236 dependencies["packages"][currentPackage]={}
0237 else:
0238 if beginBlock.match(line):
0239 inBlock = 1
0240 f.close()
0241 return dependencies
0242
0243 ###############################################################
0244 # Here real action starts:
0245
0246 # Handle options and arguments:
0247 try:
0248 opts, args = getopt.getopt( sys.argv[1:], "hpPTvf:D:d:o:" )
0249
0250 except getopt.error:
0251 usageError( sys.exc_info()[1], programName)
0252
0253 # process verbosity options separately before anything else.
0254
0255 for o, a in opts:
0256 if o == "-v":
0257 print (__version__)
0258 sys.exit()
0259 if o == "-h":
0260 usage()
0261 sys.exit()
0262 if o == "-f":
0263 packageListInFile = a
0264 if not os.path.isfile(a):
0265 usageError( "file '" + a + "' does not exist!", programName)
0266 if o == "-o":
0267 outputFormat = a
0268 if a != "all" and not a in outputFormats:
0269 usageError( "unknown output format: '" + a + "'.", programName)
0270 if o == "-D":
0271 dependencyType = a
0272 if not a in dependencyTypes:
0273 usageError( "unknown ignominy generated dependency type: '" + a + "'.", programName)
0274 if o == "-d":
0275 if not os.path.isdir(a):
0276 usageError( "directory '" + a + "' does not exist!", programName)
0277 dependencyAnalysisDir = a
0278 if o == "-p":
0279 dropPluginDeps = 1
0280 if o == "-P":
0281 dropAllPluginDeps = 1
0282 dropPluginDeps = 1
0283 if o == "-T":
0284 dropTests = 0
0285
0286 if dependencyAnalysisDir == "":
0287 usageError( "missing ignominy analysis directory which should contain at least products.txt and dependencies.txt files.", programName)
0288
0289 # Add packages from file:
0290 packages = {}
0291 if (packageListInFile):
0292 packages = readPackageFromFile(packageListInFile)
0293 # Add extra packages passed via command-line
0294 for p in args[0:]:
0295 packages[p]=1
0296
0297 if not len(packages.keys()):
0298 print ("""
0299 ERROR: You should use either '-f <file>' command-line option
0300 to provide the list of packages to look for OR provide
0301 a <space> separated list of package at the end of command-line
0302 """)
0303 sys.exit(2)
0304
0305 # Read Product information dumped by ignominy in products.txt file
0306 productCache = readProductsInfo(os.path.join(dependencyAnalysisDir, "products.txt"))
0307
0308 # Exclude test and plugins products if requested via command-line
0309 droppedProducts={}
0310 if dropTests:
0311 excludeProducts("FROM","TEST",productCache,droppedProducts)
0312 if dropAllPluginDeps:
0313 excludeProducts("FROM","PLUGINS",productCache,droppedProducts)
0314 excludeProducts("TYPE","PLUGINS",productCache,droppedProducts)
0315 elif dropPluginDeps:
0316 excludeProducts("FROM","PLUGINS",productCache,droppedProducts)
0317
0318 # Get dependency info:
0319 dependencies = readIgnominyDependencyDB(os.path.join(dependencyAnalysisDir,"dependencies.txt"),droppedProducts,packages)
0320
0321 # Cleanup, this information is not needed after this
0322 droppedProducts={}
0323
0324 myPackages = addDependencies(packages, dependencies)
0325
0326 # Cleanup, no more needed dependencies information
0327 dependencies = {}
0328
0329 # Print output and finish:
0330 pitems = []
0331 if outputFormat != "all":
0332 outputFormats = [outputFormat]
0333 for item in outputFormats:
0334 pkg = sorted(myPackages[item].keys())
0335 for p in pkg:
0336 print (p)
0337
0338 sys.exit()
