test/autoSubmitter/helpers.py

0001 from __future__ import print_function
0002 import re
0003 import os
0004 import subprocess
0005 import errno
0006 shortcuts = {}
0007
0008 # regex matching on key, replacement of groups on value
0009 # implement any other shortcuts that you want to use
0010 #sources
0011 shortcuts["mp([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm/alignments_MP.db"
0012 shortcuts["mp([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm{1}/alignments_MP.db"
0013 shortcuts["sm([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/sm{0}/alignments_iter{1}.db"
0014 shortcuts["um([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm/um{0}.db"
0015 shortcuts["um([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm{1}/um{0}.db"
0016 shortcuts["hp([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/hp{0}/alignments_iter{1}.db"
0017 shortcuts["prod"] = "frontier://FrontierProd/CMS_CONDITIONS"
0018
0019 # Exact numbers don't really matter, but it is important that each one has a unique
0020 # number, so that states are distinguishable
0021 STATE_NONE = -1
0022 STATE_ITERATION_START=0
0023 STATE_BJOBS_WAITING=1
0024 STATE_BJOBS_DONE=2
0025 STATE_BJOBS_FAILED=12
0026 STATE_MERGE_WAITING=3
0027 STATE_MERGE_DONE=4
0028 STATE_MERGE_FAILED=14
0029 STATE_SUMMARY_WAITING=5
0030 STATE_SUMMARY_DONE=6
0031 STATE_SUMMARY_FAILED=16
0032 STATE_LOCAL_WAITING=7
0033 STATE_LOCAL_DONE=8
0034 STATE_LOCAL_FAILED=18
0035 STATE_FINISHED=9
0036 STATE_INVALID_CONDITIONS = 101
0037
0038 status_map = {}
0039 status_map[STATE_NONE] = "none"
0040 status_map[STATE_ITERATION_START] = "starting iteration"
0041 status_map[STATE_BJOBS_WAITING] = "waiting for jobs"
0042 status_map[STATE_BJOBS_DONE] = "jobs finished"
0043 status_map[STATE_BJOBS_FAILED] = "jobs failed"
0044 status_map[STATE_MERGE_WAITING] = "waiting for merging"
0045 status_map[STATE_MERGE_DONE] = "merging done"
0046 status_map[STATE_MERGE_FAILED] = "merging failed"
0047 status_map[STATE_SUMMARY_WAITING] = "waiting for APE determination"
0048 status_map[STATE_SUMMARY_DONE] = "APE determination done"
0049 status_map[STATE_SUMMARY_FAILED] = "APE determination failed"
0050 status_map[STATE_LOCAL_WAITING] = "waiting for APE saving"
0051 status_map[STATE_LOCAL_DONE] = "APE saving done"
0052 status_map[STATE_LOCAL_FAILED] = "APE saving failed"
0053 status_map[STATE_FINISHED] = "finished"
0054 status_map[STATE_INVALID_CONDITIONS] = "invalid configuration"
0055
0056 records = {}
0057 records["Alignments"] = "TrackerAlignmentRcd"
0058 records["TrackerAlignment"] = "TrackerAlignmentRcd"
0059 records["Deformations"] = "TrackerSurfaceDeformationRcd"
0060 records["TrackerSurfaceDeformations"] = "TrackerSurfaceDeformationRcd"
0061 records["SiPixelTemplateDBObject"] = "SiPixelTemplateDBObjectRcd"
0062 records["BeamSpotObjects"] = "BeamSpotObjectsRcd"
0063
0064 def rootFileValid(path):
0065     from ROOT import TFile
0066     result = True
0067     file = TFile(path)
0068     result &= file.GetSize() > 0
0069     result &= not file.TestBit(TFile.kRecovered)
0070     result &= not file.IsZombie()
0071     return result
0072
0073 if not 'MODULEPATH' in os.environ:
0074     f = open(os.environ['MODULESHOME'] + "/init/.modulespath", "r")
0075     path = []
0076     for line in f.readlines():
0077         line = re.sub("#.*$", '', line)
0078         if line != '':
0079             path.append(line)
0080     os.environ['MODULEPATH'] = ':'.join(path)
0081
0082 if not 'LOADEDMODULES' in os.environ:
0083     os.environ['LOADEDMODULES'] = ''
0084
0085 def module(*args):
0086     if type(args[0]) == type([]):
0087         args = args[0]
0088     else:
0089         args = list(args)
0090     (output, error) = subprocess.Popen(['/usr/bin/modulecmd', 'python'] + args, stdout=subprocess.PIPE).communicate()
0091     exec(output)
0092
0093 def enableCAF(switch):
0094     if switch:
0095         module('load', 'lxbatch/tzero')
0096     else:
0097         module('load', 'lxbatch/share')
0098
0099
0100
0101 def ensurePathExists(path):
0102     try:
0103         os.makedirs(path)
0104     except OSError as exception:
0105         if exception.errno != errno.EEXIST:
0106             raise
0107
0108
0109 def replaceAllRanges(string):
0110     if "[" in string and "]" in string:
0111         strings = []
0112         posS = string.find("[")
0113         posE = string.find("]")
0114         nums = string[posS+1:posE].split(",")
0115         expression = string[posS:posE+1]
0116
0117         nums = string[string.find("[")+1:string.find("]")]
0118         for interval in nums.split(","):
0119             interval = interval.strip()
0120             if "-" in interval:
0121                 lowNum = int(interval.split("-")[0])
0122                 upNum = int(interval.split("-")[1])
0123                 for i in range(lowNum, upNum+1):
0124                     newstring = string[0:posS]+str(i)+string[posE+1:]
0125                     newstring = replaceAllRanges(newstring)
0126                     strings += newstring
0127             else:
0128                 newstring = string[0:posS]+interval+string[posE+1:]
0129                 newstring = replaceAllRanges(newstring)
0130                 strings += newstring
0131         return strings
0132     else:
0133         return [string,]
0134
0135
0136 def replaceShortcuts(toScan):
0137     global shortcuts
0138     for key, value in shortcuts.items():
0139         match = re.search(key, toScan)
0140         if match and match.group(0) == toScan:
0141             return value.format(*match.groups())
0142     # no match
0143     return toScan
0144
0145 def allFilesExist(dataset):
0146     passed = True
0147     missingFiles = []
0148     for fileName in dataset.fileList:
0149         if not os.path.isfile(fileName):
0150             passed = False
0151             missingFiles.append(fileName)
0152     return passed, missingFiles
0153
0154
0155
0156 def hasValidSource(condition):
0157     if condition["connect"].startswith("frontier://FrontierProd/"):
0158         # No further checks are done in this case, even though it might
0159         # still be invalid
0160         return True
0161     if condition["connect"].startswith("sqlite_file:"):
0162         fileName = condition["connect"].split("sqlite_file:")[1]
0163         if os.path.isfile(fileName) and fileName.endswith(".db"):
0164             return True
0165     return False
0166
0167 def loadConditions(dictionary):
0168     hasAlignmentCondition = False
0169     goodConditions = True
0170     conditions = []
0171     for key, value in dictionary.items():
0172         key = key.strip()
0173         value = value.strip()
0174         if key.startswith("condition"):
0175             if len(value.split(" ")) == 2 and len(key.split(" ")) == 2:
0176                 # structure is "condition rcd:source tag"
0177                 record = key.split(" ")[1]
0178                 connect, tag = value.split(" ")
0179                 if record == "TrackerAlignmentRcd":
0180                     hasAlignmentCondition = True
0181                 conditions.append({"record":record, "connect":replaceShortcuts(connect), "tag":tag})
0182             elif len(value.split(" ")) == 1 and len(key.split(" ")) == 2:
0183                 # structure is "condition tag:source", so we have to guess rcd from the tag. might also be "condition tag1+tag2+...+tagN:source"
0184                 connect = value.strip()
0185                 tags = key.split(" ")[1]
0186                 for tag in tags.split("+"):
0187                     foundTag = False
0188                     for possibleTag, possibleRcd in records.items():
0189                         if tag.startswith(possibleTag):
0190                             conditions.append({"record":possibleRcd, "connect":replaceShortcuts(connect), "tag":tag})
0191                             if possibleRcd == "TrackerAlignmentRcd":
0192                                 hasAlignmentCondition = True
0193                             foundTag = True
0194                             break
0195                     if not foundTag:
0196                         print("Unable to infer a record corresponding to {} tag.".format(tag))
0197                         goodConditions = False
0198             else:
0199                 print("Unable to parse structure of {}:{}".format(key, value))
0200                 goodConditions = False
0201
0202     # sanity checks
0203     for condition in conditions:
0204         if not hasValidSource(condition):
0205             goodConditions = False
0206             print("'{}' is not a valid source for loading conditions.".format(condition["connect"]))
0207         if not condition["record"].endswith("Rcd"):
0208             goodConditions = False
0209             print("'{}' is not a valid record name.".format(condition["record"]))
0210     return conditions, hasAlignmentCondition, goodConditions