test/autoSubmitter/helpers.py

0001 from __future__ import print_function
0002 import re
0003 import os
0004 import subprocess
0005 import errno
0006 shortcuts = {}
0007
0008 # regex matching on key, replacement of groups on value
0009 # implement any other shortcuts that you want to use
0010 #sources
0011 shortcuts["mp([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm/alignments_MP.db"
0012 shortcuts["mp([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm{1}/alignments_MP.db"
0013 shortcuts["sm([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/sm{0}/alignments_iter{1}.db"
0014 shortcuts["um([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm/um{0}.db"
0015 shortcuts["um([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm{1}/um{0}.db"
0016 shortcuts["hp([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/hp{0}/alignments_iter{1}.db"
0017 shortcuts["prod"] = "frontier://FrontierProd/CMS_CONDITIONS"
0018
0019 # Exact numbers don't really matter, but it is important that each one has a unique
0020 # number, so that states are distinguishable
0021 STATE_NONE = -1
0022 STATE_ITERATION_START=0
0023 STATE_BJOBS_WAITING=1
0024 STATE_BJOBS_DONE=2
0025 STATE_BJOBS_FAILED=12
0026 STATE_MERGE_WAITING=3
0027 STATE_MERGE_DONE=4
0028 STATE_MERGE_FAILED=14
0029 STATE_SUMMARY_WAITING=5
0030 STATE_SUMMARY_DONE=6
0031 STATE_SUMMARY_FAILED=16
0032 STATE_LOCAL_WAITING=7
0033 STATE_LOCAL_DONE=8
0034 STATE_LOCAL_FAILED=18
0035 STATE_FINISHED=9
0036 STATE_INVALID_CONDITIONS = 101
0037
0038 status_map = {}
0039 status_map[STATE_NONE] = "none"
0040 status_map[STATE_ITERATION_START] = "starting iteration"
0041 status_map[STATE_BJOBS_WAITING] = "waiting for jobs"
0042 status_map[STATE_BJOBS_DONE] = "jobs finished"
0043 status_map[STATE_BJOBS_FAILED] = "jobs failed"
0044 status_map[STATE_MERGE_WAITING] = "waiting for merging"
0045 status_map[STATE_MERGE_DONE] = "merging done"
0046 status_map[STATE_MERGE_FAILED] = "merging failed"
0047 status_map[STATE_SUMMARY_WAITING] = "waiting for APE determination"
0048 status_map[STATE_SUMMARY_DONE] = "APE determination done"
0049 status_map[STATE_SUMMARY_FAILED] = "APE determination failed"
0050 status_map[STATE_LOCAL_WAITING] = "waiting for APE saving"
0051 status_map[STATE_LOCAL_DONE] = "APE saving done"
0052 status_map[STATE_LOCAL_FAILED] = "APE saving failed"
0053 status_map[STATE_FINISHED] = "finished"
0054 status_map[STATE_INVALID_CONDITIONS] = "invalid configuration"
0055
0056 records = {}
0057 records["Alignments"] = "TrackerAlignmentRcd"
0058 records["TrackerAlignment"] = "TrackerAlignmentRcd"
0059 records["Deformations"] = "TrackerSurfaceDeformationRcd"
0060 records["TrackerSurfaceDeformations"] = "TrackerSurfaceDeformationRcd"
0061 records["SiPixelTemplateDBObject"] = "SiPixelTemplateDBObjectRcd"
0062 records["BeamSpotObjects"] = "BeamSpotObjectsRcd"
0063
0064 def rootFileValid(path):
0065     from ROOT import TFile
0066     result = True
0067     file = TFile(path)
0068     result &= file.GetSize() > 0
0069     result &= not file.TestBit(TFile.kRecovered)
0070     result &= not file.IsZombie()
0071     return result
0072
0073 def initializeModuleLoading():
0074     if not 'MODULEPATH' in os.environ:
0075         f = open(os.environ['MODULESHOME'] + "/init/.modulespath", "r")
0076         path = []
0077         for line in f.readlines():
0078             line = re.sub("#.*$", '', line)
0079             if line != '':
0080                 path.append(line)
0081         os.environ['MODULEPATH'] = ':'.join(path)
0082
0083     if not 'LOADEDMODULES' in os.environ:
0084         os.environ['LOADEDMODULES'] = ''
0085
0086 def module(*args):
0087     if type(args[0]) == type([]):
0088         args = args[0]
0089     else:
0090         args = list(args)
0091     (output, error) = subprocess.Popen(['/usr/bin/modulecmd', 'python'] + args, stdout=subprocess.PIPE).communicate()
0092     exec(output)
0093
0094 def enableCAF(switch):
0095     if switch:
0096         module('load', 'lxbatch/tzero')
0097     else:
0098         module('load', 'lxbatch/share')
0099
0100 def ensurePathExists(path):
0101     try:
0102         os.makedirs(path)
0103     except OSError as exception:
0104         if exception.errno != errno.EEXIST:
0105             raise
0106
0107 def replaceAllRanges(string):
0108     if "[" in string and "]" in string:
0109         strings = []
0110         posS = string.find("[")
0111         posE = string.find("]")
0112         nums = string[posS+1:posE].split(",")
0113         expression = string[posS:posE+1]
0114
0115         nums = string[string.find("[")+1:string.find("]")]
0116         for interval in nums.split(","):
0117             interval = interval.strip()
0118             if "-" in interval:
0119                 lowNum = int(interval.split("-")[0])
0120                 upNum = int(interval.split("-")[1])
0121                 for i in range(lowNum, upNum+1):
0122                     newstring = string[0:posS]+str(i)+string[posE+1:]
0123                     newstring = replaceAllRanges(newstring)
0124                     strings += newstring
0125             else:
0126                 newstring = string[0:posS]+interval+string[posE+1:]
0127                 newstring = replaceAllRanges(newstring)
0128                 strings += newstring
0129         return strings
0130     else:
0131         return [string,]
0132
0133
0134 def replaceShortcuts(toScan):
0135     global shortcuts
0136     for key, value in shortcuts.items():
0137         match = re.search(key, toScan)
0138         if match and match.group(0) == toScan:
0139             return value.format(*match.groups())
0140     # no match
0141     return toScan
0142
0143 def allFilesExist(dataset):
0144     passed = True
0145     missingFiles = []
0146     for fileName in dataset.fileList:
0147         if not os.path.isfile(fileName):
0148             passed = False
0149             missingFiles.append(fileName)
0150     return passed, missingFiles
0151
0152 def hasValidSource(condition):
0153     if condition["connect"].startswith("frontier://FrontierProd/"):
0154         # No further checks are done in this case, even though it might
0155         # still be invalid
0156         return True
0157     if condition["connect"].startswith("sqlite_file:"):
0158         fileName = condition["connect"].split("sqlite_file:")[1]
0159         if os.path.isfile(fileName) and fileName.endswith(".db"):
0160             return True
0161     return False
0162
0163 def loadConditions(dictionary):
0164     goodConditions = True
0165     conditions = []
0166     for key, value in dictionary.items():
0167         key = key.strip()
0168         value = value.strip()
0169         if key.startswith("condition"):
0170             if len(value.split(" ")) == 2 and len(key.split(" ")) == 2:
0171                 # structure is "condition rcd:source tag"
0172                 record = key.split(" ")[1]
0173                 connect, tag = value.split(" ")
0174                 conditions.append({"record":record, "connect":replaceShortcuts(connect), "tag":tag})
0175             elif len(value.split(" ")) == 1 and len(key.split(" ")) == 2:
0176                 # structure is "condition tag:source", so we have to guess rcd from the tag. might also be "condition tag1+tag2+...+tagN:source"
0177                 connect = value.strip()
0178                 tags = key.split(" ")[1]
0179                 for tag in tags.split("+"):
0180                     foundTag = False
0181                     for possibleTag, possibleRcd in records.items():
0182                         if tag.startswith(possibleTag):
0183                             conditions.append({"record":possibleRcd, "connect":replaceShortcuts(connect), "tag":tag})
0184                             foundTag = True
0185                             break
0186                     if not foundTag:
0187                         print("Unable to infer a record corresponding to {} tag.".format(tag))
0188                         goodConditions = False
0189             else:
0190                 print("Unable to parse structure of {}:{}".format(key, value))
0191                 goodConditions = False
0192
0193     # sanity checks
0194     for condition in conditions:
0195         if not hasValidSource(condition):
0196             goodConditions = False
0197             print("'{}' is not a valid source for loading conditions.".format(condition["connect"]))
0198         if not condition["record"].endswith("Rcd"):
0199             goodConditions = False
0200             print("'{}' is not a valid record name.".format(condition["record"]))
0201     return conditions, goodConditions