File indexing completed on 2024-11-25 02:29:01
0001 import re
0002 import os
0003 import subprocess
0004 import errno
0005 shortcuts = {}
0006
0007
0008
0009
0010 shortcuts["mp([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm/alignments_MP.db"
0011 shortcuts["mp([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm{1}/alignments_MP.db"
0012 shortcuts["sm([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/sm{0}/alignments_iter{1}.db"
0013 shortcuts["um([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm/um{0}.db"
0014 shortcuts["um([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm{1}/um{0}.db"
0015 shortcuts["hp([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/hp{0}/alignments_iter{1}.db"
0016 shortcuts["prod"] = "frontier://FrontierProd/CMS_CONDITIONS"
0017
0018
0019
0020 STATE_NONE = -1
0021 STATE_ITERATION_START=0
0022 STATE_BJOBS_WAITING=1
0023 STATE_BJOBS_DONE=2
0024 STATE_BJOBS_FAILED=12
0025 STATE_MERGE_WAITING=3
0026 STATE_MERGE_DONE=4
0027 STATE_MERGE_FAILED=14
0028 STATE_SUMMARY_WAITING=5
0029 STATE_SUMMARY_DONE=6
0030 STATE_SUMMARY_FAILED=16
0031 STATE_LOCAL_WAITING=7
0032 STATE_LOCAL_DONE=8
0033 STATE_LOCAL_FAILED=18
0034 STATE_FINISHED=9
0035 STATE_INVALID_CONDITIONS = 101
0036
0037 status_map = {}
0038 status_map[STATE_NONE] = "none"
0039 status_map[STATE_ITERATION_START] = "starting iteration"
0040 status_map[STATE_BJOBS_WAITING] = "waiting for jobs"
0041 status_map[STATE_BJOBS_DONE] = "jobs finished"
0042 status_map[STATE_BJOBS_FAILED] = "jobs failed"
0043 status_map[STATE_MERGE_WAITING] = "waiting for merging"
0044 status_map[STATE_MERGE_DONE] = "merging done"
0045 status_map[STATE_MERGE_FAILED] = "merging failed"
0046 status_map[STATE_SUMMARY_WAITING] = "waiting for APE determination"
0047 status_map[STATE_SUMMARY_DONE] = "APE determination done"
0048 status_map[STATE_SUMMARY_FAILED] = "APE determination failed"
0049 status_map[STATE_LOCAL_WAITING] = "waiting for APE saving"
0050 status_map[STATE_LOCAL_DONE] = "APE saving done"
0051 status_map[STATE_LOCAL_FAILED] = "APE saving failed"
0052 status_map[STATE_FINISHED] = "finished"
0053 status_map[STATE_INVALID_CONDITIONS] = "invalid configuration"
0054
0055 records = {}
0056 records["Alignments"] = "TrackerAlignmentRcd"
0057 records["TrackerAlignment"] = "TrackerAlignmentRcd"
0058 records["Deformations"] = "TrackerSurfaceDeformationRcd"
0059 records["TrackerSurfaceDeformations"] = "TrackerSurfaceDeformationRcd"
0060 records["SiPixelTemplateDBObject"] = "SiPixelTemplateDBObjectRcd"
0061 records["BeamSpotObjects"] = "BeamSpotObjectsRcd"
0062
0063 def rootFileValid(path):
0064 from ROOT import TFile
0065 result = True
0066 file = TFile(path)
0067 result &= file.GetSize() > 0
0068 result &= not file.TestBit(TFile.kRecovered)
0069 result &= not file.IsZombie()
0070 return result
0071
0072 def initializeModuleLoading():
0073 if not 'MODULEPATH' in os.environ:
0074 f = open(os.environ['MODULESHOME'] + "/init/.modulespath", "r")
0075 path = []
0076 for line in f.readlines():
0077 line = re.sub("#.*$", '', line)
0078 if line != '':
0079 path.append(line)
0080 os.environ['MODULEPATH'] = ':'.join(path)
0081
0082 if not 'LOADEDMODULES' in os.environ:
0083 os.environ['LOADEDMODULES'] = ''
0084
0085 def module(*args):
0086 if type(args[0]) == type([]):
0087 args = args[0]
0088 else:
0089 args = list(args)
0090 (output, error) = subprocess.Popen(['/usr/bin/modulecmd', 'python'] + args, stdout=subprocess.PIPE).communicate()
0091 exec(output)
0092
0093 def enableCAF(switch):
0094 if switch:
0095 module('load', 'lxbatch/tzero')
0096 else:
0097 module('load', 'lxbatch/share')
0098
0099 def ensurePathExists(path):
0100 try:
0101 os.makedirs(path)
0102 except OSError as exception:
0103 if exception.errno != errno.EEXIST:
0104 raise
0105
0106 def replaceAllRanges(string):
0107 if "[" in string and "]" in string:
0108 strings = []
0109 posS = string.find("[")
0110 posE = string.find("]")
0111 nums = string[posS+1:posE].split(",")
0112 expression = string[posS:posE+1]
0113
0114 nums = string[string.find("[")+1:string.find("]")]
0115 for interval in nums.split(","):
0116 interval = interval.strip()
0117 if "-" in interval:
0118 lowNum = int(interval.split("-")[0])
0119 upNum = int(interval.split("-")[1])
0120 for i in range(lowNum, upNum+1):
0121 newstring = string[0:posS]+str(i)+string[posE+1:]
0122 newstring = replaceAllRanges(newstring)
0123 strings += newstring
0124 else:
0125 newstring = string[0:posS]+interval+string[posE+1:]
0126 newstring = replaceAllRanges(newstring)
0127 strings += newstring
0128 return strings
0129 else:
0130 return [string,]
0131
0132
0133 def replaceShortcuts(toScan):
0134 global shortcuts
0135 for key, value in shortcuts.items():
0136 match = re.search(key, toScan)
0137 if match and match.group(0) == toScan:
0138 return value.format(*match.groups())
0139
0140 return toScan
0141
0142 def allFilesExist(dataset):
0143 passed = True
0144 missingFiles = []
0145 for fileName in dataset.fileList:
0146 if not os.path.isfile(fileName):
0147 passed = False
0148 missingFiles.append(fileName)
0149 return passed, missingFiles
0150
0151 def hasValidSource(condition):
0152 if condition["connect"].startswith("frontier://FrontierProd/"):
0153
0154
0155 return True
0156 if condition["connect"].startswith("sqlite_file:"):
0157 fileName = condition["connect"].split("sqlite_file:")[1]
0158 if os.path.isfile(fileName) and fileName.endswith(".db"):
0159 return True
0160 return False
0161
0162 def loadConditions(dictionary):
0163 goodConditions = True
0164 conditions = []
0165 for key, value in dictionary.items():
0166 key = key.strip()
0167 value = value.strip()
0168 if key.startswith("condition"):
0169 if len(value.split(" ")) == 2 and len(key.split(" ")) == 2:
0170
0171 record = key.split(" ")[1]
0172 connect, tag = value.split(" ")
0173 conditions.append({"record":record, "connect":replaceShortcuts(connect), "tag":tag})
0174 elif len(value.split(" ")) == 1 and len(key.split(" ")) == 2:
0175
0176 connect = value.strip()
0177 tags = key.split(" ")[1]
0178 for tag in tags.split("+"):
0179 foundTag = False
0180 for possibleTag, possibleRcd in records.items():
0181 if tag.startswith(possibleTag):
0182 conditions.append({"record":possibleRcd, "connect":replaceShortcuts(connect), "tag":tag})
0183 foundTag = True
0184 break
0185 if not foundTag:
0186 print("Unable to infer a record corresponding to {} tag.".format(tag))
0187 goodConditions = False
0188 else:
0189 print("Unable to parse structure of {}:{}".format(key, value))
0190 goodConditions = False
0191
0192
0193 for condition in conditions:
0194 if not hasValidSource(condition):
0195 goodConditions = False
0196 print("'{}' is not a valid source for loading conditions.".format(condition["connect"]))
0197 if not condition["record"].endswith("Rcd"):
0198 goodConditions = False
0199 print("'{}' is not a valid record name.".format(condition["record"]))
0200 return conditions, goodConditions