Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-25 02:29:01

0001 import re
0002 import os
0003 import subprocess
0004 import errno
0005 shortcuts = {}
0006 
0007 # regex matching on key, replacement of groups on value
0008 # implement any other shortcuts that you want to use
0009 #sources
0010 shortcuts["mp([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm/alignments_MP.db"
0011 shortcuts["mp([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm{1}/alignments_MP.db"
0012 shortcuts["sm([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/sm{0}/alignments_iter{1}.db"
0013 shortcuts["um([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm/um{0}.db"
0014 shortcuts["um([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm{1}/um{0}.db"
0015 shortcuts["hp([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/hp{0}/alignments_iter{1}.db"
0016 shortcuts["prod"] = "frontier://FrontierProd/CMS_CONDITIONS"
0017 
0018 # Exact numbers don't really matter, but it is important that each one has a unique
0019 # number, so that states are distinguishable
0020 STATE_NONE = -1
0021 STATE_ITERATION_START=0
0022 STATE_BJOBS_WAITING=1
0023 STATE_BJOBS_DONE=2
0024 STATE_BJOBS_FAILED=12
0025 STATE_MERGE_WAITING=3
0026 STATE_MERGE_DONE=4
0027 STATE_MERGE_FAILED=14
0028 STATE_SUMMARY_WAITING=5
0029 STATE_SUMMARY_DONE=6
0030 STATE_SUMMARY_FAILED=16
0031 STATE_LOCAL_WAITING=7
0032 STATE_LOCAL_DONE=8
0033 STATE_LOCAL_FAILED=18
0034 STATE_FINISHED=9
0035 STATE_INVALID_CONDITIONS = 101
0036 
0037 status_map = {}
0038 status_map[STATE_NONE] = "none"
0039 status_map[STATE_ITERATION_START] = "starting iteration"
0040 status_map[STATE_BJOBS_WAITING] = "waiting for jobs"
0041 status_map[STATE_BJOBS_DONE] = "jobs finished"
0042 status_map[STATE_BJOBS_FAILED] = "jobs failed"
0043 status_map[STATE_MERGE_WAITING] = "waiting for merging"
0044 status_map[STATE_MERGE_DONE] = "merging done"
0045 status_map[STATE_MERGE_FAILED] = "merging failed"
0046 status_map[STATE_SUMMARY_WAITING] = "waiting for APE determination"
0047 status_map[STATE_SUMMARY_DONE] = "APE determination done"
0048 status_map[STATE_SUMMARY_FAILED] = "APE determination failed"
0049 status_map[STATE_LOCAL_WAITING] = "waiting for APE saving"
0050 status_map[STATE_LOCAL_DONE] = "APE saving done"
0051 status_map[STATE_LOCAL_FAILED] = "APE saving failed"
0052 status_map[STATE_FINISHED] = "finished"
0053 status_map[STATE_INVALID_CONDITIONS] = "invalid configuration"
0054 
0055 records = {}
0056 records["Alignments"] = "TrackerAlignmentRcd"
0057 records["TrackerAlignment"] = "TrackerAlignmentRcd"
0058 records["Deformations"] = "TrackerSurfaceDeformationRcd"
0059 records["TrackerSurfaceDeformations"] = "TrackerSurfaceDeformationRcd"
0060 records["SiPixelTemplateDBObject"] = "SiPixelTemplateDBObjectRcd"
0061 records["BeamSpotObjects"] = "BeamSpotObjectsRcd"
0062 
0063 def rootFileValid(path):
0064     from ROOT import TFile
0065     result = True
0066     file = TFile(path)
0067     result &= file.GetSize() > 0
0068     result &= not file.TestBit(TFile.kRecovered)
0069     result &= not file.IsZombie()
0070     return result
0071 
0072 def initializeModuleLoading():
0073     if not 'MODULEPATH' in os.environ:
0074         f = open(os.environ['MODULESHOME'] + "/init/.modulespath", "r")
0075         path = []
0076         for line in f.readlines():
0077             line = re.sub("#.*$", '', line)
0078             if line != '':
0079                 path.append(line)
0080         os.environ['MODULEPATH'] = ':'.join(path)
0081 
0082     if not 'LOADEDMODULES' in os.environ:
0083         os.environ['LOADEDMODULES'] = ''
0084     
0085 def module(*args):
0086     if type(args[0]) == type([]):
0087         args = args[0]
0088     else:
0089         args = list(args)
0090     (output, error) = subprocess.Popen(['/usr/bin/modulecmd', 'python'] + args, stdout=subprocess.PIPE).communicate()
0091     exec(output)
0092 
0093 def enableCAF(switch):
0094     if switch:
0095         module('load', 'lxbatch/tzero')
0096     else:
0097         module('load', 'lxbatch/share')
0098 
0099 def ensurePathExists(path):
0100     try:
0101         os.makedirs(path)
0102     except OSError as exception:
0103         if exception.errno != errno.EEXIST:
0104             raise
0105 
0106 def replaceAllRanges(string):
0107     if "[" in string and "]" in string:
0108         strings = []
0109         posS = string.find("[")
0110         posE = string.find("]")
0111         nums = string[posS+1:posE].split(",")
0112         expression = string[posS:posE+1]
0113 
0114         nums = string[string.find("[")+1:string.find("]")]
0115         for interval in nums.split(","):
0116             interval = interval.strip()
0117             if "-" in interval:
0118                 lowNum = int(interval.split("-")[0])
0119                 upNum = int(interval.split("-")[1])
0120                 for i in range(lowNum, upNum+1):
0121                     newstring = string[0:posS]+str(i)+string[posE+1:]
0122                     newstring = replaceAllRanges(newstring)
0123                     strings += newstring
0124             else:
0125                 newstring = string[0:posS]+interval+string[posE+1:]
0126                 newstring = replaceAllRanges(newstring)
0127                 strings += newstring
0128         return strings
0129     else:
0130         return [string,]
0131 
0132 
0133 def replaceShortcuts(toScan):
0134     global shortcuts
0135     for key, value in shortcuts.items():
0136         match = re.search(key, toScan)
0137         if match and match.group(0) == toScan:
0138             return value.format(*match.groups())
0139     # no match
0140     return toScan
0141 
0142 def allFilesExist(dataset):
0143     passed = True
0144     missingFiles = []
0145     for fileName in dataset.fileList:
0146         if not os.path.isfile(fileName):
0147             passed = False
0148             missingFiles.append(fileName)
0149     return passed, missingFiles
0150 
0151 def hasValidSource(condition):
0152     if condition["connect"].startswith("frontier://FrontierProd/"):
0153         # No further checks are done in this case, even though it might
0154         # still be invalid
0155         return True
0156     if condition["connect"].startswith("sqlite_file:"):
0157         fileName = condition["connect"].split("sqlite_file:")[1]
0158         if os.path.isfile(fileName) and fileName.endswith(".db"):
0159             return True
0160     return False
0161 
0162 def loadConditions(dictionary):
0163     goodConditions = True
0164     conditions = []
0165     for key, value in dictionary.items():
0166         key = key.strip()
0167         value = value.strip()
0168         if key.startswith("condition"):
0169             if len(value.split(" ")) == 2 and len(key.split(" ")) == 2: 
0170                 # structure is "condition rcd:source tag"
0171                 record = key.split(" ")[1]
0172                 connect, tag = value.split(" ")
0173                 conditions.append({"record":record, "connect":replaceShortcuts(connect), "tag":tag})
0174             elif len(value.split(" ")) == 1 and len(key.split(" ")) == 2:
0175                 # structure is "condition tag:source", so we have to guess rcd from the tag. might also be "condition tag1+tag2+...+tagN:source"
0176                 connect = value.strip()
0177                 tags = key.split(" ")[1]
0178                 for tag in tags.split("+"):
0179                     foundTag = False
0180                     for possibleTag, possibleRcd in records.items():
0181                         if tag.startswith(possibleTag):
0182                             conditions.append({"record":possibleRcd, "connect":replaceShortcuts(connect), "tag":tag})
0183                             foundTag = True
0184                             break
0185                     if not foundTag:
0186                         print("Unable to infer a record corresponding to {} tag.".format(tag))
0187                         goodConditions = False
0188             else:
0189                 print("Unable to parse structure of {}:{}".format(key, value))
0190                 goodConditions = False
0191     
0192     # sanity checks
0193     for condition in conditions:
0194         if not hasValidSource(condition):
0195             goodConditions = False
0196             print("'{}' is not a valid source for loading conditions.".format(condition["connect"]))
0197         if not condition["record"].endswith("Rcd"):
0198             goodConditions = False
0199             print("'{}' is not a valid record name.".format(condition["record"]))
0200     return conditions, goodConditions