File indexing completed on 2023-03-17 11:23:14
0001 from __future__ import print_function
0002 from builtins import range
0003 import math, re, optparse, subprocess, os, sys, time, datetime
0004 from BeamSpotObj import BeamSpot
0005 from IOVObj import IOV
0006
0007 lockFile = ".lock"
0008
0009
0010 def timeoutManager(type,timeout=-1,fileName=".timeout"):
0011 if timeout == 0:
0012 return 1
0013 timeFormat = "%a,%Y/%m/%d,%H:%M:%S"
0014 currentTime = time.gmtime()
0015 timeoutLine = type + ' ' + time.strftime(timeFormat, currentTime) + '\n'
0016 isTimeout = False
0017 alreadyThere = False
0018 timeoutType = -1;
0019 fileExist = os.path.isfile(fileName)
0020 text = ''
0021 fields = []
0022 reset = False
0023 if timeout == -1:
0024 reset = True
0025 if fileExist:
0026 file = open(fileName)
0027 for line in file:
0028 text += line
0029 fields = line.strip('\n').split(' ')
0030 if fields[0] == type:
0031 alreadyThere = True
0032 if reset:
0033 text = text.replace(line,'')
0034 continue
0035
0036 fileTime = time.strptime(fields[1],timeFormat)
0037 myTime = time.mktime(fileTime)
0038 referenceTime = time.mktime(time.gmtime())
0039 daylight = 0
0040 if currentTime.tm_isdst == 0:
0041 daylight = 3600
0042 elapsedTime = referenceTime-myTime-daylight
0043 if elapsedTime > timeout:
0044 isTimeout = True
0045 timeoutType = 1
0046 print("Timeout! " + str(elapsedTime) + " seconds passed since the " + type + " timeout was set and you can't tolerate more than " + str(timeout) + " seconds!")
0047 else:
0048 timeoutType = 0
0049 print("Timeout of type " + type + " already exist and was generated " + str(elapsedTime) + " seconds ago at " + fields[1])
0050
0051 file.close()
0052
0053 if not fileExist or not alreadyThere and not reset:
0054 timeoutType = -1
0055 text += timeoutLine
0056
0057 if not fileExist or not alreadyThere or isTimeout or (reset and alreadyThere):
0058 if fileExist:
0059 subprocess.getstatusoutput("rm -rf " + fileName)
0060 file = open(fileName,'w')
0061 file.write(text)
0062 file.close()
0063
0064 return timeoutType
0065
0066
0067
0068 def setLockName(name):
0069 global lockFile
0070 lockFile = name
0071
0072
0073 def checkLock():
0074 global lockFile
0075 if os.path.isfile(lockFile):
0076 return True
0077 else:
0078 return False
0079
0080
0081 def lock():
0082 global lockFile
0083 subprocess.getstatusoutput( "touch " + lockFile)
0084
0085
0086 def rmLock():
0087 global lockFile
0088 if checkLock():
0089 subprocess.getstatusoutput( "rm " + lockFile)
0090
0091
0092 def exit(msg=""):
0093 rmLock()
0094 raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
0095
0096
0097 def isnan(num):
0098 fnum = float(num)
0099 return fnum != fnum
0100
0101
0102
0103
0104 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
0105
0106
0107 def parse(docstring, arglist=None):
0108 global optionstring
0109 global tagType
0110 optionstring = docstring
0111 match = USAGE.search(optionstring)
0112 if not match: raise ParsingError("Cannot find the option string")
0113 optlines = match.group(1).splitlines()
0114 try:
0115 p = optparse.OptionParser(optlines[0])
0116 for line in optlines[1:]:
0117 opt, help=line.split(':')[:2]
0118 short,long=opt.split(',')[:2]
0119 if '=' in opt:
0120 action='store'
0121 long=long.split('=')[0]
0122 else:
0123 action='store_true'
0124 p.add_option(short.strip(),long.strip(),
0125 action = action, help = help.strip())
0126 except (IndexError,ValueError):
0127 raise ParsingError("Cannot parse the option string correctly")
0128 return p.parse_args(arglist)
0129
0130
0131 def nonzero(self):
0132 "True if options were given"
0133 for v in self.__dict__.values():
0134 if v is not None: return True
0135 return False
0136
0137
0138 optparse.Values.__nonzero__ = nonzero
0139
0140
0141
0142
0143
0144 class ParsingError(Exception): pass
0145
0146
0147
0148
0149
0150 def sendEmail(mailList,error):
0151 print("Sending email to " + mailList + " with body: " + error)
0152 list = mailList.split(',')
0153 for email in list:
0154 p = os.popen("mail -s \"Automatic workflow error\" " + email ,"w")
0155 p.write(error)
0156 status = p.close()
0157
0158
0159 def dirExists(dir):
0160 if dir.find("castor") != -1:
0161 lsCommand = "nsls " + dir
0162 output = subprocess.getstatusoutput( lsCommand )
0163 return not output[0]
0164 else:
0165 return os.path.exists(dir)
0166
0167
0168 def ls(dir,filter=""):
0169 lsCommand = ''
0170 listOfFiles = []
0171 if dir.find('castor') != -1:
0172 lsCommand = 'ns'
0173 elif not os.path.exists(dir):
0174 print("ERROR: File or directory " + dir + " doesn't exist")
0175 return listOfFiles
0176
0177 aCommand = lsCommand + 'ls '+ dir
0178
0179 if filter != "":
0180 aCommand += " | grep " + filter
0181
0182 tmpStatus = subprocess.getstatusoutput( aCommand )
0183 listOfFiles = tmpStatus[1].split('\n')
0184 if len(listOfFiles) == 1:
0185 if listOfFiles[0].find('No such file or directory') != -1:
0186 exit("ERROR: File or directory " + dir + " doesn't exist")
0187
0188 return listOfFiles
0189
0190
0191 def cp(fromDir,toDir,listOfFiles,overwrite=False,smallList=False):
0192 cpCommand = ''
0193 copiedFiles = []
0194 if fromDir.find('castor') != -1 or toDir.find('castor') != -1 :
0195 cpCommand = 'rf'
0196 elif fromDir.find('resilient') != -1:
0197 cpCommand = 'dc'
0198 if fromDir[len(fromDir)-1] != '/':
0199 fromDir += '/'
0200
0201 if toDir[len(toDir)-1] != '/':
0202 toDir += '/'
0203
0204 for file in listOfFiles:
0205 if os.path.isfile(toDir+file):
0206 if overwrite:
0207 print("File " + file + " already exists in destination directory. We will overwrite it.")
0208 else:
0209 print("File " + file + " already exists in destination directory. We will Keep original file.")
0210 if not smallList:
0211 copiedFiles.append(file)
0212 continue
0213
0214 aCommand = cpCommand + 'cp '+ fromDir + file + " " + toDir
0215 print(" >> " + aCommand)
0216 tmpStatus = subprocess.getstatusoutput( aCommand )
0217 if tmpStatus[0] == 0:
0218 copiedFiles.append(file)
0219 else:
0220 print("[cp()]\tERROR: Can't copy file " + file)
0221 return copiedFiles
0222
0223
0224
0225
0226
0227
0228
0229 def pack(high,low):
0230 """pack high,low 32bit unsigned int to one unsigned 64bit long long
0231 Note:the print value of result number may appear signed, if the sign bit is used.
0232 """
0233 h=high<<32
0234 return (h|low)
0235
0236
0237 def unpack(i):
0238 """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
0239 """
0240 high=i>>32
0241 low=i&0xFFFFFFFF
0242 return(high,low)
0243
0244
0245 def unpackLumiid(i):
0246 """unpack 64bit lumiid to dictionary {'run','lumisection'}
0247 """
0248 j=unpack(i)
0249 return {'run':j[0],'lumisection':j[1]}
0250
0251
0252
0253
0254
0255 def cmp_list_run(a,b):
0256 if int(a.IOVfirst) < int(b.IOVfirst): return -1
0257 if int(a.IOVfirst) == int(b.IOVfirst): return 0
0258 if int(a.IOVfirst) > int(b.IOVfirst): return 1
0259
0260
0261 def cmp_list_lumi(a,b):
0262 if int(a.Run) < int(b.Run): return -1
0263 if int(a.Run) == int(b.Run):
0264 if int(a.IOVfirst) < int(b.IOVfirst): return -1
0265 if int(a.IOVfirst) == int(b.IOVfirst): return 0
0266 if int(a.IOVfirst) > int(b.IOVfirst): return 1
0267 if int(a.Run) > int(b.Run) : return 1
0268
0269
0270 def weight(x1, x1err,x2,x2err):
0271
0272 x1 = float(x1)
0273 x1err = float(x1err)
0274 x2 = float(x2)
0275 x2err = float(x2err)
0276 tmperr = 0.
0277 if x2err < 1e-6 :
0278 x2err = 1e-6
0279 if x1err < 1e-6:
0280 x1 = x2/(x2err * x2err)
0281 tmperr = 1/(x2err*x2err)
0282 else:
0283 x1 = x1/(x1err*x1err) + x2/(x2err * x2err)
0284 tmperr = 1/(x1err*x1err) + 1/(x2err*x2err)
0285 x1 = x1/tmperr
0286 x1err = 1/tmperr
0287 x1err = math.sqrt(x1err)
0288 return (str(x1), str(x1err))
0289
0290
0291 def dump( beam, file):
0292 end = "\n"
0293 file.write("Runnumber "+beam.Run+end)
0294 file.write("BeginTimeOfFit "+str(beam.IOVBeginTime)+end)
0295 file.write("EndTimeOfFit "+str(beam.IOVEndTime)+end)
0296 file.write("LumiRange "+str(beam.IOVfirst)+" - "+str(beam.IOVlast)+end)
0297 dumpValues(beam, file)
0298
0299
0300 def dumpValues( beam, file):
0301 end = "\n"
0302 file.write("Type "+str(beam.Type)+end)
0303 file.write("X0 "+str(beam.X)+end)
0304 file.write("Y0 "+str(beam.Y)+end)
0305 file.write("Z0 "+str(beam.Z)+end)
0306 file.write("sigmaZ0 "+str(beam.sigmaZ)+end)
0307 file.write("dxdz "+str(beam.dxdz)+end)
0308 file.write("dydz "+str(beam.dydz)+end)
0309 file.write("BeamWidthX "+beam.beamWidthX+end)
0310 file.write("BeamWidthY "+beam.beamWidthY+end)
0311 file.write("Cov(0,j) "+str(math.pow(float(beam.Xerr),2))+" 0 0 0 0 0 0" +end)
0312 file.write("Cov(1,j) 0 "+str(math.pow(float(beam.Yerr),2))+" 0 0 0 0 0" +end)
0313 file.write("Cov(2,j) 0 0 "+str(math.pow(float(beam.Zerr),2))+" 0 0 0 0" +end)
0314 file.write("Cov(3,j) 0 0 0 "+str(math.pow(float(beam.sigmaZerr),2))+" 0 0 0" +end)
0315 file.write("Cov(4,j) 0 0 0 0 "+str(math.pow(float(beam.dxdzerr),2))+" 0 0" +end)
0316 file.write("Cov(5,j) 0 0 0 0 0 "+str(math.pow(float(beam.dydzerr),2))+" 0" +end)
0317 file.write("Cov(6,j) 0 0 0 0 0 0 "+str(math.pow(float(beam.beamWidthXerr),2)) +end)
0318 file.write("EmittanceX 0"+end)
0319 file.write("EmittanceY 0"+end)
0320 file.write("BetaStar 0"+end)
0321
0322
0323 def delta(x,xerr,nextx,nextxerr):
0324
0325 return ( float(x) - float(nextx), math.sqrt(math.pow(float(xerr),2) + math.pow(float(nextxerr),2)) )
0326
0327 def deltaSig( x ):
0328 return math.fabs(x[0])/x[1]
0329
0330
0331 def readBeamSpotFile(fileName,listbeam=[],IOVbase="runbase", firstRun='1',lastRun='4999999999'):
0332 tmpbeam = BeamSpot()
0333 tmpbeamsize = 0
0334
0335
0336
0337 if IOVbase == "lumibase" and firstRun=='1' and lastRun=='4999999999' :
0338 firstRun = "1:1"
0339 lastRun = "4999999999:4999999999"
0340
0341 inputfiletype = 0
0342
0343
0344
0345
0346 maplist = {}
0347 hasBX = False
0348
0349 tmpfile = open(fileName)
0350 atmpline = tmpfile.readline()
0351 if atmpline.find('Runnumber') != -1:
0352 inputfiletype = 1
0353 if len(atmpline.split()) > 2:
0354 hasBX = True
0355 print(" Input data has been calculated as function of BUNCH CROSSINGS.")
0356 tmpfile.seek(0)
0357
0358
0359 if inputfiletype ==1:
0360
0361 tmpBX = 0
0362 for line in tmpfile:
0363
0364 if line.find('Type') != -1:
0365 tmpbeam.Type = int(line.split()[1])
0366 tmpbeamsize += 1
0367 if line.find('X0') != -1:
0368 tmpbeam.X = line.split()[1]
0369
0370 tmpbeamsize += 1
0371
0372 if line.find('Y0') != -1:
0373 tmpbeam.Y = line.split()[1]
0374
0375 tmpbeamsize += 1
0376
0377 if line.find('Z0') != -1 and line.find('sigmaZ0') == -1:
0378 tmpbeam.Z = line.split()[1]
0379
0380 tmpbeamsize += 1
0381 if line.find('sigmaZ0') !=-1:
0382 tmpbeam.sigmaZ = line.split()[1]
0383
0384 tmpbeamsize += 1
0385 if line.find('dxdz') != -1:
0386 tmpbeam.dxdz = line.split()[1]
0387
0388 tmpbeamsize += 1
0389 if line.find('dydz') != -1:
0390 tmpbeam.dydz = line.split()[1]
0391
0392 tmpbeamsize += 1
0393 if line.find('BeamWidthX') != -1:
0394 tmpbeam.beamWidthX = line.split()[1]
0395
0396 tmpbeamsize += 1
0397 if line.find('BeamWidthY') != -1:
0398 tmpbeam.beamWidthY = line.split()[1]
0399
0400 tmpbeamsize += 1
0401 if line.find('Cov(0,j)') != -1:
0402 tmpbeam.Xerr = str(math.sqrt( float( line.split()[1] ) ) )
0403 tmpbeamsize += 1
0404 if line.find('Cov(1,j)') != -1:
0405 tmpbeam.Yerr = str(math.sqrt( float( line.split()[2] ) ) )
0406 tmpbeamsize += 1
0407 if line.find('Cov(2,j)') != -1:
0408 tmpbeam.Zerr = str(math.sqrt( float( line.split()[3] ) ) )
0409 tmpbeamsize += 1
0410 if line.find('Cov(3,j)') != -1:
0411 tmpbeam.sigmaZerr = str(math.sqrt( float( line.split()[4] ) ) )
0412 tmpbeamsize += 1
0413 if line.find('Cov(4,j)') != -1:
0414 tmpbeam.dxdzerr = str(math.sqrt( float( line.split()[5] ) ) )
0415 tmpbeamsize += 1
0416 if line.find('Cov(5,j)') != -1:
0417 tmpbeam.dydzerr = str(math.sqrt( float( line.split()[6] ) ) )
0418 tmpbeamsize += 1
0419 if line.find('Cov(6,j)') != -1:
0420 tmpbeam.beamWidthXerr = str(math.sqrt( float( line.split()[7] ) ) )
0421 tmpbeam.beamWidthYerr = tmpbeam.beamWidthXerr
0422 tmpbeamsize += 1
0423 if line.find('LumiRange') != -1:
0424 if IOVbase=="lumibase":
0425 tmpbeam.IOVfirst = line.split()[1]
0426 tmpbeam.IOVlast = line.split()[3]
0427 tmpbeamsize += 1
0428 if line.find('Runnumber') != -1:
0429 tmpbeam.Run = line.split()[1]
0430 if IOVbase == "runbase":
0431 tmpbeam.IOVfirst = line.split()[1]
0432 tmpbeam.IOVlast = line.split()[1]
0433 if hasBX:
0434 tmpBX = line.split()[3]
0435 tmpbeamsize += 1
0436 if line.find('BeginTimeOfFit') != -1:
0437 tmpbeam.IOVBeginTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
0438 if IOVbase =="timebase":
0439 tmpbeam.IOVfirst = time.mktime( time.strptime(line.split()[1] + " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
0440 tmpbeamsize += 1
0441 if line.find('EndTimeOfFit') != -1:
0442 tmpbeam.IOVEndTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
0443 if IOVbase =="timebase":
0444 tmpbeam.IOVlast = time.mktime( time.strptime(line.split()[1] + " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
0445 tmpbeamsize += 1
0446 if tmpbeamsize == 20:
0447 if IOVbase=="lumibase":
0448 tmprunfirst = int(firstRun.split(":")[0])
0449 tmprunlast = int(lastRun.split(":")[0])
0450 tmplumifirst = int(firstRun.split(":")[1])
0451 tmplumilast = int(lastRun.split(":")[1])
0452 acceptiov1 = acceptiov2 = False
0453
0454 if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
0455 if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
0456 acceptiov1 = acceptiov2 = True
0457
0458 elif int(tmpbeam.Run) == tmprunfirst:
0459 if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
0460 elif int(tmpbeam.Run) == tmprunlast:
0461 if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
0462 elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
0463 acceptiov1 = acceptiov2 = True
0464
0465 if acceptiov1 and acceptiov2:
0466 if tmpbeam.Type != 2:
0467 print("invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0468 elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0469 print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0470 elif hasBX:
0471 if (tmpBX in maplist) == False:
0472 maplist[tmpBX] = [tmpbeam]
0473 else:
0474 maplist[tmpBX].append(tmpbeam)
0475 else:
0476 listbeam.append(tmpbeam)
0477
0478 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
0479 if tmpbeam.Type != 2:
0480 print("invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0481 elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0482 print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0483 else:
0484 listbeam.append(tmpbeam)
0485
0486 tmpbeamsize = 0
0487 tmpbeam = BeamSpot()
0488 tmpBX = 0
0489 else:
0490
0491 for line in tmpfile:
0492
0493 if line.find('X0') != -1:
0494 tmpbeam.X = line.split()[2]
0495 tmpbeam.Xerr = line.split()[4]
0496 tmpbeamsize += 1
0497
0498 if line.find('Y0') != -1:
0499 tmpbeam.Y = line.split()[2]
0500 tmpbeam.Yerr = line.split()[4]
0501 tmpbeamsize += 1
0502
0503 if line.find('Z0') != -1 and line.find('Sigma Z0') == -1:
0504 tmpbeam.Z = line.split()[2]
0505 tmpbeam.Zerr = line.split()[4]
0506 tmpbeamsize += 1
0507
0508 if line.find('Sigma Z0') !=-1:
0509 tmpbeam.sigmaZ = line.split()[3]
0510 tmpbeam.sigmaZerr = line.split()[5]
0511 tmpbeamsize += 1
0512 if line.find('dxdz') != -1:
0513 tmpbeam.dxdz = line.split()[2]
0514 tmpbeam.dxdzerr = line.split()[4]
0515 tmpbeamsize += 1
0516 if line.find('dydz') != -1:
0517 tmpbeam.dydz = line.split()[2]
0518 tmpbeam.dydzerr = line.split()[4]
0519 tmpbeamsize += 1
0520 if line.find('Beam Width X') != -1:
0521 tmpbeam.beamWidthX = line.split()[4]
0522 tmpbeam.beamWidthXerr = line.split()[6]
0523 tmpbeamsize += 1
0524 if line.find('Beam Width Y') != -1:
0525 tmpbeam.beamWidthY = line.split()[4]
0526 tmpbeam.beamWidthYerr = line.split()[6]
0527 tmpbeamsize += 1
0528
0529 if line.find('for runs') != -1:
0530
0531 tmpbeam.Run = line.split()[2]
0532 if IOVbase == "runbase":
0533 tmpbeam.IOVfirst = line.split()[2]
0534 tmpbeam.IOVlast = line.split()[4]
0535 tmpbeamsize += 1
0536 if line.find('LumiSection') != -1:
0537 if IOVbase=="lumibase":
0538 tmpbeam.IOVfirst = line.split()[10]
0539 tmpbeam.IOVlast = line.split()[10]
0540 tmpbeamsize += 1
0541 if tmpbeamsize == 10:
0542
0543 if IOVbase=="lumibase":
0544 tmprunfirst = int(firstRun.split(":")[0])
0545 tmprunlast = int(lastRun.split(":")[0])
0546 tmplumifirst = int(firstRun.split(":")[1])
0547 tmplumilast = int(lastRun.split(":")[1])
0548 acceptiov1 = acceptiov2 = False
0549
0550 if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
0551 if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
0552 acceptiov1 = acceptiov2 = True
0553
0554 elif int(tmpbeam.Run) == tmprunfirst:
0555 if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
0556 elif int(tmpbeam.Run) == tmprunlast:
0557 if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
0558 elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
0559 acceptiov1 = acceptiov2 = True
0560
0561 if acceptiov1 and acceptiov2:
0562 if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0563 print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0564 elif hasBX:
0565 if (tmpBX in maplist) == False:
0566 maplist[tmpBX] = [tmpbeam]
0567 else:
0568 maplist[tmpBX].append(tmpbeam)
0569 else:
0570 listbeam.append(tmpbeam)
0571
0572 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
0573 if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0574 print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0575 else:
0576 listbeam.append(tmpbeam)
0577
0578 tmpbeamsize = 0
0579 tmpbeam = BeamSpot()
0580 tmpBX = 0
0581
0582 tmpfile.close()
0583 print(" got total number of IOVs = " + str(len(listbeam)) + " from file " + fileName)
0584
0585 if hasBX:
0586 return maplist
0587 else:
0588 return listbeam
0589
0590
0591
0592 def sortAndCleanBeamList(listbeam=[],IOVbase="lumibase"):
0593
0594 if IOVbase == "lumibase":
0595 listbeam.sort( cmp = cmp_list_lumi )
0596 else:
0597 listbeam.sort( cmp = cmp_list_run )
0598
0599
0600 tmpremovelist = []
0601 for ii in range(0,len(listbeam)):
0602 ibeam = listbeam[ii]
0603 datax = ibeam.IOVfirst
0604
0605 if datax == '0' and IOVbase =="runbase":
0606 print(" iov = 0? skip this IOV = "+ str(ibeam.IOVfirst) + " to " + str(ibeam.IOVlast))
0607 tmpremovelist.append(ibeam)
0608
0609 if ii < len(listbeam) -1:
0610
0611 if IOVbase =="lumibase":
0612 if ibeam.Run == listbeam[ii+1].Run and ibeam.IOVfirst == listbeam[ii+1].IOVfirst:
0613 print(" duplicate IOV = "+datax+", keep only last duplicate entry")
0614 tmpremovelist.append(ibeam)
0615 elif datax == listbeam[ii+1].IOVfirst:
0616 print(" duplicate IOV = "+datax+", keep only last duplicate entry")
0617 tmpremovelist.append(ibeam)
0618
0619 for itmp in tmpremovelist:
0620 listbeam.remove(itmp)
0621
0622
0623
0624 def createWeightedPayloads(fileName,listbeam=[],weighted=True):
0625 newlistbeam = []
0626 tmpbeam = BeamSpot()
0627 docreate = True
0628 countlumi = 0
0629 tmprun = ""
0630 maxNlumis = 60
0631 if weighted:
0632 maxNlumis = 999999999
0633 for ii in range(0,len(listbeam)):
0634 ibeam = listbeam[ii]
0635 inextbeam = BeamSpot()
0636 iNNbeam = BeamSpot()
0637 if docreate:
0638 tmpbeam.IOVfirst = ibeam.IOVfirst
0639 tmpbeam.IOVBeginTime = ibeam.IOVBeginTime
0640 tmpbeam.Run = ibeam.Run
0641 tmpbeam.Type = 2
0642 docheck = False
0643 docreate = False
0644
0645
0646
0647 if ii < len(listbeam) - 1:
0648 inextbeam = listbeam[ii+1]
0649 docheck = True
0650 if ii < len(listbeam) -2:
0651 iNNbeam = listbeam[ii+2]
0652 else:
0653 print("close payload because end of data has been reached. Run "+ibeam.Run)
0654 docreate = True
0655
0656 if ibeam.Run != inextbeam.Run:
0657 print("close payload because end of run "+ibeam.Run)
0658 docreate = True
0659
0660 if countlumi == maxNlumis -1:
0661 print("close payload because maximum lumi sections accumulated within run "+ibeam.Run)
0662 docreate = True
0663 countlumi = 0
0664
0665 (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
0666 (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
0667 (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
0668 (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
0669 (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
0670 (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
0671
0672 (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
0673 (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
0674
0675 if weighted:
0676 docheck = False
0677
0678
0679 if docheck:
0680
0681
0682 min_limit = 0.0025
0683
0684
0685 limit = float(ibeam.beamWidthX)/2.
0686 if limit < min_limit: limit = min_limit
0687
0688
0689 adelta1 = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr)
0690 adelta2 = (0.,1.e9)
0691 adelta1dxdz = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
0692 adelta2dxdz = (0.,1.e9)
0693 adelta1dydz = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
0694 adelta2dydz = (0.,1.e9)
0695 adelta1widthx = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
0696 adelta2widthx = (0.,1.e9)
0697 adelta1widthy = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
0698 adelta2widthy = (0.,1.e9)
0699 adelta1z0 = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr)
0700 adelta1sigmaZ = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr)
0701
0702 if iNNbeam.Type != -1:
0703 adelta2 = delta(inextbeam.X, inextbeam.Xerr, iNNbeam.X, iNNbeam.Xerr)
0704 adelta2dxdz = delta(inextbeam.dxdz, inextbeam.dxdzerr, iNNbeam.dxdz, iNNbeam.dxdzerr)
0705 adelta2dydz = delta(inextbeam.dydz, inextbeam.dydzerr, iNNbeam.dydz, iNNbeam.dydzerr)
0706 adelta2widthx = delta(inextbeam.beamWidthX, inextbeam.beamWidthXerr, iNNbeam.beamWidthX, iNNbeam.beamWidthXerr)
0707 adelta2widthy = delta(inextbeam.beamWidthY, inextbeam.beamWidthYerr, iNNbeam.beamWidthY, iNNbeam.beamWidthYerr)
0708
0709 deltaX = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
0710 if ii < len(listbeam) -2:
0711 if deltaX==False and adelta1[0]*adelta2[0] > 0. and math.fabs(adelta1[0]+adelta2[0]) >= limit:
0712
0713 deltaX = True
0714 elif deltaX==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
0715 deltaX = False
0716
0717
0718
0719
0720
0721 adelta1 = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr)
0722 adelta2 = (0.,1.e9)
0723 if iNNbeam.Type != -1:
0724 adelta2 = delta(inextbeam.Y, inextbeam.Yerr, iNNbeam.Y, iNNbeam.Yerr)
0725
0726 deltaY = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
0727 if ii < len(listbeam) -2:
0728 if deltaY==False and adelta1[0]*adelta2[0] > 0. and math.fabs(adelta1[0]+adelta2[0]) >= limit:
0729 deltaY = True
0730 elif deltaY==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
0731 deltaY = False
0732
0733
0734 limit = float(ibeam.sigmaZ)/2.
0735 deltaZ = deltaSig(adelta1z0) > 3.5 and math.fabs(adelta1z0[0]) >= limit
0736
0737 deltasigmaZ = deltaSig(adelta1sigmaZ) > 5.0
0738
0739
0740 adelta = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
0741 deltadxdz = deltaSig(adelta) > 5.0
0742 if deltadxdz and adelta1dxdz[0]*adelta2dxdz[0]<=0 and adelta2dxdz[0] != 0 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) > 0.33 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) < 3:
0743 deltadxdz = False
0744
0745 adelta = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
0746 deltadydz = deltaSig(adelta) > 5.0
0747 if deltadydz and adelta1dydz[0]*adelta2dydz[0]<=0 and adelta2dydz[0] != 0 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) > 0.33 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) < 3:
0748 deltadydz = False
0749
0750 adelta = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
0751 deltawidthX = deltaSig(adelta) > 5
0752 if deltawidthX and adelta1widthx[0]*adelta2widthx[0]<=0 and adelta2widthx[0] != 0 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) > 0.33 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) < 3:
0753 deltawidthX = False
0754
0755 adelta = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
0756 deltawidthY = deltaSig(adelta) > 5
0757 if deltawidthY and adelta1widthy[0]*adelta2widthy[0]<=0 and adelta2widthy[0] != 0 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) > 0.33 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) < 3:
0758 deltawidthY = False
0759
0760
0761
0762
0763
0764
0765
0766
0767
0768
0769
0770
0771 if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
0772 docreate = True
0773
0774
0775
0776
0777
0778
0779 print("close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY))
0780 if docreate:
0781
0782 tmpbeam.IOVlast = ibeam.IOVlast
0783 tmpbeam.IOVEndTime = ibeam.IOVEndTime
0784 print(" Run: "+tmpbeam.Run +" Lumi1: "+str(tmpbeam.IOVfirst) + " Lumi2: "+str(tmpbeam.IOVlast))
0785 newlistbeam.append(tmpbeam)
0786 tmpbeam = BeamSpot()
0787 countlumi = 0
0788 tmprun = ibeam.Run
0789 countlumi += 1
0790
0791 payloadfile = open(fileName,"w")
0792 for iload in newlistbeam:
0793 dump( iload, payloadfile )
0794 payloadfile.close()
0795 return newlistbeam
0796
0797 def createWeightedPayloadsNew(fileName,listbeam=[],weighted=True):
0798 newlistbeam = []
0799 docreate = False
0800 docheck = False
0801 lastPayload = listbeam[0]
0802
0803 firstToUse = 0
0804 lastToUse = 0
0805 for ii in range(0,len(listbeam)):
0806 docreate = False
0807 if docheck:
0808 deltaX = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr) > 1.5
0809 deltaY = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr) > 1.5
0810 deltaZ = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr) > 2.5
0811
0812 deltasigmaZ = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr) > 2.5
0813 deltadxdz = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr) > 2.5
0814 deltadydz = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr) > 2.5
0815
0816 deltawidthX = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr) > 3
0817 deltawidthY = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr) > 3
0818
0819
0820
0821
0822
0823
0824
0825
0826
0827
0828
0829
0830
0831 if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
0832 if ii != 0:
0833 docreate = True
0834 lastToUse = ii-1
0835
0836
0837
0838
0839
0840
0841 print("close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY))
0842
0843
0844 if ii >= len(listbeam) - 1 or listbeam[ii].Run != listbeam[ii+1].Run :
0845 print("close payload because end of run has been reached. Run " + listbeam[ii].Run)
0846 docreate = True
0847 lastToUse = ii
0848
0849
0850
0851
0852
0853
0854
0855 if docreate:
0856 tmpbeam = BeamSpot()
0857 for ibeam in listbeam[firstToUse:lastToUse]:
0858 (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
0859 (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
0860 (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
0861 (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
0862 (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
0863 (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
0864
0865 (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
0866 (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
0867 tmpbeam.IOVfirst = listbeam[firstToUse].IOVfirst
0868 tmpbeam.IOVBeginTime = listbeam[firstToUse].IOVBeginTime
0869 tmpbeam.Run = listbeam[firstToUse].Run
0870 tmpbeam.Type = 2
0871 tmpbeam.IOVlast = listbeam[lastToUse].IOVlast
0872 tmpbeam.IOVEndTime = listbeam[lastToUse].IOVEndTime
0873 newlistbeam.append(tmpbeam)
0874 firstToUse = lastToUse+1
0875 print("Run: " + tmpbeam.Run + " Lumi1: " + str(tmpbeam.IOVfirst) + " Lumi2: " + str(tmpbeam.IOVlast))
0876
0877 payloadfile = open(fileName,"w")
0878 for iload in newlistbeam:
0879 dump( iload, payloadfile )
0880 payloadfile.close()
0881
0882
0883 def writeSqliteFile(sqliteFileName,tagName,timeType,beamSpotFile,sqliteTemplateFile,tmpDir="/tmp/"):
0884 writeDBOut = tmpDir + "write2DB_" + tagName + ".py"
0885 wFile = open(sqliteTemplateFile)
0886 wNewFile = open(writeDBOut,'w')
0887
0888 writeDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
0889 ('TAGNAME',tagName),
0890 ('TIMETYPE',timeType),
0891 ('BEAMSPOTFILE',beamSpotFile)]
0892
0893 for line in wFile:
0894 for itag in writeDBTags:
0895 line = line.replace(itag[0],itag[1])
0896 wNewFile.write(line)
0897
0898 wNewFile.close()
0899 print("writing sqlite file ...")
0900 status_wDB = subprocess.getstatusoutput('cmsRun '+ writeDBOut)
0901 print(status_wDB[1])
0902
0903 os.system("rm -f " + writeDBOut)
0904 return not status_wDB[0]
0905
0906
0907 def readSqliteFile(sqliteFileName,tagName,sqliteTemplateFile,tmpDir="/tmp/"):
0908 readDBOut = tmpDir + "readDB_" + tagName + ".py"
0909
0910 rFile = open(sqliteTemplateFile)
0911 rNewFile = open(readDBOut,'w')
0912
0913 readDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
0914 ('TAGNAME',tagName)]
0915
0916 for line in rFile:
0917 for itag in readDBTags:
0918 line = line.replace(itag[0],itag[1])
0919 rNewFile.write(line)
0920
0921 rNewFile.close()
0922 status_rDB = subprocess.getstatusoutput('cmsRun '+ readDBOut)
0923
0924 outtext = status_rDB[1]
0925 print(outtext)
0926 os.system("rm -f " + readDBOut)
0927 return not status_rDB[0]
0928
0929
0930 def appendSqliteFile(combinedSqliteFileName, sqliteFileName, tagName, IOVSince, IOVTill ,tmpDir="/tmp/"):
0931 aCommand = "conddb_import -c sqlite_file:" + tmpDir + combinedSqliteFileName + " -f sqlite_file:" + sqliteFileName + " -i " + tagName + " -t " + tagName + " -b " + IOVSince + " -e " + IOVTill
0932 print(aCommand)
0933 std = subprocess.getstatusoutput(aCommand)
0934 print(std[1])
0935 return not std[0]
0936
0937
0938 def uploadSqliteFile(sqliteFileDirName, sqliteFileName, dropbox="/DropBox"):
0939
0940 acmd = "chmod a+w " + sqliteFileDirName + sqliteFileName + ".txt"
0941 outcmd = subprocess.getstatusoutput(acmd)
0942 print(acmd)
0943
0944 if outcmd[0]:
0945 print("Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".txt")
0946 return False
0947
0948 acmd = "cp " + sqliteFileDirName + sqliteFileName + ".db " + sqliteFileDirName + sqliteFileName + ".txt ."
0949 print(acmd)
0950 outcmd = subprocess.getstatusoutput(acmd)
0951 print(outcmd[1])
0952 if outcmd[0]:
0953 print("Couldn't cd to " + sqliteFileDirName)
0954 return False
0955
0956 acmd = "tar -cvjf " + sqliteFileName + ".tar.bz2 " + sqliteFileName + ".db " + sqliteFileName + ".txt"
0957 print(acmd)
0958 outcmd = subprocess.getstatusoutput(acmd)
0959 print(outcmd[1])
0960 if outcmd[0]:
0961 print("Couldn't zip the files!")
0962 return False
0963
0964 acmd = "chmod a+w " + sqliteFileName + ".tar.bz2"
0965 outcmd = subprocess.getstatusoutput(acmd)
0966 print(acmd)
0967
0968 if outcmd[0]:
0969 print("Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".tar.bz2")
0970 return False
0971
0972 acmd = "scp -p " + sqliteFileName + ".tar.bz2" + " webcondvm.cern.ch:" + dropbox
0973 print(acmd)
0974 outcmd = subprocess.getstatusoutput(acmd)
0975 print(outcmd[1])
0976 if outcmd[0]:
0977 print("Couldn't scp the files to DropBox!")
0978 return False
0979
0980
0981 acmd = "mv " + sqliteFileName + ".tar.bz2 " + sqliteFileDirName
0982 print(acmd)
0983 outcmd = subprocess.getstatusoutput(acmd)
0984 print(outcmd[1])
0985 if outcmd[0]:
0986 print("Couldn't mv the file to " + sqliteFileDirName)
0987 return False
0988
0989 acmd = "rm " + sqliteFileName + ".db " + sqliteFileName + ".txt"
0990 print(acmd)
0991 outcmd = subprocess.getstatusoutput(acmd)
0992 print(outcmd[1])
0993 if outcmd[0]:
0994 print("Couldn't rm the db and txt files")
0995 return False
0996
0997
0998
0999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011 return False
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021 return True
1022