File indexing completed on 2024-11-27 03:18:06
0001 from builtins import range
0002 import math, re, optparse, subprocess, os, sys, time, datetime
0003 from BeamSpotObj import BeamSpot
0004 from IOVObj import IOV
0005
0006 lockFile = ".lock"
0007
0008
0009 def timeoutManager(type,timeout=-1,fileName=".timeout"):
0010 if timeout == 0:
0011 return 1
0012 timeFormat = "%a,%Y/%m/%d,%H:%M:%S"
0013 currentTime = time.gmtime()
0014 timeoutLine = type + ' ' + time.strftime(timeFormat, currentTime) + '\n'
0015 isTimeout = False
0016 alreadyThere = False
0017 timeoutType = -1;
0018 fileExist = os.path.isfile(fileName)
0019 text = ''
0020 fields = []
0021 reset = False
0022 if timeout == -1:
0023 reset = True
0024 if fileExist:
0025 file = open(fileName)
0026 for line in file:
0027 text += line
0028 fields = line.strip('\n').split(' ')
0029 if fields[0] == type:
0030 alreadyThere = True
0031 if reset:
0032 text = text.replace(line,'')
0033 continue
0034
0035 fileTime = time.strptime(fields[1],timeFormat)
0036 myTime = time.mktime(fileTime)
0037 referenceTime = time.mktime(time.gmtime())
0038 daylight = 0
0039 if currentTime.tm_isdst == 0:
0040 daylight = 3600
0041 elapsedTime = referenceTime-myTime-daylight
0042 if elapsedTime > timeout:
0043 isTimeout = True
0044 timeoutType = 1
0045 print("Timeout! " + str(elapsedTime) + " seconds passed since the " + type + " timeout was set and you can't tolerate more than " + str(timeout) + " seconds!")
0046 else:
0047 timeoutType = 0
0048 print("Timeout of type " + type + " already exist and was generated " + str(elapsedTime) + " seconds ago at " + fields[1])
0049
0050 file.close()
0051
0052 if not fileExist or not alreadyThere and not reset:
0053 timeoutType = -1
0054 text += timeoutLine
0055
0056 if not fileExist or not alreadyThere or isTimeout or (reset and alreadyThere):
0057 if fileExist:
0058 subprocess.getstatusoutput("rm -rf " + fileName)
0059 file = open(fileName,'w')
0060 file.write(text)
0061 file.close()
0062
0063 return timeoutType
0064
0065
0066
0067 def setLockName(name):
0068 global lockFile
0069 lockFile = name
0070
0071
0072 def checkLock():
0073 global lockFile
0074 if os.path.isfile(lockFile):
0075 return True
0076 else:
0077 return False
0078
0079
0080 def lock():
0081 global lockFile
0082 subprocess.getstatusoutput( "touch " + lockFile)
0083
0084
0085 def rmLock():
0086 global lockFile
0087 if checkLock():
0088 subprocess.getstatusoutput( "rm " + lockFile)
0089
0090
0091 def exit(msg=""):
0092 rmLock()
0093 raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
0094
0095
0096 def isnan(num):
0097 fnum = float(num)
0098 return fnum != fnum
0099
0100
0101
0102
0103 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
0104
0105
0106 def parse(docstring, arglist=None):
0107 global optionstring
0108 global tagType
0109 optionstring = docstring
0110 match = USAGE.search(optionstring)
0111 if not match: raise ParsingError("Cannot find the option string")
0112 optlines = match.group(1).splitlines()
0113 try:
0114 p = optparse.OptionParser(optlines[0])
0115 for line in optlines[1:]:
0116 opt, help=line.split(':')[:2]
0117 short,long=opt.split(',')[:2]
0118 if '=' in opt:
0119 action='store'
0120 long=long.split('=')[0]
0121 else:
0122 action='store_true'
0123 p.add_option(short.strip(),long.strip(),
0124 action = action, help = help.strip())
0125 except (IndexError,ValueError):
0126 raise ParsingError("Cannot parse the option string correctly")
0127 return p.parse_args(arglist)
0128
0129
0130 def nonzero(self):
0131 "True if options were given"
0132 for v in self.__dict__.values():
0133 if v is not None: return True
0134 return False
0135
0136
0137 optparse.Values.__nonzero__ = nonzero
0138
0139
0140
0141
0142
0143 class ParsingError(Exception): pass
0144
0145
0146
0147
0148
0149 def sendEmail(mailList,error):
0150 print("Sending email to " + mailList + " with body: " + error)
0151 list = mailList.split(',')
0152 for email in list:
0153 p = os.popen("mail -s \"Automatic workflow error\" " + email ,"w")
0154 p.write(error)
0155 status = p.close()
0156
0157
0158 def dirExists(dir):
0159 if dir.find("castor") != -1:
0160 lsCommand = "nsls " + dir
0161 output = subprocess.getstatusoutput( lsCommand )
0162 return not output[0]
0163 else:
0164 return os.path.exists(dir)
0165
0166
0167 def ls(dir,filter=""):
0168 lsCommand = ''
0169 listOfFiles = []
0170 if dir.find('castor') != -1:
0171 lsCommand = 'ns'
0172 elif not os.path.exists(dir):
0173 print("ERROR: File or directory " + dir + " doesn't exist")
0174 return listOfFiles
0175
0176 aCommand = lsCommand + 'ls '+ dir
0177
0178 if filter != "":
0179 aCommand += " | grep " + filter
0180
0181 tmpStatus = subprocess.getstatusoutput( aCommand )
0182 listOfFiles = tmpStatus[1].split('\n')
0183 if len(listOfFiles) == 1:
0184 if listOfFiles[0].find('No such file or directory') != -1:
0185 exit("ERROR: File or directory " + dir + " doesn't exist")
0186
0187 return listOfFiles
0188
0189
0190 def cp(fromDir,toDir,listOfFiles,overwrite=False,smallList=False):
0191 cpCommand = ''
0192 copiedFiles = []
0193 if fromDir.find('castor') != -1 or toDir.find('castor') != -1 :
0194 cpCommand = 'rf'
0195 elif fromDir.find('resilient') != -1:
0196 cpCommand = 'dc'
0197 if fromDir[len(fromDir)-1] != '/':
0198 fromDir += '/'
0199
0200 if toDir[len(toDir)-1] != '/':
0201 toDir += '/'
0202
0203 for file in listOfFiles:
0204 if os.path.isfile(toDir+file):
0205 if overwrite:
0206 print("File " + file + " already exists in destination directory. We will overwrite it.")
0207 else:
0208 print("File " + file + " already exists in destination directory. We will Keep original file.")
0209 if not smallList:
0210 copiedFiles.append(file)
0211 continue
0212
0213 aCommand = cpCommand + 'cp '+ fromDir + file + " " + toDir
0214 print(" >> " + aCommand)
0215 tmpStatus = subprocess.getstatusoutput( aCommand )
0216 if tmpStatus[0] == 0:
0217 copiedFiles.append(file)
0218 else:
0219 print("[cp()]\tERROR: Can't copy file " + file)
0220 return copiedFiles
0221
0222
0223
0224
0225
0226
0227
0228 def pack(high,low):
0229 """pack high,low 32bit unsigned int to one unsigned 64bit long long
0230 Note:the print value of result number may appear signed, if the sign bit is used.
0231 """
0232 h=high<<32
0233 return (h|low)
0234
0235
0236 def unpack(i):
0237 """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
0238 """
0239 high=i>>32
0240 low=i&0xFFFFFFFF
0241 return(high,low)
0242
0243
0244 def unpackLumiid(i):
0245 """unpack 64bit lumiid to dictionary {'run','lumisection'}
0246 """
0247 j=unpack(i)
0248 return {'run':j[0],'lumisection':j[1]}
0249
0250
0251
0252
0253
0254 def cmp_list_run(a,b):
0255 if int(a.IOVfirst) < int(b.IOVfirst): return -1
0256 if int(a.IOVfirst) == int(b.IOVfirst): return 0
0257 if int(a.IOVfirst) > int(b.IOVfirst): return 1
0258
0259
0260 def cmp_list_lumi(a,b):
0261 if int(a.Run) < int(b.Run): return -1
0262 if int(a.Run) == int(b.Run):
0263 if int(a.IOVfirst) < int(b.IOVfirst): return -1
0264 if int(a.IOVfirst) == int(b.IOVfirst): return 0
0265 if int(a.IOVfirst) > int(b.IOVfirst): return 1
0266 if int(a.Run) > int(b.Run) : return 1
0267
0268
0269 def weight(x1, x1err,x2,x2err):
0270
0271 x1 = float(x1)
0272 x1err = float(x1err)
0273 x2 = float(x2)
0274 x2err = float(x2err)
0275 tmperr = 0.
0276 if x2err < 1e-6 :
0277 x2err = 1e-6
0278 if x1err < 1e-6:
0279 x1 = x2/(x2err * x2err)
0280 tmperr = 1/(x2err*x2err)
0281 else:
0282 x1 = x1/(x1err*x1err) + x2/(x2err * x2err)
0283 tmperr = 1/(x1err*x1err) + 1/(x2err*x2err)
0284 x1 = x1/tmperr
0285 x1err = 1/tmperr
0286 x1err = math.sqrt(x1err)
0287 return (str(x1), str(x1err))
0288
0289
0290 def dump( beam, file):
0291 end = "\n"
0292 file.write("Runnumber "+beam.Run+end)
0293 file.write("BeginTimeOfFit "+str(beam.IOVBeginTime)+end)
0294 file.write("EndTimeOfFit "+str(beam.IOVEndTime)+end)
0295 file.write("LumiRange "+str(beam.IOVfirst)+" - "+str(beam.IOVlast)+end)
0296 dumpValues(beam, file)
0297
0298
0299 def dumpValues( beam, file):
0300 end = "\n"
0301 file.write("Type "+str(beam.Type)+end)
0302 file.write("X0 "+str(beam.X)+end)
0303 file.write("Y0 "+str(beam.Y)+end)
0304 file.write("Z0 "+str(beam.Z)+end)
0305 file.write("sigmaZ0 "+str(beam.sigmaZ)+end)
0306 file.write("dxdz "+str(beam.dxdz)+end)
0307 file.write("dydz "+str(beam.dydz)+end)
0308 file.write("BeamWidthX "+beam.beamWidthX+end)
0309 file.write("BeamWidthY "+beam.beamWidthY+end)
0310 file.write("Cov(0,j) "+str(math.pow(float(beam.Xerr),2))+" 0 0 0 0 0 0" +end)
0311 file.write("Cov(1,j) 0 "+str(math.pow(float(beam.Yerr),2))+" 0 0 0 0 0" +end)
0312 file.write("Cov(2,j) 0 0 "+str(math.pow(float(beam.Zerr),2))+" 0 0 0 0" +end)
0313 file.write("Cov(3,j) 0 0 0 "+str(math.pow(float(beam.sigmaZerr),2))+" 0 0 0" +end)
0314 file.write("Cov(4,j) 0 0 0 0 "+str(math.pow(float(beam.dxdzerr),2))+" 0 0" +end)
0315 file.write("Cov(5,j) 0 0 0 0 0 "+str(math.pow(float(beam.dydzerr),2))+" 0" +end)
0316 file.write("Cov(6,j) 0 0 0 0 0 0 "+str(math.pow(float(beam.beamWidthXerr),2)) +end)
0317 file.write("EmittanceX 0"+end)
0318 file.write("EmittanceY 0"+end)
0319 file.write("BetaStar 0"+end)
0320
0321
0322 def delta(x,xerr,nextx,nextxerr):
0323
0324 return ( float(x) - float(nextx), math.sqrt(math.pow(float(xerr),2) + math.pow(float(nextxerr),2)) )
0325
0326 def deltaSig( x ):
0327 return math.fabs(x[0])/x[1]
0328
0329
0330 def readBeamSpotFile(fileName,listbeam=[],IOVbase="runbase", firstRun='1',lastRun='4999999999'):
0331 tmpbeam = BeamSpot()
0332 tmpbeamsize = 0
0333
0334
0335
0336 if IOVbase == "lumibase" and firstRun=='1' and lastRun=='4999999999' :
0337 firstRun = "1:1"
0338 lastRun = "4999999999:4999999999"
0339
0340 inputfiletype = 0
0341
0342
0343
0344
0345 maplist = {}
0346 hasBX = False
0347
0348 tmpfile = open(fileName)
0349 atmpline = tmpfile.readline()
0350 if atmpline.find('Runnumber') != -1:
0351 inputfiletype = 1
0352 if len(atmpline.split()) > 2:
0353 hasBX = True
0354 print(" Input data has been calculated as function of BUNCH CROSSINGS.")
0355 tmpfile.seek(0)
0356
0357
0358 if inputfiletype ==1:
0359
0360 tmpBX = 0
0361 for line in tmpfile:
0362
0363 if line.find('Type') != -1:
0364 tmpbeam.Type = int(line.split()[1])
0365 tmpbeamsize += 1
0366 if line.find('X0') != -1:
0367 tmpbeam.X = line.split()[1]
0368
0369 tmpbeamsize += 1
0370
0371 if line.find('Y0') != -1:
0372 tmpbeam.Y = line.split()[1]
0373
0374 tmpbeamsize += 1
0375
0376 if line.find('Z0') != -1 and line.find('sigmaZ0') == -1:
0377 tmpbeam.Z = line.split()[1]
0378
0379 tmpbeamsize += 1
0380 if line.find('sigmaZ0') !=-1:
0381 tmpbeam.sigmaZ = line.split()[1]
0382
0383 tmpbeamsize += 1
0384 if line.find('dxdz') != -1:
0385 tmpbeam.dxdz = line.split()[1]
0386
0387 tmpbeamsize += 1
0388 if line.find('dydz') != -1:
0389 tmpbeam.dydz = line.split()[1]
0390
0391 tmpbeamsize += 1
0392 if line.find('BeamWidthX') != -1:
0393 tmpbeam.beamWidthX = line.split()[1]
0394
0395 tmpbeamsize += 1
0396 if line.find('BeamWidthY') != -1:
0397 tmpbeam.beamWidthY = line.split()[1]
0398
0399 tmpbeamsize += 1
0400 if line.find('Cov(0,j)') != -1:
0401 tmpbeam.Xerr = str(math.sqrt( float( line.split()[1] ) ) )
0402 tmpbeamsize += 1
0403 if line.find('Cov(1,j)') != -1:
0404 tmpbeam.Yerr = str(math.sqrt( float( line.split()[2] ) ) )
0405 tmpbeamsize += 1
0406 if line.find('Cov(2,j)') != -1:
0407 tmpbeam.Zerr = str(math.sqrt( float( line.split()[3] ) ) )
0408 tmpbeamsize += 1
0409 if line.find('Cov(3,j)') != -1:
0410 tmpbeam.sigmaZerr = str(math.sqrt( float( line.split()[4] ) ) )
0411 tmpbeamsize += 1
0412 if line.find('Cov(4,j)') != -1:
0413 tmpbeam.dxdzerr = str(math.sqrt( float( line.split()[5] ) ) )
0414 tmpbeamsize += 1
0415 if line.find('Cov(5,j)') != -1:
0416 tmpbeam.dydzerr = str(math.sqrt( float( line.split()[6] ) ) )
0417 tmpbeamsize += 1
0418 if line.find('Cov(6,j)') != -1:
0419 tmpbeam.beamWidthXerr = str(math.sqrt( float( line.split()[7] ) ) )
0420 tmpbeam.beamWidthYerr = tmpbeam.beamWidthXerr
0421 tmpbeamsize += 1
0422 if line.find('LumiRange') != -1:
0423 if IOVbase=="lumibase":
0424 tmpbeam.IOVfirst = line.split()[1]
0425 tmpbeam.IOVlast = line.split()[3]
0426 tmpbeamsize += 1
0427 if line.find('Runnumber') != -1:
0428 tmpbeam.Run = line.split()[1]
0429 if IOVbase == "runbase":
0430 tmpbeam.IOVfirst = line.split()[1]
0431 tmpbeam.IOVlast = line.split()[1]
0432 if hasBX:
0433 tmpBX = line.split()[3]
0434 tmpbeamsize += 1
0435 if line.find('BeginTimeOfFit') != -1:
0436 tmpbeam.IOVBeginTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
0437 if IOVbase =="timebase":
0438 tmpbeam.IOVfirst = time.mktime( time.strptime(line.split()[1] + " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
0439 tmpbeamsize += 1
0440 if line.find('EndTimeOfFit') != -1:
0441 tmpbeam.IOVEndTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
0442 if IOVbase =="timebase":
0443 tmpbeam.IOVlast = time.mktime( time.strptime(line.split()[1] + " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
0444 tmpbeamsize += 1
0445 if tmpbeamsize == 20:
0446 if IOVbase=="lumibase":
0447 tmprunfirst = int(firstRun.split(":")[0])
0448 tmprunlast = int(lastRun.split(":")[0])
0449 tmplumifirst = int(firstRun.split(":")[1])
0450 tmplumilast = int(lastRun.split(":")[1])
0451 acceptiov1 = acceptiov2 = False
0452
0453 if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
0454 if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
0455 acceptiov1 = acceptiov2 = True
0456
0457 elif int(tmpbeam.Run) == tmprunfirst:
0458 if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
0459 elif int(tmpbeam.Run) == tmprunlast:
0460 if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
0461 elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
0462 acceptiov1 = acceptiov2 = True
0463
0464 if acceptiov1 and acceptiov2:
0465 if tmpbeam.Type != 2:
0466 print("invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0467 elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0468 print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0469 elif hasBX:
0470 if (tmpBX in maplist) == False:
0471 maplist[tmpBX] = [tmpbeam]
0472 else:
0473 maplist[tmpBX].append(tmpbeam)
0474 else:
0475 listbeam.append(tmpbeam)
0476
0477 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
0478 if tmpbeam.Type != 2:
0479 print("invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0480 elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0481 print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0482 else:
0483 listbeam.append(tmpbeam)
0484
0485 tmpbeamsize = 0
0486 tmpbeam = BeamSpot()
0487 tmpBX = 0
0488 else:
0489
0490 for line in tmpfile:
0491
0492 if line.find('X0') != -1:
0493 tmpbeam.X = line.split()[2]
0494 tmpbeam.Xerr = line.split()[4]
0495 tmpbeamsize += 1
0496
0497 if line.find('Y0') != -1:
0498 tmpbeam.Y = line.split()[2]
0499 tmpbeam.Yerr = line.split()[4]
0500 tmpbeamsize += 1
0501
0502 if line.find('Z0') != -1 and line.find('Sigma Z0') == -1:
0503 tmpbeam.Z = line.split()[2]
0504 tmpbeam.Zerr = line.split()[4]
0505 tmpbeamsize += 1
0506
0507 if line.find('Sigma Z0') !=-1:
0508 tmpbeam.sigmaZ = line.split()[3]
0509 tmpbeam.sigmaZerr = line.split()[5]
0510 tmpbeamsize += 1
0511 if line.find('dxdz') != -1:
0512 tmpbeam.dxdz = line.split()[2]
0513 tmpbeam.dxdzerr = line.split()[4]
0514 tmpbeamsize += 1
0515 if line.find('dydz') != -1:
0516 tmpbeam.dydz = line.split()[2]
0517 tmpbeam.dydzerr = line.split()[4]
0518 tmpbeamsize += 1
0519 if line.find('Beam Width X') != -1:
0520 tmpbeam.beamWidthX = line.split()[4]
0521 tmpbeam.beamWidthXerr = line.split()[6]
0522 tmpbeamsize += 1
0523 if line.find('Beam Width Y') != -1:
0524 tmpbeam.beamWidthY = line.split()[4]
0525 tmpbeam.beamWidthYerr = line.split()[6]
0526 tmpbeamsize += 1
0527
0528 if line.find('for runs') != -1:
0529
0530 tmpbeam.Run = line.split()[2]
0531 if IOVbase == "runbase":
0532 tmpbeam.IOVfirst = line.split()[2]
0533 tmpbeam.IOVlast = line.split()[4]
0534 tmpbeamsize += 1
0535 if line.find('LumiSection') != -1:
0536 if IOVbase=="lumibase":
0537 tmpbeam.IOVfirst = line.split()[10]
0538 tmpbeam.IOVlast = line.split()[10]
0539 tmpbeamsize += 1
0540 if tmpbeamsize == 10:
0541
0542 if IOVbase=="lumibase":
0543 tmprunfirst = int(firstRun.split(":")[0])
0544 tmprunlast = int(lastRun.split(":")[0])
0545 tmplumifirst = int(firstRun.split(":")[1])
0546 tmplumilast = int(lastRun.split(":")[1])
0547 acceptiov1 = acceptiov2 = False
0548
0549 if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
0550 if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
0551 acceptiov1 = acceptiov2 = True
0552
0553 elif int(tmpbeam.Run) == tmprunfirst:
0554 if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
0555 elif int(tmpbeam.Run) == tmprunlast:
0556 if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
0557 elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
0558 acceptiov1 = acceptiov2 = True
0559
0560 if acceptiov1 and acceptiov2:
0561 if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0562 print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0563 elif hasBX:
0564 if (tmpBX in maplist) == False:
0565 maplist[tmpBX] = [tmpbeam]
0566 else:
0567 maplist[tmpBX].append(tmpbeam)
0568 else:
0569 listbeam.append(tmpbeam)
0570
0571 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
0572 if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0573 print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0574 else:
0575 listbeam.append(tmpbeam)
0576
0577 tmpbeamsize = 0
0578 tmpbeam = BeamSpot()
0579 tmpBX = 0
0580
0581 tmpfile.close()
0582 print(" got total number of IOVs = " + str(len(listbeam)) + " from file " + fileName)
0583
0584 if hasBX:
0585 return maplist
0586 else:
0587 return listbeam
0588
0589
0590
0591 def sortAndCleanBeamList(listbeam=[],IOVbase="lumibase"):
0592
0593 if IOVbase == "lumibase":
0594 listbeam.sort( cmp = cmp_list_lumi )
0595 else:
0596 listbeam.sort( cmp = cmp_list_run )
0597
0598
0599 tmpremovelist = []
0600 for ii in range(0,len(listbeam)):
0601 ibeam = listbeam[ii]
0602 datax = ibeam.IOVfirst
0603
0604 if datax == '0' and IOVbase =="runbase":
0605 print(" iov = 0? skip this IOV = "+ str(ibeam.IOVfirst) + " to " + str(ibeam.IOVlast))
0606 tmpremovelist.append(ibeam)
0607
0608 if ii < len(listbeam) -1:
0609
0610 if IOVbase =="lumibase":
0611 if ibeam.Run == listbeam[ii+1].Run and ibeam.IOVfirst == listbeam[ii+1].IOVfirst:
0612 print(" duplicate IOV = "+datax+", keep only last duplicate entry")
0613 tmpremovelist.append(ibeam)
0614 elif datax == listbeam[ii+1].IOVfirst:
0615 print(" duplicate IOV = "+datax+", keep only last duplicate entry")
0616 tmpremovelist.append(ibeam)
0617
0618 for itmp in tmpremovelist:
0619 listbeam.remove(itmp)
0620
0621
0622
0623 def createWeightedPayloads(fileName,listbeam=[],weighted=True):
0624 newlistbeam = []
0625 tmpbeam = BeamSpot()
0626 docreate = True
0627 countlumi = 0
0628 tmprun = ""
0629 maxNlumis = 60
0630 if weighted:
0631 maxNlumis = 999999999
0632 for ii in range(0,len(listbeam)):
0633 ibeam = listbeam[ii]
0634 inextbeam = BeamSpot()
0635 iNNbeam = BeamSpot()
0636 if docreate:
0637 tmpbeam.IOVfirst = ibeam.IOVfirst
0638 tmpbeam.IOVBeginTime = ibeam.IOVBeginTime
0639 tmpbeam.Run = ibeam.Run
0640 tmpbeam.Type = 2
0641 docheck = False
0642 docreate = False
0643
0644
0645
0646 if ii < len(listbeam) - 1:
0647 inextbeam = listbeam[ii+1]
0648 docheck = True
0649 if ii < len(listbeam) -2:
0650 iNNbeam = listbeam[ii+2]
0651 else:
0652 print("close payload because end of data has been reached. Run "+ibeam.Run)
0653 docreate = True
0654
0655 if ibeam.Run != inextbeam.Run:
0656 print("close payload because end of run "+ibeam.Run)
0657 docreate = True
0658
0659 if countlumi == maxNlumis -1:
0660 print("close payload because maximum lumi sections accumulated within run "+ibeam.Run)
0661 docreate = True
0662 countlumi = 0
0663
0664 (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
0665 (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
0666 (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
0667 (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
0668 (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
0669 (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
0670
0671 (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
0672 (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
0673
0674 if weighted:
0675 docheck = False
0676
0677
0678 if docheck:
0679
0680
0681 min_limit = 0.0025
0682
0683
0684 limit = float(ibeam.beamWidthX)/2.
0685 if limit < min_limit: limit = min_limit
0686
0687
0688 adelta1 = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr)
0689 adelta2 = (0.,1.e9)
0690 adelta1dxdz = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
0691 adelta2dxdz = (0.,1.e9)
0692 adelta1dydz = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
0693 adelta2dydz = (0.,1.e9)
0694 adelta1widthx = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
0695 adelta2widthx = (0.,1.e9)
0696 adelta1widthy = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
0697 adelta2widthy = (0.,1.e9)
0698 adelta1z0 = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr)
0699 adelta1sigmaZ = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr)
0700
0701 if iNNbeam.Type != -1:
0702 adelta2 = delta(inextbeam.X, inextbeam.Xerr, iNNbeam.X, iNNbeam.Xerr)
0703 adelta2dxdz = delta(inextbeam.dxdz, inextbeam.dxdzerr, iNNbeam.dxdz, iNNbeam.dxdzerr)
0704 adelta2dydz = delta(inextbeam.dydz, inextbeam.dydzerr, iNNbeam.dydz, iNNbeam.dydzerr)
0705 adelta2widthx = delta(inextbeam.beamWidthX, inextbeam.beamWidthXerr, iNNbeam.beamWidthX, iNNbeam.beamWidthXerr)
0706 adelta2widthy = delta(inextbeam.beamWidthY, inextbeam.beamWidthYerr, iNNbeam.beamWidthY, iNNbeam.beamWidthYerr)
0707
0708 deltaX = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
0709 if ii < len(listbeam) -2:
0710 if deltaX==False and adelta1[0]*adelta2[0] > 0. and math.fabs(adelta1[0]+adelta2[0]) >= limit:
0711
0712 deltaX = True
0713 elif deltaX==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
0714 deltaX = False
0715
0716
0717
0718
0719
0720 adelta1 = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr)
0721 adelta2 = (0.,1.e9)
0722 if iNNbeam.Type != -1:
0723 adelta2 = delta(inextbeam.Y, inextbeam.Yerr, iNNbeam.Y, iNNbeam.Yerr)
0724
0725 deltaY = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
0726 if ii < len(listbeam) -2:
0727 if deltaY==False and adelta1[0]*adelta2[0] > 0. and math.fabs(adelta1[0]+adelta2[0]) >= limit:
0728 deltaY = True
0729 elif deltaY==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
0730 deltaY = False
0731
0732
0733 limit = float(ibeam.sigmaZ)/2.
0734 deltaZ = deltaSig(adelta1z0) > 3.5 and math.fabs(adelta1z0[0]) >= limit
0735
0736 deltasigmaZ = deltaSig(adelta1sigmaZ) > 5.0
0737
0738
0739 adelta = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
0740 deltadxdz = deltaSig(adelta) > 5.0
0741 if deltadxdz and adelta1dxdz[0]*adelta2dxdz[0]<=0 and adelta2dxdz[0] != 0 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) > 0.33 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) < 3:
0742 deltadxdz = False
0743
0744 adelta = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
0745 deltadydz = deltaSig(adelta) > 5.0
0746 if deltadydz and adelta1dydz[0]*adelta2dydz[0]<=0 and adelta2dydz[0] != 0 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) > 0.33 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) < 3:
0747 deltadydz = False
0748
0749 adelta = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
0750 deltawidthX = deltaSig(adelta) > 5
0751 if deltawidthX and adelta1widthx[0]*adelta2widthx[0]<=0 and adelta2widthx[0] != 0 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) > 0.33 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) < 3:
0752 deltawidthX = False
0753
0754 adelta = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
0755 deltawidthY = deltaSig(adelta) > 5
0756 if deltawidthY and adelta1widthy[0]*adelta2widthy[0]<=0 and adelta2widthy[0] != 0 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) > 0.33 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) < 3:
0757 deltawidthY = False
0758
0759
0760
0761
0762
0763
0764
0765
0766
0767
0768
0769
0770 if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
0771 docreate = True
0772
0773
0774
0775
0776
0777
0778 print("close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY))
0779 if docreate:
0780
0781 tmpbeam.IOVlast = ibeam.IOVlast
0782 tmpbeam.IOVEndTime = ibeam.IOVEndTime
0783 print(" Run: "+tmpbeam.Run +" Lumi1: "+str(tmpbeam.IOVfirst) + " Lumi2: "+str(tmpbeam.IOVlast))
0784 newlistbeam.append(tmpbeam)
0785 tmpbeam = BeamSpot()
0786 countlumi = 0
0787 tmprun = ibeam.Run
0788 countlumi += 1
0789
0790 payloadfile = open(fileName,"w")
0791 for iload in newlistbeam:
0792 dump( iload, payloadfile )
0793 payloadfile.close()
0794 return newlistbeam
0795
0796 def createWeightedPayloadsNew(fileName,listbeam=[],weighted=True):
0797 newlistbeam = []
0798 docreate = False
0799 docheck = False
0800 lastPayload = listbeam[0]
0801
0802 firstToUse = 0
0803 lastToUse = 0
0804 for ii in range(0,len(listbeam)):
0805 docreate = False
0806 if docheck:
0807 deltaX = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr) > 1.5
0808 deltaY = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr) > 1.5
0809 deltaZ = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr) > 2.5
0810
0811 deltasigmaZ = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr) > 2.5
0812 deltadxdz = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr) > 2.5
0813 deltadydz = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr) > 2.5
0814
0815 deltawidthX = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr) > 3
0816 deltawidthY = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr) > 3
0817
0818
0819
0820
0821
0822
0823
0824
0825
0826
0827
0828
0829
0830 if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
0831 if ii != 0:
0832 docreate = True
0833 lastToUse = ii-1
0834
0835
0836
0837
0838
0839
0840 print("close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY))
0841
0842
0843 if ii >= len(listbeam) - 1 or listbeam[ii].Run != listbeam[ii+1].Run :
0844 print("close payload because end of run has been reached. Run " + listbeam[ii].Run)
0845 docreate = True
0846 lastToUse = ii
0847
0848
0849
0850
0851
0852
0853
0854 if docreate:
0855 tmpbeam = BeamSpot()
0856 for ibeam in listbeam[firstToUse:lastToUse]:
0857 (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
0858 (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
0859 (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
0860 (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
0861 (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
0862 (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
0863
0864 (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
0865 (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
0866 tmpbeam.IOVfirst = listbeam[firstToUse].IOVfirst
0867 tmpbeam.IOVBeginTime = listbeam[firstToUse].IOVBeginTime
0868 tmpbeam.Run = listbeam[firstToUse].Run
0869 tmpbeam.Type = 2
0870 tmpbeam.IOVlast = listbeam[lastToUse].IOVlast
0871 tmpbeam.IOVEndTime = listbeam[lastToUse].IOVEndTime
0872 newlistbeam.append(tmpbeam)
0873 firstToUse = lastToUse+1
0874 print("Run: " + tmpbeam.Run + " Lumi1: " + str(tmpbeam.IOVfirst) + " Lumi2: " + str(tmpbeam.IOVlast))
0875
0876 payloadfile = open(fileName,"w")
0877 for iload in newlistbeam:
0878 dump( iload, payloadfile )
0879 payloadfile.close()
0880
0881
0882 def writeSqliteFile(sqliteFileName,tagName,timeType,beamSpotFile,sqliteTemplateFile,tmpDir="/tmp/"):
0883 writeDBOut = tmpDir + "write2DB_" + tagName + ".py"
0884 wFile = open(sqliteTemplateFile)
0885 wNewFile = open(writeDBOut,'w')
0886
0887 writeDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
0888 ('TAGNAME',tagName),
0889 ('TIMETYPE',timeType),
0890 ('BEAMSPOTFILE',beamSpotFile)]
0891
0892 for line in wFile:
0893 for itag in writeDBTags:
0894 line = line.replace(itag[0],itag[1])
0895 wNewFile.write(line)
0896
0897 wNewFile.close()
0898 print("writing sqlite file ...")
0899 status_wDB = subprocess.getstatusoutput('cmsRun '+ writeDBOut)
0900 print(status_wDB[1])
0901
0902 os.system("rm -f " + writeDBOut)
0903 return not status_wDB[0]
0904
0905
0906 def readSqliteFile(sqliteFileName,tagName,sqliteTemplateFile,tmpDir="/tmp/"):
0907 readDBOut = tmpDir + "readDB_" + tagName + ".py"
0908
0909 rFile = open(sqliteTemplateFile)
0910 rNewFile = open(readDBOut,'w')
0911
0912 readDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
0913 ('TAGNAME',tagName)]
0914
0915 for line in rFile:
0916 for itag in readDBTags:
0917 line = line.replace(itag[0],itag[1])
0918 rNewFile.write(line)
0919
0920 rNewFile.close()
0921 status_rDB = subprocess.getstatusoutput('cmsRun '+ readDBOut)
0922
0923 outtext = status_rDB[1]
0924 print(outtext)
0925 os.system("rm -f " + readDBOut)
0926 return not status_rDB[0]
0927
0928
0929 def appendSqliteFile(combinedSqliteFileName, sqliteFileName, tagName, IOVSince, IOVTill ,tmpDir="/tmp/"):
0930 aCommand = "conddb_import -c sqlite_file:" + tmpDir + combinedSqliteFileName + " -f sqlite_file:" + sqliteFileName + " -i " + tagName + " -t " + tagName + " -b " + IOVSince + " -e " + IOVTill
0931 print(aCommand)
0932 std = subprocess.getstatusoutput(aCommand)
0933 print(std[1])
0934 return not std[0]
0935
0936
0937 def uploadSqliteFile(sqliteFileDirName, sqliteFileName, dropbox="/DropBox"):
0938
0939 acmd = "chmod a+w " + sqliteFileDirName + sqliteFileName + ".txt"
0940 outcmd = subprocess.getstatusoutput(acmd)
0941 print(acmd)
0942
0943 if outcmd[0]:
0944 print("Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".txt")
0945 return False
0946
0947 acmd = "cp " + sqliteFileDirName + sqliteFileName + ".db " + sqliteFileDirName + sqliteFileName + ".txt ."
0948 print(acmd)
0949 outcmd = subprocess.getstatusoutput(acmd)
0950 print(outcmd[1])
0951 if outcmd[0]:
0952 print("Couldn't cd to " + sqliteFileDirName)
0953 return False
0954
0955 acmd = "tar -cvjf " + sqliteFileName + ".tar.bz2 " + sqliteFileName + ".db " + sqliteFileName + ".txt"
0956 print(acmd)
0957 outcmd = subprocess.getstatusoutput(acmd)
0958 print(outcmd[1])
0959 if outcmd[0]:
0960 print("Couldn't zip the files!")
0961 return False
0962
0963 acmd = "chmod a+w " + sqliteFileName + ".tar.bz2"
0964 outcmd = subprocess.getstatusoutput(acmd)
0965 print(acmd)
0966
0967 if outcmd[0]:
0968 print("Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".tar.bz2")
0969 return False
0970
0971 acmd = "scp -p " + sqliteFileName + ".tar.bz2" + " webcondvm.cern.ch:" + dropbox
0972 print(acmd)
0973 outcmd = subprocess.getstatusoutput(acmd)
0974 print(outcmd[1])
0975 if outcmd[0]:
0976 print("Couldn't scp the files to DropBox!")
0977 return False
0978
0979
0980 acmd = "mv " + sqliteFileName + ".tar.bz2 " + sqliteFileDirName
0981 print(acmd)
0982 outcmd = subprocess.getstatusoutput(acmd)
0983 print(outcmd[1])
0984 if outcmd[0]:
0985 print("Couldn't mv the file to " + sqliteFileDirName)
0986 return False
0987
0988 acmd = "rm " + sqliteFileName + ".db " + sqliteFileName + ".txt"
0989 print(acmd)
0990 outcmd = subprocess.getstatusoutput(acmd)
0991 print(outcmd[1])
0992 if outcmd[0]:
0993 print("Couldn't rm the db and txt files")
0994 return False
0995
0996
0997
0998
0999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010 return False
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020 return True
1021