BeamSpotProducer/scripts/CommonMethods.py

0001 from __future__ import print_function
0002 from builtins import range
0003 import math, re, optparse, subprocess, os, sys, time, datetime
0004 from BeamSpotObj import BeamSpot
0005 from IOVObj import IOV
0006
0007 lockFile = ".lock"
0008
0009 ###########################################################################################
0010 def timeoutManager(type,timeout=-1,fileName=".timeout"):
0011     if timeout == 0:
0012         return 1
0013     timeFormat = "%a,%Y/%m/%d,%H:%M:%S"
0014     currentTime = time.gmtime()
0015     timeoutLine = type + ' ' + time.strftime(timeFormat, currentTime) + '\n'
0016     isTimeout = False
0017     alreadyThere = False
0018     timeoutType = -1;
0019     fileExist = os.path.isfile(fileName)
0020     text = ''
0021     fields = []
0022     reset = False
0023     if timeout == -1:
0024         reset = True
0025     if fileExist:
0026         file = open(fileName)
0027         for line in file:
0028             text += line
0029             fields = line.strip('\n').split(' ')
0030             if fields[0] == type:
0031                 alreadyThere = True
0032                 if reset:
0033                     text = text.replace(line,'')
0034                     continue
0035
0036                 fileTime = time.strptime(fields[1],timeFormat)
0037                 myTime = time.mktime(fileTime)
0038                 referenceTime = time.mktime(time.gmtime())
0039                 daylight = 0
0040                 if currentTime.tm_isdst == 0:
0041                     daylight = 3600
0042                 elapsedTime = referenceTime-myTime-daylight
0043                 if elapsedTime > timeout:
0044                     isTimeout = True
0045                     timeoutType = 1
0046                     print("Timeout! " + str(elapsedTime) + " seconds passed since the " + type + " timeout was set and you can't tolerate more than " + str(timeout) + " seconds!")
0047                 else:
0048                     timeoutType = 0
0049                     print("Timeout of type " + type + " already exist and was generated " + str(elapsedTime) + " seconds ago at " + fields[1])
0050
0051         file.close()
0052
0053     if not fileExist or not alreadyThere and not reset:
0054         timeoutType = -1
0055         text += timeoutLine
0056
0057     if not fileExist or not alreadyThere or isTimeout or (reset and alreadyThere):
0058         if fileExist:
0059             subprocess.getstatusoutput("rm -rf " + fileName)
0060         file = open(fileName,'w')
0061         file.write(text)
0062         file.close()
0063
0064     return timeoutType
0065
0066
0067 ###########################################################################################
0068 def setLockName(name):
0069     global lockFile
0070     lockFile = name
0071
0072 ###########################################################################################
0073 def checkLock():
0074     global lockFile
0075     if os.path.isfile(lockFile):
0076         return True
0077     else:
0078         return False
0079
0080 ###########################################################################################
0081 def lock():
0082     global lockFile
0083     subprocess.getstatusoutput( "touch " + lockFile)
0084
0085 ###########################################################################################
0086 def rmLock():
0087     global lockFile
0088     if checkLock():
0089         subprocess.getstatusoutput( "rm " + lockFile)
0090
0091 ###########################################################################################
0092 def exit(msg=""):
0093     rmLock()
0094     raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
0095
0096 ###########################################################################################
0097 def isnan(num):
0098     fnum = float(num)
0099     return fnum != fnum
0100
0101 ###########################################################################################
0102 # OPTIONS
0103 ###########################################################################################
0104 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
0105
0106 ###########################################################################################
0107 def parse(docstring, arglist=None):
0108     global optionstring
0109     global tagType
0110     optionstring = docstring
0111     match = USAGE.search(optionstring)
0112     if not match: raise ParsingError("Cannot find the option string")
0113     optlines = match.group(1).splitlines()
0114     try:
0115         p = optparse.OptionParser(optlines[0])
0116         for line in optlines[1:]:
0117             opt, help=line.split(':')[:2]
0118             short,long=opt.split(',')[:2]
0119             if '=' in opt:
0120                 action='store'
0121                 long=long.split('=')[0]
0122             else:
0123                 action='store_true'
0124             p.add_option(short.strip(),long.strip(),
0125                          action = action, help = help.strip())
0126     except (IndexError,ValueError):
0127         raise ParsingError("Cannot parse the option string correctly")
0128     return p.parse_args(arglist)
0129
0130 ###########################################################################################
0131 def nonzero(self): # will become the nonzero method of optparse.Values
0132     "True if options were given"
0133     for v in self.__dict__.values():
0134         if v is not None: return True
0135     return False
0136
0137 ###########################################################################################
0138 optparse.Values.__nonzero__ = nonzero # dynamically fix optparse.Values
0139
0140 # END OPTIONS
0141 ###########################################################################################
0142
0143 ###########################################################################################
0144 class ParsingError(Exception): pass
0145
0146 ###########################################################################################
0147 # General utilities
0148 ###########################################################################################
0149 ###########################################################################################
0150 def sendEmail(mailList,error):
0151     print("Sending email to " + mailList + " with body: " + error)
0152     list = mailList.split(',')
0153     for email in list:
0154         p = os.popen("mail -s \"Automatic workflow error\" " + email ,"w")
0155         p.write(error)
0156         status = p.close()
0157
0158 ###########################################################################################
0159 def dirExists(dir):
0160     if dir.find("castor") != -1:
0161         lsCommand = "nsls " + dir
0162         output = subprocess.getstatusoutput( lsCommand )
0163         return not output[0]
0164     else:
0165         return os.path.exists(dir)
0166
0167 ########################################################################
0168 def ls(dir,filter=""):
0169     lsCommand      = ''
0170     listOfFiles    = []
0171     if dir.find('castor') != -1:
0172         lsCommand = 'ns'
0173     elif not os.path.exists(dir):
0174         print("ERROR: File or directory " + dir + " doesn't exist")
0175         return listOfFiles
0176
0177     aCommand  = lsCommand  + 'ls '+ dir
0178     #aCommand  = lsCommand  + 'ls '+ dir + " | grep .txt"
0179     if filter != "":
0180         aCommand  += " | grep " + filter
0181
0182     tmpStatus = subprocess.getstatusoutput( aCommand )
0183     listOfFiles = tmpStatus[1].split('\n')
0184     if len(listOfFiles) == 1:
0185         if listOfFiles[0].find('No such file or directory') != -1:
0186             exit("ERROR: File or directory " + dir + " doesn't exist")
0187
0188     return listOfFiles
0189
0190 ########################################################################
0191 def cp(fromDir,toDir,listOfFiles,overwrite=False,smallList=False):
0192     cpCommand   = ''
0193     copiedFiles = []
0194     if fromDir.find('castor') != -1 or toDir.find('castor') != -1 :
0195         cpCommand = 'rf'
0196     elif fromDir.find('resilient') != -1:
0197         cpCommand = 'dc'
0198     if fromDir[len(fromDir)-1] != '/':
0199         fromDir += '/'
0200
0201     if toDir[len(toDir)-1] != '/':
0202         toDir += '/'
0203
0204     for file in listOfFiles:
0205         if os.path.isfile(toDir+file):
0206             if overwrite:
0207                 print("File " + file + " already exists in destination directory. We will overwrite it.")
0208             else:
0209                 print("File " + file + " already exists in destination directory. We will Keep original file.")
0210                 if not smallList:
0211                     copiedFiles.append(file)
0212                 continue
0213         # copy to local disk
0214         aCommand = cpCommand + 'cp '+ fromDir + file + " " + toDir
0215         print(" >> " + aCommand)
0216         tmpStatus = subprocess.getstatusoutput( aCommand )
0217         if tmpStatus[0] == 0:
0218             copiedFiles.append(file)
0219         else:
0220             print("[cp()]\tERROR: Can't copy file " + file)
0221     return copiedFiles
0222
0223 ########################################################################
0224
0225
0226 ###########################################################################################
0227 # lumi tools CondCore/Utilities/python/timeUnitHelper.py
0228 ###########################################################################################
0229 def pack(high,low):
0230     """pack high,low 32bit unsigned int to one unsigned 64bit long long
0231        Note:the print value of result number may appear signed, if the sign bit is used.
0232     """
0233     h=high<<32
0234     return (h|low)
0235
0236 ###########################################################################################
0237 def unpack(i):
0238     """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
0239     """
0240     high=i>>32
0241     low=i&0xFFFFFFFF
0242     return(high,low)
0243
0244 ###########################################################################################
0245 def unpackLumiid(i):
0246     """unpack 64bit lumiid to dictionary {'run','lumisection'}
0247     """
0248     j=unpack(i)
0249     return {'run':j[0],'lumisection':j[1]}
0250 ###########################################################################################
0251 # end lumi tools
0252 ###########################################################################################
0253
0254 ###########################################################################################
0255 def cmp_list_run(a,b):
0256     if int(a.IOVfirst) < int(b.IOVfirst): return -1
0257     if int(a.IOVfirst) == int(b.IOVfirst): return 0
0258     if int(a.IOVfirst) > int(b.IOVfirst): return 1
0259
0260 ###########################################################################################
0261 def cmp_list_lumi(a,b):
0262     if int(a.Run) < int(b.Run): return -1
0263     if int(a.Run) == int(b.Run):
0264         if int(a.IOVfirst) < int(b.IOVfirst): return -1
0265         if int(a.IOVfirst) == int(b.IOVfirst): return 0
0266         if int(a.IOVfirst) > int(b.IOVfirst): return 1
0267     if int(a.Run) > int(b.Run) : return 1
0268
0269 ###########################################################################################
0270 def weight(x1, x1err,x2,x2err):
0271     #print "x1 = "+str(x1)+" +/- "+str(x1err)+" x2 = "+str(x2)+" +/- "+str(x2err)
0272     x1     = float(x1)
0273     x1err  = float(x1err)
0274     x2     = float(x2)
0275     x2err  = float(x2err)
0276     tmperr = 0.
0277     if x2err < 1e-6 :
0278         x2err = 1e-6
0279     if x1err < 1e-6:
0280         x1 = x2/(x2err * x2err)
0281         tmperr = 1/(x2err*x2err)
0282     else:
0283         x1 = x1/(x1err*x1err) + x2/(x2err * x2err)
0284         tmperr = 1/(x1err*x1err) + 1/(x2err*x2err)
0285     x1 = x1/tmperr
0286     x1err = 1/tmperr
0287     x1err = math.sqrt(x1err)
0288     return (str(x1), str(x1err))
0289
0290 ###########################################################################################
0291 def dump( beam, file):
0292     end = "\n"
0293     file.write("Runnumber "+beam.Run+end)
0294     file.write("BeginTimeOfFit "+str(beam.IOVBeginTime)+end)
0295     file.write("EndTimeOfFit "+str(beam.IOVEndTime)+end)
0296     file.write("LumiRange "+str(beam.IOVfirst)+" - "+str(beam.IOVlast)+end)
0297     dumpValues(beam, file)
0298
0299 ###########################################################################################
0300 def dumpValues( beam, file):
0301     end = "\n"
0302     file.write("Type "+str(beam.Type)+end)
0303     file.write("X0 "+str(beam.X)+end)
0304     file.write("Y0 "+str(beam.Y)+end)
0305     file.write("Z0 "+str(beam.Z)+end)
0306     file.write("sigmaZ0 "+str(beam.sigmaZ)+end)
0307     file.write("dxdz "+str(beam.dxdz)+end)
0308     file.write("dydz "+str(beam.dydz)+end)
0309     file.write("BeamWidthX "+beam.beamWidthX+end)
0310     file.write("BeamWidthY "+beam.beamWidthY+end)
0311     file.write("Cov(0,j) "+str(math.pow(float(beam.Xerr),2))+" 0 0 0 0 0 0"  +end)
0312     file.write("Cov(1,j) 0 "+str(math.pow(float(beam.Yerr),2))+" 0 0 0 0 0"  +end)
0313     file.write("Cov(2,j) 0 0 "+str(math.pow(float(beam.Zerr),2))+" 0 0 0 0"  +end)
0314     file.write("Cov(3,j) 0 0 0 "+str(math.pow(float(beam.sigmaZerr),2))+" 0 0 0"  +end)
0315     file.write("Cov(4,j) 0 0 0 0 "+str(math.pow(float(beam.dxdzerr),2))+" 0 0"  +end)
0316     file.write("Cov(5,j) 0 0 0 0 0 "+str(math.pow(float(beam.dydzerr),2))+" 0"  +end)
0317     file.write("Cov(6,j) 0 0 0 0 0 0 "+str(math.pow(float(beam.beamWidthXerr),2))  +end)
0318     file.write("EmittanceX 0"+end)
0319     file.write("EmittanceY 0"+end)
0320     file.write("BetaStar 0"+end)
0321
0322 ###########################################################################################
0323 def delta(x,xerr,nextx,nextxerr):
0324     #return math.fabs( float(x) - float(nextx) )/math.sqrt(math.pow(float(xerr),2) + math.pow(float(nextxerr),2))
0325     return ( float(x) - float(nextx), math.sqrt(math.pow(float(xerr),2) + math.pow(float(nextxerr),2)) )
0326
0327 def deltaSig( x ):
0328     return math.fabs(x[0])/x[1]
0329
0330 ###########################################################################################
0331 def readBeamSpotFile(fileName,listbeam=[],IOVbase="runbase", firstRun='1',lastRun='4999999999'):
0332     tmpbeam = BeamSpot()
0333     tmpbeamsize = 0
0334
0335     #firstRun = "1"
0336     #lastRun  = "4999999999"
0337     if IOVbase == "lumibase" and firstRun=='1' and lastRun=='4999999999' :
0338         firstRun = "1:1"
0339         lastRun = "4999999999:4999999999"
0340
0341     inputfiletype = 0
0342     #print "first = " +firstRun
0343     #print "last = " +lastRun
0344
0345     # for bx
0346     maplist = {}
0347     hasBX = False
0348
0349     tmpfile = open(fileName)
0350     atmpline = tmpfile.readline()
0351     if atmpline.find('Runnumber') != -1:
0352         inputfiletype = 1
0353         if len(atmpline.split()) > 2:
0354             hasBX = True
0355             print(" Input data has been calculated as function of BUNCH CROSSINGS.")
0356     tmpfile.seek(0)
0357
0358
0359     if inputfiletype ==1:
0360
0361         tmpBX = 0
0362         for line in tmpfile:
0363
0364             if line.find('Type') != -1:
0365                 tmpbeam.Type = int(line.split()[1])
0366                 tmpbeamsize += 1
0367             if line.find('X0') != -1:
0368                 tmpbeam.X = line.split()[1]
0369                 #tmpbeam.Xerr = line.split()[4]
0370                 tmpbeamsize += 1
0371             #print " x = " + str(tmpbeam.X)
0372             if line.find('Y0') != -1:
0373                 tmpbeam.Y = line.split()[1]
0374                 #tmpbeam.Yerr = line.split()[4]
0375                 tmpbeamsize += 1
0376             #print " y =" + str(tmpbeam.Y)
0377             if line.find('Z0') != -1 and line.find('sigmaZ0') == -1:
0378                 tmpbeam.Z = line.split()[1]
0379                 #tmpbeam.Zerr = line.split()[4]
0380                 tmpbeamsize += 1
0381             if line.find('sigmaZ0') !=-1:
0382                 tmpbeam.sigmaZ = line.split()[1]
0383                 #tmpbeam.sigmaZerr = line.split()[5]
0384                 tmpbeamsize += 1
0385             if line.find('dxdz') != -1:
0386                 tmpbeam.dxdz = line.split()[1]
0387                 #tmpbeam.dxdzerr = line.split()[4]
0388                 tmpbeamsize += 1
0389             if line.find('dydz') != -1:
0390                 tmpbeam.dydz = line.split()[1]
0391                 #tmpbeam.dydzerr = line.split()[4]
0392                 tmpbeamsize += 1
0393             if line.find('BeamWidthX') != -1:
0394                 tmpbeam.beamWidthX = line.split()[1]
0395                 #tmpbeam.beamWidthXerr = line.split()[6]
0396                 tmpbeamsize += 1
0397             if line.find('BeamWidthY') != -1:
0398                 tmpbeam.beamWidthY = line.split()[1]
0399                 #tmpbeam.beamWidthYerr = line.split()[6]
0400                 tmpbeamsize += 1
0401             if line.find('Cov(0,j)') != -1:
0402                 tmpbeam.Xerr = str(math.sqrt( float( line.split()[1] ) ) )
0403                 tmpbeamsize += 1
0404             if line.find('Cov(1,j)') != -1:
0405                 tmpbeam.Yerr = str(math.sqrt( float( line.split()[2] ) ) )
0406                 tmpbeamsize += 1
0407             if line.find('Cov(2,j)') != -1:
0408                 tmpbeam.Zerr = str(math.sqrt( float( line.split()[3] ) ) )
0409                 tmpbeamsize += 1
0410             if line.find('Cov(3,j)') != -1:
0411                 tmpbeam.sigmaZerr = str(math.sqrt( float( line.split()[4] ) ) )
0412                 tmpbeamsize += 1
0413             if line.find('Cov(4,j)') != -1:
0414                 tmpbeam.dxdzerr = str(math.sqrt( float( line.split()[5] ) ) )
0415                 tmpbeamsize += 1
0416             if line.find('Cov(5,j)') != -1:
0417                 tmpbeam.dydzerr = str(math.sqrt( float( line.split()[6] ) ) )
0418                 tmpbeamsize += 1
0419             if line.find('Cov(6,j)') != -1:
0420                 tmpbeam.beamWidthXerr = str(math.sqrt( float( line.split()[7] ) ) )
0421                 tmpbeam.beamWidthYerr = tmpbeam.beamWidthXerr
0422                 tmpbeamsize += 1
0423             if line.find('LumiRange')  != -1:
0424                 if IOVbase=="lumibase":
0425                     tmpbeam.IOVfirst = line.split()[1]
0426                     tmpbeam.IOVlast = line.split()[3]
0427                 tmpbeamsize += 1
0428             if line.find('Runnumber') != -1:
0429                 tmpbeam.Run = line.split()[1]
0430                 if IOVbase == "runbase":
0431                     tmpbeam.IOVfirst = line.split()[1]
0432                     tmpbeam.IOVlast = line.split()[1]
0433                 if hasBX:
0434                     tmpBX = line.split()[3]
0435                 tmpbeamsize += 1
0436             if line.find('BeginTimeOfFit') != -1:
0437                 tmpbeam.IOVBeginTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
0438                 if IOVbase =="timebase":
0439                     tmpbeam.IOVfirst =  time.mktime( time.strptime(line.split()[1] +  " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
0440                 tmpbeamsize += 1
0441             if line.find('EndTimeOfFit') != -1:
0442                 tmpbeam.IOVEndTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
0443                 if IOVbase =="timebase":
0444                     tmpbeam.IOVlast = time.mktime( time.strptime(line.split()[1] +  " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
0445                 tmpbeamsize += 1
0446             if tmpbeamsize == 20:
0447                 if IOVbase=="lumibase":
0448                     tmprunfirst = int(firstRun.split(":")[0])
0449                     tmprunlast  = int(lastRun.split(":")[0])
0450                     tmplumifirst = int(firstRun.split(":")[1])
0451                     tmplumilast  = int(lastRun.split(":")[1])
0452                     acceptiov1 = acceptiov2 = False
0453                     # check lumis in the same run
0454                     if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
0455                         if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
0456                             acceptiov1 = acceptiov2 = True
0457                     # if different runs make sure you select the correct range of lumis
0458                     elif int(tmpbeam.Run) == tmprunfirst:
0459                         if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
0460                     elif int(tmpbeam.Run) == tmprunlast:
0461                         if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
0462                     elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
0463                         acceptiov1 = acceptiov2 = True
0464
0465                     if acceptiov1 and acceptiov2:
0466                         if tmpbeam.Type != 2:
0467                             print("invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0468                         elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0469                             print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0470                         elif hasBX:
0471                             if (tmpBX in maplist) == False:
0472                                 maplist[tmpBX] = [tmpbeam]
0473                             else:
0474                                 maplist[tmpBX].append(tmpbeam)
0475                         else:
0476                             listbeam.append(tmpbeam)
0477
0478                 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
0479                     if tmpbeam.Type != 2:
0480                         print("invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0481                     elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0482                         print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0483                     else:
0484                         listbeam.append(tmpbeam)
0485
0486                 tmpbeamsize = 0
0487                 tmpbeam = BeamSpot()
0488                 tmpBX = 0
0489     else:
0490
0491         for line in tmpfile:
0492
0493             if line.find('X0') != -1:
0494                 tmpbeam.X = line.split()[2]
0495                 tmpbeam.Xerr = line.split()[4]
0496                 tmpbeamsize += 1
0497             #print " x = " + str(tmpbeam.X)
0498             if line.find('Y0') != -1:
0499                 tmpbeam.Y = line.split()[2]
0500                 tmpbeam.Yerr = line.split()[4]
0501                 tmpbeamsize += 1
0502             #print " y =" + str(tmpbeam.Y)
0503             if line.find('Z0') != -1 and line.find('Sigma Z0') == -1:
0504                 tmpbeam.Z = line.split()[2]
0505                 tmpbeam.Zerr = line.split()[4]
0506                 tmpbeamsize += 1
0507             #print " z =" + str(tmpbeam.Z)
0508             if line.find('Sigma Z0') !=-1:
0509                 tmpbeam.sigmaZ = line.split()[3]
0510                 tmpbeam.sigmaZerr = line.split()[5]
0511                 tmpbeamsize += 1
0512             if line.find('dxdz') != -1:
0513                 tmpbeam.dxdz = line.split()[2]
0514                 tmpbeam.dxdzerr = line.split()[4]
0515                 tmpbeamsize += 1
0516             if line.find('dydz') != -1:
0517                 tmpbeam.dydz = line.split()[2]
0518                 tmpbeam.dydzerr = line.split()[4]
0519                 tmpbeamsize += 1
0520             if line.find('Beam Width X') != -1:
0521                 tmpbeam.beamWidthX = line.split()[4]
0522                 tmpbeam.beamWidthXerr = line.split()[6]
0523                 tmpbeamsize += 1
0524             if line.find('Beam Width Y') != -1:
0525                 tmpbeam.beamWidthY = line.split()[4]
0526                 tmpbeam.beamWidthYerr = line.split()[6]
0527                 tmpbeamsize += 1
0528         #if line.find('Run ') != -1:
0529             if line.find('for runs')  != -1:
0530             #tmpbeam.IOVfirst = line.split()[6].strip(',')
0531                 tmpbeam.Run      = line.split()[2]
0532                 if IOVbase == "runbase":
0533                     tmpbeam.IOVfirst = line.split()[2]
0534                     tmpbeam.IOVlast = line.split()[4]
0535                 tmpbeamsize += 1
0536             if line.find('LumiSection')  != -1:
0537                 if IOVbase=="lumibase":
0538                     tmpbeam.IOVfirst = line.split()[10]
0539                     tmpbeam.IOVlast = line.split()[10]
0540                 tmpbeamsize += 1
0541             if tmpbeamsize == 10:
0542
0543                 if IOVbase=="lumibase":
0544                     tmprunfirst = int(firstRun.split(":")[0])
0545                     tmprunlast  = int(lastRun.split(":")[0])
0546                     tmplumifirst = int(firstRun.split(":")[1])
0547                     tmplumilast  = int(lastRun.split(":")[1])
0548                     acceptiov1 = acceptiov2 = False
0549                     # check lumis in the same run
0550                     if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
0551                         if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
0552                             acceptiov1 = acceptiov2 = True
0553                     # if different runs make sure you select the correct range of lumis
0554                     elif int(tmpbeam.Run) == tmprunfirst:
0555                         if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
0556                     elif int(tmpbeam.Run) == tmprunlast:
0557                         if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
0558                     elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
0559                         acceptiov1 = acceptiov2 = True
0560
0561                     if acceptiov1 and acceptiov2:
0562                         if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0563                             print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0564                         elif hasBX:
0565                             if (tmpBX in maplist) == False:
0566                                 maplist[tmpBX] = [tmpbeam]
0567                             else:
0568                                 maplist[tmpBX].append(tmpbeam)
0569                         else:
0570                             listbeam.append(tmpbeam)
0571
0572                 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
0573                     if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0574                         print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0575                     else:
0576                         listbeam.append(tmpbeam)
0577
0578                 tmpbeamsize = 0
0579                 tmpbeam = BeamSpot()
0580                 tmpBX = 0
0581
0582     tmpfile.close()
0583     print(" got total number of IOVs = " + str(len(listbeam)) + " from file " + fileName)
0584     #print " run " + str(listbeam[3].IOVfirst ) + " " + str( listbeam[3].X )
0585     if hasBX:
0586         return maplist
0587     else:
0588         return listbeam
0589
0590 ###########################################################################################
0591 # Sort and clean list of data for consecutive duplicates and bad fits
0592 def sortAndCleanBeamList(listbeam=[],IOVbase="lumibase"):
0593     # sort the list
0594     if IOVbase == "lumibase":
0595         listbeam.sort( cmp = cmp_list_lumi )
0596     else:
0597         listbeam.sort( cmp = cmp_list_run )
0598
0599     # first clean list of data for consecutive duplicates and bad fits
0600     tmpremovelist = []
0601     for ii in range(0,len(listbeam)):
0602         ibeam = listbeam[ii]
0603         datax = ibeam.IOVfirst
0604         #print str(ii) + "  " +datax
0605         if datax == '0' and IOVbase =="runbase":
0606             print(" iov = 0? skip this IOV = "+ str(ibeam.IOVfirst) + " to " + str(ibeam.IOVlast))
0607             tmpremovelist.append(ibeam)
0608
0609         if ii < len(listbeam) -1:
0610             #print listbeam[ii+1].IOVfirst
0611             if IOVbase =="lumibase":
0612                 if ibeam.Run == listbeam[ii+1].Run and ibeam.IOVfirst == listbeam[ii+1].IOVfirst:
0613                     print(" duplicate IOV = "+datax+", keep only last duplicate entry")
0614                     tmpremovelist.append(ibeam)
0615             elif datax == listbeam[ii+1].IOVfirst:
0616                 print(" duplicate IOV = "+datax+", keep only last duplicate entry")
0617                 tmpremovelist.append(ibeam)
0618
0619     for itmp in tmpremovelist:
0620         listbeam.remove(itmp)
0621
0622 ###########################################################################################
0623 # CREATE FILE FOR PAYLOADS
0624 def createWeightedPayloads(fileName,listbeam=[],weighted=True):
0625     newlistbeam = []
0626     tmpbeam = BeamSpot()
0627     docreate = True
0628     countlumi = 0
0629     tmprun = ""
0630     maxNlumis = 60
0631     if weighted:
0632         maxNlumis = 999999999
0633     for ii in range(0,len(listbeam)):
0634         ibeam = listbeam[ii]
0635         inextbeam = BeamSpot()
0636         iNNbeam = BeamSpot()
0637         if docreate:
0638             tmpbeam.IOVfirst = ibeam.IOVfirst
0639             tmpbeam.IOVBeginTime = ibeam.IOVBeginTime
0640             tmpbeam.Run = ibeam.Run
0641             tmpbeam.Type = 2
0642         docheck = False
0643         docreate = False
0644         #print "Currently testing ii="+str(ii)+" Lumi1: "+str(ibeam.IOVfirst)
0645
0646         # check last iov
0647         if ii < len(listbeam) - 1:
0648             inextbeam = listbeam[ii+1]
0649             docheck = True
0650             if ii < len(listbeam) -2:
0651                 iNNbeam = listbeam[ii+2]
0652         else:
0653             print("close payload because end of data has been reached. Run "+ibeam.Run)
0654             docreate = True
0655         # check we run over the same run
0656         if ibeam.Run != inextbeam.Run:
0657             print("close payload because end of run "+ibeam.Run)
0658             docreate = True
0659         # check maximum lumi counts
0660         if countlumi == maxNlumis -1:
0661             print("close payload because maximum lumi sections accumulated within run "+ibeam.Run)
0662             docreate = True
0663             countlumi = 0
0664         # weighted average position
0665         (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
0666         (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
0667         (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
0668         (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
0669         (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
0670         (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
0671         #print "wx = " + ibeam.beamWidthX + " err= "+ ibeam.beamWidthXerr
0672         (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
0673         (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
0674
0675         if weighted:
0676             docheck = False
0677         # check offsets
0678         #if False:
0679         if docheck:
0680
0681             # define minimum limit
0682             min_limit = 0.0025
0683
0684             # limit for x and y
0685             limit = float(ibeam.beamWidthX)/2.
0686             if limit < min_limit: limit = min_limit
0687
0688             # check movements in X
0689             adelta1 = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr)
0690             adelta2 = (0.,1.e9)
0691             adelta1dxdz = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
0692             adelta2dxdz = (0.,1.e9)
0693             adelta1dydz = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
0694             adelta2dydz = (0.,1.e9)
0695             adelta1widthx = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
0696             adelta2widthx = (0.,1.e9)
0697             adelta1widthy = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
0698             adelta2widthy = (0.,1.e9)
0699             adelta1z0 = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr)
0700             adelta1sigmaZ = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr)
0701
0702             if iNNbeam.Type != -1:
0703                 adelta2 = delta(inextbeam.X, inextbeam.Xerr, iNNbeam.X, iNNbeam.Xerr)
0704                 adelta2dxdz = delta(inextbeam.dxdz, inextbeam.dxdzerr, iNNbeam.dxdz, iNNbeam.dxdzerr)
0705                 adelta2dydz = delta(inextbeam.dydz, inextbeam.dydzerr, iNNbeam.dydz, iNNbeam.dydzerr)
0706                 adelta2widthx = delta(inextbeam.beamWidthX, inextbeam.beamWidthXerr, iNNbeam.beamWidthX, iNNbeam.beamWidthXerr)
0707                 adelta2widthy = delta(inextbeam.beamWidthY, inextbeam.beamWidthYerr, iNNbeam.beamWidthY, iNNbeam.beamWidthYerr)
0708
0709             deltaX = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
0710             if ii < len(listbeam) -2:
0711                 if deltaX==False and adelta1[0]*adelta2[0] > 0. and  math.fabs(adelta1[0]+adelta2[0]) >= limit:
0712                     #print " positive, "+str(adelta1[0]+adelta2[0])+ " limit="+str(limit)
0713                     deltaX = True
0714                 elif deltaX==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
0715                     deltaX = False
0716                     #print " negative, "+str(adelta1[0]/adelta2[0])
0717                 #else:
0718                 #    print str(adelta1[0]/adelta2[0])
0719
0720             # check movemnts in Y
0721             adelta1 = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr)
0722             adelta2 = (0.,1.e9)
0723             if iNNbeam.Type != -1:
0724                 adelta2 = delta(inextbeam.Y, inextbeam.Yerr, iNNbeam.Y, iNNbeam.Yerr)
0725
0726             deltaY = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
0727             if ii < len(listbeam) -2:
0728                 if deltaY==False and adelta1[0]*adelta2[0] > 0. and  math.fabs(adelta1[0]+adelta2[0]) >= limit:
0729                     deltaY = True
0730                 elif deltaY==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
0731                     deltaY = False
0732             # check movements in Z
0733
0734             limit = float(ibeam.sigmaZ)/2.
0735             deltaZ = deltaSig(adelta1z0) > 3.5 and math.fabs(adelta1z0[0]) >= limit
0736
0737             deltasigmaZ = deltaSig(adelta1sigmaZ) > 5.0
0738
0739             # check dxdz
0740             adelta = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
0741             deltadxdz   = deltaSig(adelta) > 5.0
0742             if deltadxdz and adelta1dxdz[0]*adelta2dxdz[0]<=0 and adelta2dxdz[0] != 0 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) > 0.33 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) < 3:
0743                 deltadxdz = False
0744             # check dydz
0745             adelta = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
0746             deltadydz   = deltaSig(adelta) > 5.0
0747             if deltadydz and adelta1dydz[0]*adelta2dydz[0]<=0 and adelta2dydz[0] != 0 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) > 0.33 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) < 3:
0748                 deltadydz = False
0749
0750             adelta = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
0751             deltawidthX = deltaSig(adelta) > 5
0752             if deltawidthX and adelta1widthx[0]*adelta2widthx[0]<=0 and adelta2widthx[0] != 0 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) > 0.33 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) < 3:
0753                 deltawidthX = False
0754
0755             adelta = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
0756             deltawidthY = deltaSig(adelta) > 5
0757             if deltawidthY and adelta1widthy[0]*adelta2widthy[0]<=0 and adelta2widthy[0] != 0 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) > 0.33 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) < 3:
0758                 deltawidthY = False
0759             #if iNNbeam.Type != -1:
0760             #    deltaX = deltaX and delta(ibeam.X, ibeam.Xerr, iNNbeam.X, iNNbeam.Xerr) > 1.5
0761             #    deltaY = deltaY and delta(ibeam.Y, ibeam.Yerr, iNNbeam.Y, iNNbeam.Yerr) > 1.5
0762             #    deltaZ = deltaZ and delta(ibeam.Z, ibeam.Zerr, iNNbeam.Z, iNNbeam.Zerr) > 1.5
0763             #
0764             #    deltasigmaZ = deltasigmaZ and delta(ibeam.sigmaZ, ibeam.sigmaZerr, iNNbeam.sigmaZ, iNNbeam.sigmaZerr) > 2.5
0765             #    deltadxdz   = deltadxdz and delta(ibeam.dxdz, ibeam.dxdzerr, iNNbeam.dxdz, iNNbeam.dxdzerr) > 2.5
0766             #    deltadydz   = deltadydz and delta(ibeam.dydz, ibeam.dydzerr, iNNbeam.dydz, iNNbeam.dydzerr) > 2.5
0767             #
0768             #    deltawidthX = deltawidthX and delta(ibeam.beamWidthX, ibeam.beamWidthXerr, iNNbeam.beamWidthX, iNNbeam.beamWidthXerr) > 3
0769             #    deltawidthY = deltawidthY and delta(ibeam.beamWidthY, ibeam.beamWidthYerr, iNNbeam.beamWidthY, iNNbeam.beamWidthYerr) > 3
0770
0771             if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
0772                 docreate = True
0773                 #print "shift here: x="+str(deltaX)+" y="+str(deltaY)
0774                 #print "x1 = "+ibeam.X + " x1err = "+ibeam.Xerr
0775                 #print "x2 = "+inextbeam.X + " x2err = "+inextbeam.Xerr
0776                 #print "Lumi1: "+str(ibeam.IOVfirst) + " Lumi2: "+str(inextbeam.IOVfirst)
0777                 #print " x= "+ibeam.X+" +/- "+ibeam.Xerr
0778                 #print "weighted average x = "+tmpbeam.X +" +//- "+tmpbeam.Xerr
0779                 print("close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY))
0780         if docreate:
0781             #if ii == len(listbeam)-1:
0782             tmpbeam.IOVlast = ibeam.IOVlast
0783             tmpbeam.IOVEndTime = ibeam.IOVEndTime
0784             print("  Run: "+tmpbeam.Run +" Lumi1: "+str(tmpbeam.IOVfirst) + " Lumi2: "+str(tmpbeam.IOVlast))
0785             newlistbeam.append(tmpbeam)
0786             tmpbeam = BeamSpot()
0787             countlumi = 0
0788         tmprun = ibeam.Run
0789         countlumi += 1
0790
0791     payloadfile = open(fileName,"w")
0792     for iload in newlistbeam:
0793         dump( iload, payloadfile )
0794     payloadfile.close()
0795     return newlistbeam
0796 ###########################################################################################
0797 def createWeightedPayloadsNew(fileName,listbeam=[],weighted=True):
0798     newlistbeam = []
0799     docreate = False
0800     docheck = False
0801     lastPayload = listbeam[0]
0802
0803     firstToUse = 0
0804     lastToUse = 0
0805     for ii in range(0,len(listbeam)):
0806         docreate = False
0807         if docheck:
0808             deltaX = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr) > 1.5
0809             deltaY = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr) > 1.5
0810             deltaZ = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr) > 2.5
0811
0812             deltasigmaZ = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr) > 2.5
0813             deltadxdz   = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr) > 2.5
0814             deltadydz   = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr) > 2.5
0815
0816             deltawidthX = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr) > 3
0817             deltawidthY = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr) > 3
0818
0819             #if iNNbeam.Type != -1:
0820             #    deltaX = deltaX and delta(ibeam.X, ibeam.Xerr, iNNbeam.X, iNNbeam.Xerr) > 1.5
0821             #    deltaY = deltaY and delta(ibeam.Y, ibeam.Yerr, iNNbeam.Y, iNNbeam.Yerr) > 1.5
0822             #    deltaZ = deltaZ and delta(ibeam.Z, ibeam.Zerr, iNNbeam.Z, iNNbeam.Zerr) > 1.5
0823             #
0824             #    deltasigmaZ = deltasigmaZ and delta(ibeam.sigmaZ, ibeam.sigmaZerr, iNNbeam.sigmaZ, iNNbeam.sigmaZerr) > 2.5
0825             #    deltadxdz   = deltadxdz and delta(ibeam.dxdz, ibeam.dxdzerr, iNNbeam.dxdz, iNNbeam.dxdzerr) > 2.5
0826             #    deltadydz   = deltadydz and delta(ibeam.dydz, ibeam.dydzerr, iNNbeam.dydz, iNNbeam.dydzerr) > 2.5
0827             #
0828             #    deltawidthX = deltawidthX and delta(ibeam.beamWidthX, ibeam.beamWidthXerr, iNNbeam.beamWidthX, iNNbeam.beamWidthXerr) > 3
0829             #    deltawidthY = deltawidthY and delta(ibeam.beamWidthY, ibeam.beamWidthYerr, iNNbeam.beamWidthY, iNNbeam.beamWidthYerr) > 3
0830
0831             if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
0832                 if ii != 0:
0833                     docreate = True
0834                     lastToUse = ii-1
0835                 #print "shift here: x="+str(deltaX)+" y="+str(deltaY)
0836                 #print "x1 = "+ibeam.X + " x1err = "+ibeam.Xerr
0837                 #print "x2 = "+inextbeam.X + " x2err = "+inextbeam.Xerr
0838                 #print "Lumi1: "+str(ibeam.IOVfirst) + " Lumi2: "+str(inextbeam.IOVfirst)
0839                 #print " x= "+ibeam.X+" +/- "+ibeam.Xerr
0840                 #print "weighted average x = "+tmpbeam.X +" +//- "+tmpbeam.Xerr
0841                 print("close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY))
0842
0843         #WARNING this will only be fine for Run based IOVs
0844         if ii >= len(listbeam) - 1 or listbeam[ii].Run != listbeam[ii+1].Run :
0845             print("close payload because end of run has been reached. Run " + listbeam[ii].Run)
0846             docreate = True
0847             lastToUse = ii
0848
0849
0850          # check maximum lumi counts
0851 #?        if countlumi == maxNlumis:
0852 #?            print "close payload because maximum lumi sections accumulated within run "+ibeam.Run
0853 #?            docreate = True
0854 #?            countlumi = 0
0855         if docreate:
0856             tmpbeam = BeamSpot()
0857             for ibeam in listbeam[firstToUse:lastToUse]:
0858                 (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
0859                 (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
0860                 (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
0861                 (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
0862                 (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
0863                 (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
0864                 #print "wx = " + ibeam.beamWidthX + " err= "+ ibeam.beamWidthXerr
0865                 (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
0866                 (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
0867             tmpbeam.IOVfirst     = listbeam[firstToUse].IOVfirst
0868             tmpbeam.IOVBeginTime = listbeam[firstToUse].IOVBeginTime
0869             tmpbeam.Run          = listbeam[firstToUse].Run
0870             tmpbeam.Type         = 2
0871             tmpbeam.IOVlast      = listbeam[lastToUse].IOVlast
0872             tmpbeam.IOVEndTime   = listbeam[lastToUse].IOVEndTime
0873             newlistbeam.append(tmpbeam)
0874             firstToUse = lastToUse+1
0875             print("Run: " + tmpbeam.Run + " Lumi1: " + str(tmpbeam.IOVfirst) + " Lumi2: " + str(tmpbeam.IOVlast))
0876
0877     payloadfile = open(fileName,"w")
0878     for iload in newlistbeam:
0879         dump( iload, payloadfile )
0880     payloadfile.close()
0881
0882 ###########################################################################################
0883 def writeSqliteFile(sqliteFileName,tagName,timeType,beamSpotFile,sqliteTemplateFile,tmpDir="/tmp/"):
0884     writeDBOut = tmpDir + "write2DB_" + tagName + ".py"
0885     wFile      = open(sqliteTemplateFile)
0886     wNewFile   = open(writeDBOut,'w')
0887
0888     writeDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
0889                    ('TAGNAME',tagName),
0890                    ('TIMETYPE',timeType),
0891                    ('BEAMSPOTFILE',beamSpotFile)]
0892
0893     for line in wFile:
0894         for itag in writeDBTags:
0895             line = line.replace(itag[0],itag[1])
0896         wNewFile.write(line)
0897
0898     wNewFile.close()
0899     print("writing sqlite file ...")
0900     status_wDB = subprocess.getstatusoutput('cmsRun '+ writeDBOut)
0901     print(status_wDB[1])
0902
0903     os.system("rm -f " + writeDBOut)
0904     return not status_wDB[0]
0905
0906 ###########################################################################################
0907 def readSqliteFile(sqliteFileName,tagName,sqliteTemplateFile,tmpDir="/tmp/"):
0908     readDBOut = tmpDir + "readDB_" + tagName + ".py"
0909
0910     rFile = open(sqliteTemplateFile)
0911     rNewFile = open(readDBOut,'w')
0912
0913     readDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
0914                   ('TAGNAME',tagName)]
0915
0916     for line in rFile:
0917         for itag in readDBTags:
0918             line = line.replace(itag[0],itag[1])
0919         rNewFile.write(line)
0920
0921     rNewFile.close()
0922     status_rDB = subprocess.getstatusoutput('cmsRun '+ readDBOut)
0923
0924     outtext = status_rDB[1]
0925     print(outtext)
0926     os.system("rm -f " + readDBOut)
0927     return not status_rDB[0]
0928
0929 ###########################################################################################
0930 def appendSqliteFile(combinedSqliteFileName, sqliteFileName, tagName, IOVSince, IOVTill ,tmpDir="/tmp/"):
0931     aCommand = "conddb_import -c sqlite_file:" + tmpDir + combinedSqliteFileName + " -f sqlite_file:" + sqliteFileName + " -i " + tagName + " -t " + tagName + " -b " + IOVSince + " -e " + IOVTill
0932     print(aCommand)
0933     std = subprocess.getstatusoutput(aCommand)
0934     print(std[1])
0935     return not std[0]
0936
0937 ###########################################################################################
0938 def uploadSqliteFile(sqliteFileDirName, sqliteFileName, dropbox="/DropBox"):
0939     # Changing permissions to metadata
0940     acmd = "chmod a+w " + sqliteFileDirName + sqliteFileName + ".txt"
0941     outcmd = subprocess.getstatusoutput(acmd)
0942     print(acmd)
0943 #    print outcmd[1]
0944     if outcmd[0]:
0945         print("Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".txt")
0946         return False
0947
0948     acmd = "cp " + sqliteFileDirName + sqliteFileName + ".db " + sqliteFileDirName + sqliteFileName + ".txt ."
0949     print(acmd)
0950     outcmd = subprocess.getstatusoutput(acmd)
0951     print(outcmd[1])
0952     if outcmd[0]:
0953         print("Couldn't cd to " + sqliteFileDirName)
0954         return False
0955
0956     acmd = "tar -cvjf " + sqliteFileName + ".tar.bz2 " + sqliteFileName + ".db " + sqliteFileName + ".txt"
0957     print(acmd)
0958     outcmd = subprocess.getstatusoutput(acmd)
0959     print(outcmd[1])
0960     if outcmd[0]:
0961         print("Couldn't zip the files!")
0962         return False
0963
0964     acmd = "chmod a+w " + sqliteFileName + ".tar.bz2"
0965     outcmd = subprocess.getstatusoutput(acmd)
0966     print(acmd)
0967 #    print outcmd[1]
0968     if outcmd[0]:
0969         print("Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".tar.bz2")
0970         return False
0971
0972     acmd = "scp -p " + sqliteFileName + ".tar.bz2" + " webcondvm.cern.ch:" + dropbox
0973     print(acmd)
0974     outcmd = subprocess.getstatusoutput(acmd)
0975     print(outcmd[1])
0976     if outcmd[0]:
0977         print("Couldn't scp the files to DropBox!")
0978         return False
0979
0980
0981     acmd = "mv " + sqliteFileName + ".tar.bz2 " + sqliteFileDirName
0982     print(acmd)
0983     outcmd = subprocess.getstatusoutput(acmd)
0984     print(outcmd[1])
0985     if outcmd[0]:
0986         print("Couldn't mv the file to " + sqliteFileDirName)
0987         return False
0988
0989     acmd = "rm " + sqliteFileName + ".db " + sqliteFileName + ".txt"
0990     print(acmd)
0991     outcmd = subprocess.getstatusoutput(acmd)
0992     print(outcmd[1])
0993     if outcmd[0]:
0994         print("Couldn't rm the db and txt files")
0995         return False
0996
0997 #    acmd = "scp -p " + sqliteFileDirName + sqliteFileName + ".txt webcondvm.cern.ch:/tmp"
0998 #    outcmd = subprocess.getstatusoutput(acmd)
0999 #    print acmd
1000 #    print outcmd[1]
1001 #    if outcmd[0]:
1002 #        print "Can't change permission to file: " + sqliteFileName + ".txt"
1003 #        return False
1004
1005 #    acmd = "ssh webcondvm.cern.ch \"mv /tmp/" + sqliteFileName + ".db /tmp/" + sqliteFileName + ".txt " + dropbox +"\""
1006 #    print acmd
1007 #    outcmd = subprocess.getstatusoutput(acmd)
1008 #    print outcmd[1]
1009 #    if outcmd[0]:
1010 #        print "Can't move files from tmp to dropbox!"
1011         return False
1012
1013 #    acmd = "ssh webcondvm.cern.ch \"mv /tmp/" + final_sqlite_file_name + ".txt "+dropbox +"\""
1014 #    outcmd = subprocess.getstatusoutput(acmd)
1015 #    print acmd
1016 #    print outcmd[1]
1017 #    if outcmd[0]:
1018 #        print "Can't change permission to file: " + sqliteFileName + ".txt"
1019 #        return False
1020
1021     return True
1022