BeamSpotProducer/scripts/CommonMethods.py

0001 from builtins import range
0002 import math, re, optparse, subprocess, os, sys, time, datetime
0003 from BeamSpotObj import BeamSpot
0004 from IOVObj import IOV
0005
0006 lockFile = ".lock"
0007
0008 ###########################################################################################
0009 def timeoutManager(type,timeout=-1,fileName=".timeout"):
0010     if timeout == 0:
0011         return 1
0012     timeFormat = "%a,%Y/%m/%d,%H:%M:%S"
0013     currentTime = time.gmtime()
0014     timeoutLine = type + ' ' + time.strftime(timeFormat, currentTime) + '\n'
0015     isTimeout = False
0016     alreadyThere = False
0017     timeoutType = -1;
0018     fileExist = os.path.isfile(fileName)
0019     text = ''
0020     fields = []
0021     reset = False
0022     if timeout == -1:
0023         reset = True
0024     if fileExist:
0025         file = open(fileName)
0026         for line in file:
0027             text += line
0028             fields = line.strip('\n').split(' ')
0029             if fields[0] == type:
0030                 alreadyThere = True
0031                 if reset:
0032                     text = text.replace(line,'')
0033                     continue
0034
0035                 fileTime = time.strptime(fields[1],timeFormat)
0036                 myTime = time.mktime(fileTime)
0037                 referenceTime = time.mktime(time.gmtime())
0038                 daylight = 0
0039                 if currentTime.tm_isdst == 0:
0040                     daylight = 3600
0041                 elapsedTime = referenceTime-myTime-daylight
0042                 if elapsedTime > timeout:
0043                     isTimeout = True
0044                     timeoutType = 1
0045                     print("Timeout! " + str(elapsedTime) + " seconds passed since the " + type + " timeout was set and you can't tolerate more than " + str(timeout) + " seconds!")
0046                 else:
0047                     timeoutType = 0
0048                     print("Timeout of type " + type + " already exist and was generated " + str(elapsedTime) + " seconds ago at " + fields[1])
0049
0050         file.close()
0051
0052     if not fileExist or not alreadyThere and not reset:
0053         timeoutType = -1
0054         text += timeoutLine
0055
0056     if not fileExist or not alreadyThere or isTimeout or (reset and alreadyThere):
0057         if fileExist:
0058             subprocess.getstatusoutput("rm -rf " + fileName)
0059         file = open(fileName,'w')
0060         file.write(text)
0061         file.close()
0062
0063     return timeoutType
0064
0065
0066 ###########################################################################################
0067 def setLockName(name):
0068     global lockFile
0069     lockFile = name
0070
0071 ###########################################################################################
0072 def checkLock():
0073     global lockFile
0074     if os.path.isfile(lockFile):
0075         return True
0076     else:
0077         return False
0078
0079 ###########################################################################################
0080 def lock():
0081     global lockFile
0082     subprocess.getstatusoutput( "touch " + lockFile)
0083
0084 ###########################################################################################
0085 def rmLock():
0086     global lockFile
0087     if checkLock():
0088         subprocess.getstatusoutput( "rm " + lockFile)
0089
0090 ###########################################################################################
0091 def exit(msg=""):
0092     rmLock()
0093     raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
0094
0095 ###########################################################################################
0096 def isnan(num):
0097     fnum = float(num)
0098     return fnum != fnum
0099
0100 ###########################################################################################
0101 # OPTIONS
0102 ###########################################################################################
0103 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
0104
0105 ###########################################################################################
0106 def parse(docstring, arglist=None):
0107     global optionstring
0108     global tagType
0109     optionstring = docstring
0110     match = USAGE.search(optionstring)
0111     if not match: raise ParsingError("Cannot find the option string")
0112     optlines = match.group(1).splitlines()
0113     try:
0114         p = optparse.OptionParser(optlines[0])
0115         for line in optlines[1:]:
0116             opt, help=line.split(':')[:2]
0117             short,long=opt.split(',')[:2]
0118             if '=' in opt:
0119                 action='store'
0120                 long=long.split('=')[0]
0121             else:
0122                 action='store_true'
0123             p.add_option(short.strip(),long.strip(),
0124                          action = action, help = help.strip())
0125     except (IndexError,ValueError):
0126         raise ParsingError("Cannot parse the option string correctly")
0127     return p.parse_args(arglist)
0128
0129 ###########################################################################################
0130 def nonzero(self): # will become the nonzero method of optparse.Values
0131     "True if options were given"
0132     for v in self.__dict__.values():
0133         if v is not None: return True
0134     return False
0135
0136 ###########################################################################################
0137 optparse.Values.__nonzero__ = nonzero # dynamically fix optparse.Values
0138
0139 # END OPTIONS
0140 ###########################################################################################
0141
0142 ###########################################################################################
0143 class ParsingError(Exception): pass
0144
0145 ###########################################################################################
0146 # General utilities
0147 ###########################################################################################
0148 ###########################################################################################
0149 def sendEmail(mailList,error):
0150     print("Sending email to " + mailList + " with body: " + error)
0151     list = mailList.split(',')
0152     for email in list:
0153         p = os.popen("mail -s \"Automatic workflow error\" " + email ,"w")
0154         p.write(error)
0155         status = p.close()
0156
0157 ###########################################################################################
0158 def dirExists(dir):
0159     if dir.find("castor") != -1:
0160         lsCommand = "nsls " + dir
0161         output = subprocess.getstatusoutput( lsCommand )
0162         return not output[0]
0163     else:
0164         return os.path.exists(dir)
0165
0166 ########################################################################
0167 def ls(dir,filter=""):
0168     lsCommand      = ''
0169     listOfFiles    = []
0170     if dir.find('castor') != -1:
0171         lsCommand = 'ns'
0172     elif not os.path.exists(dir):
0173         print("ERROR: File or directory " + dir + " doesn't exist")
0174         return listOfFiles
0175
0176     aCommand  = lsCommand  + 'ls '+ dir
0177     #aCommand  = lsCommand  + 'ls '+ dir + " | grep .txt"
0178     if filter != "":
0179         aCommand  += " | grep " + filter
0180
0181     tmpStatus = subprocess.getstatusoutput( aCommand )
0182     listOfFiles = tmpStatus[1].split('\n')
0183     if len(listOfFiles) == 1:
0184         if listOfFiles[0].find('No such file or directory') != -1:
0185             exit("ERROR: File or directory " + dir + " doesn't exist")
0186
0187     return listOfFiles
0188
0189 ########################################################################
0190 def cp(fromDir,toDir,listOfFiles,overwrite=False,smallList=False):
0191     cpCommand   = ''
0192     copiedFiles = []
0193     if fromDir.find('castor') != -1 or toDir.find('castor') != -1 :
0194         cpCommand = 'rf'
0195     elif fromDir.find('resilient') != -1:
0196         cpCommand = 'dc'
0197     if fromDir[len(fromDir)-1] != '/':
0198         fromDir += '/'
0199
0200     if toDir[len(toDir)-1] != '/':
0201         toDir += '/'
0202
0203     for file in listOfFiles:
0204         if os.path.isfile(toDir+file):
0205             if overwrite:
0206                 print("File " + file + " already exists in destination directory. We will overwrite it.")
0207             else:
0208                 print("File " + file + " already exists in destination directory. We will Keep original file.")
0209                 if not smallList:
0210                     copiedFiles.append(file)
0211                 continue
0212         # copy to local disk
0213         aCommand = cpCommand + 'cp '+ fromDir + file + " " + toDir
0214         print(" >> " + aCommand)
0215         tmpStatus = subprocess.getstatusoutput( aCommand )
0216         if tmpStatus[0] == 0:
0217             copiedFiles.append(file)
0218         else:
0219             print("[cp()]\tERROR: Can't copy file " + file)
0220     return copiedFiles
0221
0222 ########################################################################
0223
0224
0225 ###########################################################################################
0226 # lumi tools CondCore/Utilities/python/timeUnitHelper.py
0227 ###########################################################################################
0228 def pack(high,low):
0229     """pack high,low 32bit unsigned int to one unsigned 64bit long long
0230        Note:the print value of result number may appear signed, if the sign bit is used.
0231     """
0232     h=high<<32
0233     return (h|low)
0234
0235 ###########################################################################################
0236 def unpack(i):
0237     """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
0238     """
0239     high=i>>32
0240     low=i&0xFFFFFFFF
0241     return(high,low)
0242
0243 ###########################################################################################
0244 def unpackLumiid(i):
0245     """unpack 64bit lumiid to dictionary {'run','lumisection'}
0246     """
0247     j=unpack(i)
0248     return {'run':j[0],'lumisection':j[1]}
0249 ###########################################################################################
0250 # end lumi tools
0251 ###########################################################################################
0252
0253 ###########################################################################################
0254 def cmp_list_run(a,b):
0255     if int(a.IOVfirst) < int(b.IOVfirst): return -1
0256     if int(a.IOVfirst) == int(b.IOVfirst): return 0
0257     if int(a.IOVfirst) > int(b.IOVfirst): return 1
0258
0259 ###########################################################################################
0260 def cmp_list_lumi(a,b):
0261     if int(a.Run) < int(b.Run): return -1
0262     if int(a.Run) == int(b.Run):
0263         if int(a.IOVfirst) < int(b.IOVfirst): return -1
0264         if int(a.IOVfirst) == int(b.IOVfirst): return 0
0265         if int(a.IOVfirst) > int(b.IOVfirst): return 1
0266     if int(a.Run) > int(b.Run) : return 1
0267
0268 ###########################################################################################
0269 def weight(x1, x1err,x2,x2err):
0270     #print "x1 = "+str(x1)+" +/- "+str(x1err)+" x2 = "+str(x2)+" +/- "+str(x2err)
0271     x1     = float(x1)
0272     x1err  = float(x1err)
0273     x2     = float(x2)
0274     x2err  = float(x2err)
0275     tmperr = 0.
0276     if x2err < 1e-6 :
0277         x2err = 1e-6
0278     if x1err < 1e-6:
0279         x1 = x2/(x2err * x2err)
0280         tmperr = 1/(x2err*x2err)
0281     else:
0282         x1 = x1/(x1err*x1err) + x2/(x2err * x2err)
0283         tmperr = 1/(x1err*x1err) + 1/(x2err*x2err)
0284     x1 = x1/tmperr
0285     x1err = 1/tmperr
0286     x1err = math.sqrt(x1err)
0287     return (str(x1), str(x1err))
0288
0289 ###########################################################################################
0290 def dump( beam, file):
0291     end = "\n"
0292     file.write("Runnumber "+beam.Run+end)
0293     file.write("BeginTimeOfFit "+str(beam.IOVBeginTime)+end)
0294     file.write("EndTimeOfFit "+str(beam.IOVEndTime)+end)
0295     file.write("LumiRange "+str(beam.IOVfirst)+" - "+str(beam.IOVlast)+end)
0296     dumpValues(beam, file)
0297
0298 ###########################################################################################
0299 def dumpValues( beam, file):
0300     end = "\n"
0301     file.write("Type "+str(beam.Type)+end)
0302     file.write("X0 "+str(beam.X)+end)
0303     file.write("Y0 "+str(beam.Y)+end)
0304     file.write("Z0 "+str(beam.Z)+end)
0305     file.write("sigmaZ0 "+str(beam.sigmaZ)+end)
0306     file.write("dxdz "+str(beam.dxdz)+end)
0307     file.write("dydz "+str(beam.dydz)+end)
0308     file.write("BeamWidthX "+beam.beamWidthX+end)
0309     file.write("BeamWidthY "+beam.beamWidthY+end)
0310     file.write("Cov(0,j) "+str(math.pow(float(beam.Xerr),2))+" 0 0 0 0 0 0"  +end)
0311     file.write("Cov(1,j) 0 "+str(math.pow(float(beam.Yerr),2))+" 0 0 0 0 0"  +end)
0312     file.write("Cov(2,j) 0 0 "+str(math.pow(float(beam.Zerr),2))+" 0 0 0 0"  +end)
0313     file.write("Cov(3,j) 0 0 0 "+str(math.pow(float(beam.sigmaZerr),2))+" 0 0 0"  +end)
0314     file.write("Cov(4,j) 0 0 0 0 "+str(math.pow(float(beam.dxdzerr),2))+" 0 0"  +end)
0315     file.write("Cov(5,j) 0 0 0 0 0 "+str(math.pow(float(beam.dydzerr),2))+" 0"  +end)
0316     file.write("Cov(6,j) 0 0 0 0 0 0 "+str(math.pow(float(beam.beamWidthXerr),2))  +end)
0317     file.write("EmittanceX 0"+end)
0318     file.write("EmittanceY 0"+end)
0319     file.write("BetaStar 0"+end)
0320
0321 ###########################################################################################
0322 def delta(x,xerr,nextx,nextxerr):
0323     #return math.fabs( float(x) - float(nextx) )/math.sqrt(math.pow(float(xerr),2) + math.pow(float(nextxerr),2))
0324     return ( float(x) - float(nextx), math.sqrt(math.pow(float(xerr),2) + math.pow(float(nextxerr),2)) )
0325
0326 def deltaSig( x ):
0327     return math.fabs(x[0])/x[1]
0328
0329 ###########################################################################################
0330 def readBeamSpotFile(fileName,listbeam=[],IOVbase="runbase", firstRun='1',lastRun='4999999999'):
0331     tmpbeam = BeamSpot()
0332     tmpbeamsize = 0
0333
0334     #firstRun = "1"
0335     #lastRun  = "4999999999"
0336     if IOVbase == "lumibase" and firstRun=='1' and lastRun=='4999999999' :
0337         firstRun = "1:1"
0338         lastRun = "4999999999:4999999999"
0339
0340     inputfiletype = 0
0341     #print "first = " +firstRun
0342     #print "last = " +lastRun
0343
0344     # for bx
0345     maplist = {}
0346     hasBX = False
0347
0348     tmpfile = open(fileName)
0349     atmpline = tmpfile.readline()
0350     if atmpline.find('Runnumber') != -1:
0351         inputfiletype = 1
0352         if len(atmpline.split()) > 2:
0353             hasBX = True
0354             print(" Input data has been calculated as function of BUNCH CROSSINGS.")
0355     tmpfile.seek(0)
0356
0357
0358     if inputfiletype ==1:
0359
0360         tmpBX = 0
0361         for line in tmpfile:
0362
0363             if line.find('Type') != -1:
0364                 tmpbeam.Type = int(line.split()[1])
0365                 tmpbeamsize += 1
0366             if line.find('X0') != -1:
0367                 tmpbeam.X = line.split()[1]
0368                 #tmpbeam.Xerr = line.split()[4]
0369                 tmpbeamsize += 1
0370             #print " x = " + str(tmpbeam.X)
0371             if line.find('Y0') != -1:
0372                 tmpbeam.Y = line.split()[1]
0373                 #tmpbeam.Yerr = line.split()[4]
0374                 tmpbeamsize += 1
0375             #print " y =" + str(tmpbeam.Y)
0376             if line.find('Z0') != -1 and line.find('sigmaZ0') == -1:
0377                 tmpbeam.Z = line.split()[1]
0378                 #tmpbeam.Zerr = line.split()[4]
0379                 tmpbeamsize += 1
0380             if line.find('sigmaZ0') !=-1:
0381                 tmpbeam.sigmaZ = line.split()[1]
0382                 #tmpbeam.sigmaZerr = line.split()[5]
0383                 tmpbeamsize += 1
0384             if line.find('dxdz') != -1:
0385                 tmpbeam.dxdz = line.split()[1]
0386                 #tmpbeam.dxdzerr = line.split()[4]
0387                 tmpbeamsize += 1
0388             if line.find('dydz') != -1:
0389                 tmpbeam.dydz = line.split()[1]
0390                 #tmpbeam.dydzerr = line.split()[4]
0391                 tmpbeamsize += 1
0392             if line.find('BeamWidthX') != -1:
0393                 tmpbeam.beamWidthX = line.split()[1]
0394                 #tmpbeam.beamWidthXerr = line.split()[6]
0395                 tmpbeamsize += 1
0396             if line.find('BeamWidthY') != -1:
0397                 tmpbeam.beamWidthY = line.split()[1]
0398                 #tmpbeam.beamWidthYerr = line.split()[6]
0399                 tmpbeamsize += 1
0400             if line.find('Cov(0,j)') != -1:
0401                 tmpbeam.Xerr = str(math.sqrt( float( line.split()[1] ) ) )
0402                 tmpbeamsize += 1
0403             if line.find('Cov(1,j)') != -1:
0404                 tmpbeam.Yerr = str(math.sqrt( float( line.split()[2] ) ) )
0405                 tmpbeamsize += 1
0406             if line.find('Cov(2,j)') != -1:
0407                 tmpbeam.Zerr = str(math.sqrt( float( line.split()[3] ) ) )
0408                 tmpbeamsize += 1
0409             if line.find('Cov(3,j)') != -1:
0410                 tmpbeam.sigmaZerr = str(math.sqrt( float( line.split()[4] ) ) )
0411                 tmpbeamsize += 1
0412             if line.find('Cov(4,j)') != -1:
0413                 tmpbeam.dxdzerr = str(math.sqrt( float( line.split()[5] ) ) )
0414                 tmpbeamsize += 1
0415             if line.find('Cov(5,j)') != -1:
0416                 tmpbeam.dydzerr = str(math.sqrt( float( line.split()[6] ) ) )
0417                 tmpbeamsize += 1
0418             if line.find('Cov(6,j)') != -1:
0419                 tmpbeam.beamWidthXerr = str(math.sqrt( float( line.split()[7] ) ) )
0420                 tmpbeam.beamWidthYerr = tmpbeam.beamWidthXerr
0421                 tmpbeamsize += 1
0422             if line.find('LumiRange')  != -1:
0423                 if IOVbase=="lumibase":
0424                     tmpbeam.IOVfirst = line.split()[1]
0425                     tmpbeam.IOVlast = line.split()[3]
0426                 tmpbeamsize += 1
0427             if line.find('Runnumber') != -1:
0428                 tmpbeam.Run = line.split()[1]
0429                 if IOVbase == "runbase":
0430                     tmpbeam.IOVfirst = line.split()[1]
0431                     tmpbeam.IOVlast = line.split()[1]
0432                 if hasBX:
0433                     tmpBX = line.split()[3]
0434                 tmpbeamsize += 1
0435             if line.find('BeginTimeOfFit') != -1:
0436                 tmpbeam.IOVBeginTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
0437                 if IOVbase =="timebase":
0438                     tmpbeam.IOVfirst =  time.mktime( time.strptime(line.split()[1] +  " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
0439                 tmpbeamsize += 1
0440             if line.find('EndTimeOfFit') != -1:
0441                 tmpbeam.IOVEndTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
0442                 if IOVbase =="timebase":
0443                     tmpbeam.IOVlast = time.mktime( time.strptime(line.split()[1] +  " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
0444                 tmpbeamsize += 1
0445             if tmpbeamsize == 20:
0446                 if IOVbase=="lumibase":
0447                     tmprunfirst = int(firstRun.split(":")[0])
0448                     tmprunlast  = int(lastRun.split(":")[0])
0449                     tmplumifirst = int(firstRun.split(":")[1])
0450                     tmplumilast  = int(lastRun.split(":")[1])
0451                     acceptiov1 = acceptiov2 = False
0452                     # check lumis in the same run
0453                     if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
0454                         if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
0455                             acceptiov1 = acceptiov2 = True
0456                     # if different runs make sure you select the correct range of lumis
0457                     elif int(tmpbeam.Run) == tmprunfirst:
0458                         if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
0459                     elif int(tmpbeam.Run) == tmprunlast:
0460                         if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
0461                     elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
0462                         acceptiov1 = acceptiov2 = True
0463
0464                     if acceptiov1 and acceptiov2:
0465                         if tmpbeam.Type != 2:
0466                             print("invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0467                         elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0468                             print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0469                         elif hasBX:
0470                             if (tmpBX in maplist) == False:
0471                                 maplist[tmpBX] = [tmpbeam]
0472                             else:
0473                                 maplist[tmpBX].append(tmpbeam)
0474                         else:
0475                             listbeam.append(tmpbeam)
0476
0477                 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
0478                     if tmpbeam.Type != 2:
0479                         print("invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0480                     elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0481                         print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0482                     else:
0483                         listbeam.append(tmpbeam)
0484
0485                 tmpbeamsize = 0
0486                 tmpbeam = BeamSpot()
0487                 tmpBX = 0
0488     else:
0489
0490         for line in tmpfile:
0491
0492             if line.find('X0') != -1:
0493                 tmpbeam.X = line.split()[2]
0494                 tmpbeam.Xerr = line.split()[4]
0495                 tmpbeamsize += 1
0496             #print " x = " + str(tmpbeam.X)
0497             if line.find('Y0') != -1:
0498                 tmpbeam.Y = line.split()[2]
0499                 tmpbeam.Yerr = line.split()[4]
0500                 tmpbeamsize += 1
0501             #print " y =" + str(tmpbeam.Y)
0502             if line.find('Z0') != -1 and line.find('Sigma Z0') == -1:
0503                 tmpbeam.Z = line.split()[2]
0504                 tmpbeam.Zerr = line.split()[4]
0505                 tmpbeamsize += 1
0506             #print " z =" + str(tmpbeam.Z)
0507             if line.find('Sigma Z0') !=-1:
0508                 tmpbeam.sigmaZ = line.split()[3]
0509                 tmpbeam.sigmaZerr = line.split()[5]
0510                 tmpbeamsize += 1
0511             if line.find('dxdz') != -1:
0512                 tmpbeam.dxdz = line.split()[2]
0513                 tmpbeam.dxdzerr = line.split()[4]
0514                 tmpbeamsize += 1
0515             if line.find('dydz') != -1:
0516                 tmpbeam.dydz = line.split()[2]
0517                 tmpbeam.dydzerr = line.split()[4]
0518                 tmpbeamsize += 1
0519             if line.find('Beam Width X') != -1:
0520                 tmpbeam.beamWidthX = line.split()[4]
0521                 tmpbeam.beamWidthXerr = line.split()[6]
0522                 tmpbeamsize += 1
0523             if line.find('Beam Width Y') != -1:
0524                 tmpbeam.beamWidthY = line.split()[4]
0525                 tmpbeam.beamWidthYerr = line.split()[6]
0526                 tmpbeamsize += 1
0527         #if line.find('Run ') != -1:
0528             if line.find('for runs')  != -1:
0529             #tmpbeam.IOVfirst = line.split()[6].strip(',')
0530                 tmpbeam.Run      = line.split()[2]
0531                 if IOVbase == "runbase":
0532                     tmpbeam.IOVfirst = line.split()[2]
0533                     tmpbeam.IOVlast = line.split()[4]
0534                 tmpbeamsize += 1
0535             if line.find('LumiSection')  != -1:
0536                 if IOVbase=="lumibase":
0537                     tmpbeam.IOVfirst = line.split()[10]
0538                     tmpbeam.IOVlast = line.split()[10]
0539                 tmpbeamsize += 1
0540             if tmpbeamsize == 10:
0541
0542                 if IOVbase=="lumibase":
0543                     tmprunfirst = int(firstRun.split(":")[0])
0544                     tmprunlast  = int(lastRun.split(":")[0])
0545                     tmplumifirst = int(firstRun.split(":")[1])
0546                     tmplumilast  = int(lastRun.split(":")[1])
0547                     acceptiov1 = acceptiov2 = False
0548                     # check lumis in the same run
0549                     if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
0550                         if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
0551                             acceptiov1 = acceptiov2 = True
0552                     # if different runs make sure you select the correct range of lumis
0553                     elif int(tmpbeam.Run) == tmprunfirst:
0554                         if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
0555                     elif int(tmpbeam.Run) == tmprunlast:
0556                         if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
0557                     elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
0558                         acceptiov1 = acceptiov2 = True
0559
0560                     if acceptiov1 and acceptiov2:
0561                         if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0562                             print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0563                         elif hasBX:
0564                             if (tmpBX in maplist) == False:
0565                                 maplist[tmpBX] = [tmpbeam]
0566                             else:
0567                                 maplist[tmpBX].append(tmpbeam)
0568                         else:
0569                             listbeam.append(tmpbeam)
0570
0571                 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
0572                     if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
0573                         print("invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast))
0574                     else:
0575                         listbeam.append(tmpbeam)
0576
0577                 tmpbeamsize = 0
0578                 tmpbeam = BeamSpot()
0579                 tmpBX = 0
0580
0581     tmpfile.close()
0582     print(" got total number of IOVs = " + str(len(listbeam)) + " from file " + fileName)
0583     #print " run " + str(listbeam[3].IOVfirst ) + " " + str( listbeam[3].X )
0584     if hasBX:
0585         return maplist
0586     else:
0587         return listbeam
0588
0589 ###########################################################################################
0590 # Sort and clean list of data for consecutive duplicates and bad fits
0591 def sortAndCleanBeamList(listbeam=[],IOVbase="lumibase"):
0592     # sort the list
0593     if IOVbase == "lumibase":
0594         listbeam.sort( cmp = cmp_list_lumi )
0595     else:
0596         listbeam.sort( cmp = cmp_list_run )
0597
0598     # first clean list of data for consecutive duplicates and bad fits
0599     tmpremovelist = []
0600     for ii in range(0,len(listbeam)):
0601         ibeam = listbeam[ii]
0602         datax = ibeam.IOVfirst
0603         #print str(ii) + "  " +datax
0604         if datax == '0' and IOVbase =="runbase":
0605             print(" iov = 0? skip this IOV = "+ str(ibeam.IOVfirst) + " to " + str(ibeam.IOVlast))
0606             tmpremovelist.append(ibeam)
0607
0608         if ii < len(listbeam) -1:
0609             #print listbeam[ii+1].IOVfirst
0610             if IOVbase =="lumibase":
0611                 if ibeam.Run == listbeam[ii+1].Run and ibeam.IOVfirst == listbeam[ii+1].IOVfirst:
0612                     print(" duplicate IOV = "+datax+", keep only last duplicate entry")
0613                     tmpremovelist.append(ibeam)
0614             elif datax == listbeam[ii+1].IOVfirst:
0615                 print(" duplicate IOV = "+datax+", keep only last duplicate entry")
0616                 tmpremovelist.append(ibeam)
0617
0618     for itmp in tmpremovelist:
0619         listbeam.remove(itmp)
0620
0621 ###########################################################################################
0622 # CREATE FILE FOR PAYLOADS
0623 def createWeightedPayloads(fileName,listbeam=[],weighted=True):
0624     newlistbeam = []
0625     tmpbeam = BeamSpot()
0626     docreate = True
0627     countlumi = 0
0628     tmprun = ""
0629     maxNlumis = 60
0630     if weighted:
0631         maxNlumis = 999999999
0632     for ii in range(0,len(listbeam)):
0633         ibeam = listbeam[ii]
0634         inextbeam = BeamSpot()
0635         iNNbeam = BeamSpot()
0636         if docreate:
0637             tmpbeam.IOVfirst = ibeam.IOVfirst
0638             tmpbeam.IOVBeginTime = ibeam.IOVBeginTime
0639             tmpbeam.Run = ibeam.Run
0640             tmpbeam.Type = 2
0641         docheck = False
0642         docreate = False
0643         #print "Currently testing ii="+str(ii)+" Lumi1: "+str(ibeam.IOVfirst)
0644
0645         # check last iov
0646         if ii < len(listbeam) - 1:
0647             inextbeam = listbeam[ii+1]
0648             docheck = True
0649             if ii < len(listbeam) -2:
0650                 iNNbeam = listbeam[ii+2]
0651         else:
0652             print("close payload because end of data has been reached. Run "+ibeam.Run)
0653             docreate = True
0654         # check we run over the same run
0655         if ibeam.Run != inextbeam.Run:
0656             print("close payload because end of run "+ibeam.Run)
0657             docreate = True
0658         # check maximum lumi counts
0659         if countlumi == maxNlumis -1:
0660             print("close payload because maximum lumi sections accumulated within run "+ibeam.Run)
0661             docreate = True
0662             countlumi = 0
0663         # weighted average position
0664         (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
0665         (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
0666         (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
0667         (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
0668         (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
0669         (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
0670         #print "wx = " + ibeam.beamWidthX + " err= "+ ibeam.beamWidthXerr
0671         (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
0672         (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
0673
0674         if weighted:
0675             docheck = False
0676         # check offsets
0677         #if False:
0678         if docheck:
0679
0680             # define minimum limit
0681             min_limit = 0.0025
0682
0683             # limit for x and y
0684             limit = float(ibeam.beamWidthX)/2.
0685             if limit < min_limit: limit = min_limit
0686
0687             # check movements in X
0688             adelta1 = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr)
0689             adelta2 = (0.,1.e9)
0690             adelta1dxdz = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
0691             adelta2dxdz = (0.,1.e9)
0692             adelta1dydz = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
0693             adelta2dydz = (0.,1.e9)
0694             adelta1widthx = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
0695             adelta2widthx = (0.,1.e9)
0696             adelta1widthy = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
0697             adelta2widthy = (0.,1.e9)
0698             adelta1z0 = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr)
0699             adelta1sigmaZ = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr)
0700
0701             if iNNbeam.Type != -1:
0702                 adelta2 = delta(inextbeam.X, inextbeam.Xerr, iNNbeam.X, iNNbeam.Xerr)
0703                 adelta2dxdz = delta(inextbeam.dxdz, inextbeam.dxdzerr, iNNbeam.dxdz, iNNbeam.dxdzerr)
0704                 adelta2dydz = delta(inextbeam.dydz, inextbeam.dydzerr, iNNbeam.dydz, iNNbeam.dydzerr)
0705                 adelta2widthx = delta(inextbeam.beamWidthX, inextbeam.beamWidthXerr, iNNbeam.beamWidthX, iNNbeam.beamWidthXerr)
0706                 adelta2widthy = delta(inextbeam.beamWidthY, inextbeam.beamWidthYerr, iNNbeam.beamWidthY, iNNbeam.beamWidthYerr)
0707
0708             deltaX = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
0709             if ii < len(listbeam) -2:
0710                 if deltaX==False and adelta1[0]*adelta2[0] > 0. and  math.fabs(adelta1[0]+adelta2[0]) >= limit:
0711                     #print " positive, "+str(adelta1[0]+adelta2[0])+ " limit="+str(limit)
0712                     deltaX = True
0713                 elif deltaX==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
0714                     deltaX = False
0715                     #print " negative, "+str(adelta1[0]/adelta2[0])
0716                 #else:
0717                 #    print str(adelta1[0]/adelta2[0])
0718
0719             # check movemnts in Y
0720             adelta1 = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr)
0721             adelta2 = (0.,1.e9)
0722             if iNNbeam.Type != -1:
0723                 adelta2 = delta(inextbeam.Y, inextbeam.Yerr, iNNbeam.Y, iNNbeam.Yerr)
0724
0725             deltaY = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
0726             if ii < len(listbeam) -2:
0727                 if deltaY==False and adelta1[0]*adelta2[0] > 0. and  math.fabs(adelta1[0]+adelta2[0]) >= limit:
0728                     deltaY = True
0729                 elif deltaY==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
0730                     deltaY = False
0731             # check movements in Z
0732
0733             limit = float(ibeam.sigmaZ)/2.
0734             deltaZ = deltaSig(adelta1z0) > 3.5 and math.fabs(adelta1z0[0]) >= limit
0735
0736             deltasigmaZ = deltaSig(adelta1sigmaZ) > 5.0
0737
0738             # check dxdz
0739             adelta = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
0740             deltadxdz   = deltaSig(adelta) > 5.0
0741             if deltadxdz and adelta1dxdz[0]*adelta2dxdz[0]<=0 and adelta2dxdz[0] != 0 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) > 0.33 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) < 3:
0742                 deltadxdz = False
0743             # check dydz
0744             adelta = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
0745             deltadydz   = deltaSig(adelta) > 5.0
0746             if deltadydz and adelta1dydz[0]*adelta2dydz[0]<=0 and adelta2dydz[0] != 0 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) > 0.33 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) < 3:
0747                 deltadydz = False
0748
0749             adelta = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
0750             deltawidthX = deltaSig(adelta) > 5
0751             if deltawidthX and adelta1widthx[0]*adelta2widthx[0]<=0 and adelta2widthx[0] != 0 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) > 0.33 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) < 3:
0752                 deltawidthX = False
0753
0754             adelta = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
0755             deltawidthY = deltaSig(adelta) > 5
0756             if deltawidthY and adelta1widthy[0]*adelta2widthy[0]<=0 and adelta2widthy[0] != 0 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) > 0.33 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) < 3:
0757                 deltawidthY = False
0758             #if iNNbeam.Type != -1:
0759             #    deltaX = deltaX and delta(ibeam.X, ibeam.Xerr, iNNbeam.X, iNNbeam.Xerr) > 1.5
0760             #    deltaY = deltaY and delta(ibeam.Y, ibeam.Yerr, iNNbeam.Y, iNNbeam.Yerr) > 1.5
0761             #    deltaZ = deltaZ and delta(ibeam.Z, ibeam.Zerr, iNNbeam.Z, iNNbeam.Zerr) > 1.5
0762             #
0763             #    deltasigmaZ = deltasigmaZ and delta(ibeam.sigmaZ, ibeam.sigmaZerr, iNNbeam.sigmaZ, iNNbeam.sigmaZerr) > 2.5
0764             #    deltadxdz   = deltadxdz and delta(ibeam.dxdz, ibeam.dxdzerr, iNNbeam.dxdz, iNNbeam.dxdzerr) > 2.5
0765             #    deltadydz   = deltadydz and delta(ibeam.dydz, ibeam.dydzerr, iNNbeam.dydz, iNNbeam.dydzerr) > 2.5
0766             #
0767             #    deltawidthX = deltawidthX and delta(ibeam.beamWidthX, ibeam.beamWidthXerr, iNNbeam.beamWidthX, iNNbeam.beamWidthXerr) > 3
0768             #    deltawidthY = deltawidthY and delta(ibeam.beamWidthY, ibeam.beamWidthYerr, iNNbeam.beamWidthY, iNNbeam.beamWidthYerr) > 3
0769
0770             if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
0771                 docreate = True
0772                 #print "shift here: x="+str(deltaX)+" y="+str(deltaY)
0773                 #print "x1 = "+ibeam.X + " x1err = "+ibeam.Xerr
0774                 #print "x2 = "+inextbeam.X + " x2err = "+inextbeam.Xerr
0775                 #print "Lumi1: "+str(ibeam.IOVfirst) + " Lumi2: "+str(inextbeam.IOVfirst)
0776                 #print " x= "+ibeam.X+" +/- "+ibeam.Xerr
0777                 #print "weighted average x = "+tmpbeam.X +" +//- "+tmpbeam.Xerr
0778                 print("close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY))
0779         if docreate:
0780             #if ii == len(listbeam)-1:
0781             tmpbeam.IOVlast = ibeam.IOVlast
0782             tmpbeam.IOVEndTime = ibeam.IOVEndTime
0783             print("  Run: "+tmpbeam.Run +" Lumi1: "+str(tmpbeam.IOVfirst) + " Lumi2: "+str(tmpbeam.IOVlast))
0784             newlistbeam.append(tmpbeam)
0785             tmpbeam = BeamSpot()
0786             countlumi = 0
0787         tmprun = ibeam.Run
0788         countlumi += 1
0789
0790     payloadfile = open(fileName,"w")
0791     for iload in newlistbeam:
0792         dump( iload, payloadfile )
0793     payloadfile.close()
0794     return newlistbeam
0795 ###########################################################################################
0796 def createWeightedPayloadsNew(fileName,listbeam=[],weighted=True):
0797     newlistbeam = []
0798     docreate = False
0799     docheck = False
0800     lastPayload = listbeam[0]
0801
0802     firstToUse = 0
0803     lastToUse = 0
0804     for ii in range(0,len(listbeam)):
0805         docreate = False
0806         if docheck:
0807             deltaX = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr) > 1.5
0808             deltaY = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr) > 1.5
0809             deltaZ = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr) > 2.5
0810
0811             deltasigmaZ = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr) > 2.5
0812             deltadxdz   = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr) > 2.5
0813             deltadydz   = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr) > 2.5
0814
0815             deltawidthX = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr) > 3
0816             deltawidthY = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr) > 3
0817
0818             #if iNNbeam.Type != -1:
0819             #    deltaX = deltaX and delta(ibeam.X, ibeam.Xerr, iNNbeam.X, iNNbeam.Xerr) > 1.5
0820             #    deltaY = deltaY and delta(ibeam.Y, ibeam.Yerr, iNNbeam.Y, iNNbeam.Yerr) > 1.5
0821             #    deltaZ = deltaZ and delta(ibeam.Z, ibeam.Zerr, iNNbeam.Z, iNNbeam.Zerr) > 1.5
0822             #
0823             #    deltasigmaZ = deltasigmaZ and delta(ibeam.sigmaZ, ibeam.sigmaZerr, iNNbeam.sigmaZ, iNNbeam.sigmaZerr) > 2.5
0824             #    deltadxdz   = deltadxdz and delta(ibeam.dxdz, ibeam.dxdzerr, iNNbeam.dxdz, iNNbeam.dxdzerr) > 2.5
0825             #    deltadydz   = deltadydz and delta(ibeam.dydz, ibeam.dydzerr, iNNbeam.dydz, iNNbeam.dydzerr) > 2.5
0826             #
0827             #    deltawidthX = deltawidthX and delta(ibeam.beamWidthX, ibeam.beamWidthXerr, iNNbeam.beamWidthX, iNNbeam.beamWidthXerr) > 3
0828             #    deltawidthY = deltawidthY and delta(ibeam.beamWidthY, ibeam.beamWidthYerr, iNNbeam.beamWidthY, iNNbeam.beamWidthYerr) > 3
0829
0830             if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
0831                 if ii != 0:
0832                     docreate = True
0833                     lastToUse = ii-1
0834                 #print "shift here: x="+str(deltaX)+" y="+str(deltaY)
0835                 #print "x1 = "+ibeam.X + " x1err = "+ibeam.Xerr
0836                 #print "x2 = "+inextbeam.X + " x2err = "+inextbeam.Xerr
0837                 #print "Lumi1: "+str(ibeam.IOVfirst) + " Lumi2: "+str(inextbeam.IOVfirst)
0838                 #print " x= "+ibeam.X+" +/- "+ibeam.Xerr
0839                 #print "weighted average x = "+tmpbeam.X +" +//- "+tmpbeam.Xerr
0840                 print("close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY))
0841
0842         #WARNING this will only be fine for Run based IOVs
0843         if ii >= len(listbeam) - 1 or listbeam[ii].Run != listbeam[ii+1].Run :
0844             print("close payload because end of run has been reached. Run " + listbeam[ii].Run)
0845             docreate = True
0846             lastToUse = ii
0847
0848
0849          # check maximum lumi counts
0850 #?        if countlumi == maxNlumis:
0851 #?            print "close payload because maximum lumi sections accumulated within run "+ibeam.Run
0852 #?            docreate = True
0853 #?            countlumi = 0
0854         if docreate:
0855             tmpbeam = BeamSpot()
0856             for ibeam in listbeam[firstToUse:lastToUse]:
0857                 (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
0858                 (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
0859                 (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
0860                 (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
0861                 (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
0862                 (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
0863                 #print "wx = " + ibeam.beamWidthX + " err= "+ ibeam.beamWidthXerr
0864                 (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
0865                 (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
0866             tmpbeam.IOVfirst     = listbeam[firstToUse].IOVfirst
0867             tmpbeam.IOVBeginTime = listbeam[firstToUse].IOVBeginTime
0868             tmpbeam.Run          = listbeam[firstToUse].Run
0869             tmpbeam.Type         = 2
0870             tmpbeam.IOVlast      = listbeam[lastToUse].IOVlast
0871             tmpbeam.IOVEndTime   = listbeam[lastToUse].IOVEndTime
0872             newlistbeam.append(tmpbeam)
0873             firstToUse = lastToUse+1
0874             print("Run: " + tmpbeam.Run + " Lumi1: " + str(tmpbeam.IOVfirst) + " Lumi2: " + str(tmpbeam.IOVlast))
0875
0876     payloadfile = open(fileName,"w")
0877     for iload in newlistbeam:
0878         dump( iload, payloadfile )
0879     payloadfile.close()
0880
0881 ###########################################################################################
0882 def writeSqliteFile(sqliteFileName,tagName,timeType,beamSpotFile,sqliteTemplateFile,tmpDir="/tmp/"):
0883     writeDBOut = tmpDir + "write2DB_" + tagName + ".py"
0884     wFile      = open(sqliteTemplateFile)
0885     wNewFile   = open(writeDBOut,'w')
0886
0887     writeDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
0888                    ('TAGNAME',tagName),
0889                    ('TIMETYPE',timeType),
0890                    ('BEAMSPOTFILE',beamSpotFile)]
0891
0892     for line in wFile:
0893         for itag in writeDBTags:
0894             line = line.replace(itag[0],itag[1])
0895         wNewFile.write(line)
0896
0897     wNewFile.close()
0898     print("writing sqlite file ...")
0899     status_wDB = subprocess.getstatusoutput('cmsRun '+ writeDBOut)
0900     print(status_wDB[1])
0901
0902     os.system("rm -f " + writeDBOut)
0903     return not status_wDB[0]
0904
0905 ###########################################################################################
0906 def readSqliteFile(sqliteFileName,tagName,sqliteTemplateFile,tmpDir="/tmp/"):
0907     readDBOut = tmpDir + "readDB_" + tagName + ".py"
0908
0909     rFile = open(sqliteTemplateFile)
0910     rNewFile = open(readDBOut,'w')
0911
0912     readDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
0913                   ('TAGNAME',tagName)]
0914
0915     for line in rFile:
0916         for itag in readDBTags:
0917             line = line.replace(itag[0],itag[1])
0918         rNewFile.write(line)
0919
0920     rNewFile.close()
0921     status_rDB = subprocess.getstatusoutput('cmsRun '+ readDBOut)
0922
0923     outtext = status_rDB[1]
0924     print(outtext)
0925     os.system("rm -f " + readDBOut)
0926     return not status_rDB[0]
0927
0928 ###########################################################################################
0929 def appendSqliteFile(combinedSqliteFileName, sqliteFileName, tagName, IOVSince, IOVTill ,tmpDir="/tmp/"):
0930     aCommand = "conddb_import -c sqlite_file:" + tmpDir + combinedSqliteFileName + " -f sqlite_file:" + sqliteFileName + " -i " + tagName + " -t " + tagName + " -b " + IOVSince + " -e " + IOVTill
0931     print(aCommand)
0932     std = subprocess.getstatusoutput(aCommand)
0933     print(std[1])
0934     return not std[0]
0935
0936 ###########################################################################################
0937 def uploadSqliteFile(sqliteFileDirName, sqliteFileName, dropbox="/DropBox"):
0938     # Changing permissions to metadata
0939     acmd = "chmod a+w " + sqliteFileDirName + sqliteFileName + ".txt"
0940     outcmd = subprocess.getstatusoutput(acmd)
0941     print(acmd)
0942 #    print outcmd[1]
0943     if outcmd[0]:
0944         print("Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".txt")
0945         return False
0946
0947     acmd = "cp " + sqliteFileDirName + sqliteFileName + ".db " + sqliteFileDirName + sqliteFileName + ".txt ."
0948     print(acmd)
0949     outcmd = subprocess.getstatusoutput(acmd)
0950     print(outcmd[1])
0951     if outcmd[0]:
0952         print("Couldn't cd to " + sqliteFileDirName)
0953         return False
0954
0955     acmd = "tar -cvjf " + sqliteFileName + ".tar.bz2 " + sqliteFileName + ".db " + sqliteFileName + ".txt"
0956     print(acmd)
0957     outcmd = subprocess.getstatusoutput(acmd)
0958     print(outcmd[1])
0959     if outcmd[0]:
0960         print("Couldn't zip the files!")
0961         return False
0962
0963     acmd = "chmod a+w " + sqliteFileName + ".tar.bz2"
0964     outcmd = subprocess.getstatusoutput(acmd)
0965     print(acmd)
0966 #    print outcmd[1]
0967     if outcmd[0]:
0968         print("Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".tar.bz2")
0969         return False
0970
0971     acmd = "scp -p " + sqliteFileName + ".tar.bz2" + " webcondvm.cern.ch:" + dropbox
0972     print(acmd)
0973     outcmd = subprocess.getstatusoutput(acmd)
0974     print(outcmd[1])
0975     if outcmd[0]:
0976         print("Couldn't scp the files to DropBox!")
0977         return False
0978
0979
0980     acmd = "mv " + sqliteFileName + ".tar.bz2 " + sqliteFileDirName
0981     print(acmd)
0982     outcmd = subprocess.getstatusoutput(acmd)
0983     print(outcmd[1])
0984     if outcmd[0]:
0985         print("Couldn't mv the file to " + sqliteFileDirName)
0986         return False
0987
0988     acmd = "rm " + sqliteFileName + ".db " + sqliteFileName + ".txt"
0989     print(acmd)
0990     outcmd = subprocess.getstatusoutput(acmd)
0991     print(outcmd[1])
0992     if outcmd[0]:
0993         print("Couldn't rm the db and txt files")
0994         return False
0995
0996 #    acmd = "scp -p " + sqliteFileDirName + sqliteFileName + ".txt webcondvm.cern.ch:/tmp"
0997 #    outcmd = subprocess.getstatusoutput(acmd)
0998 #    print acmd
0999 #    print outcmd[1]
1000 #    if outcmd[0]:
1001 #        print "Can't change permission to file: " + sqliteFileName + ".txt"
1002 #        return False
1003
1004 #    acmd = "ssh webcondvm.cern.ch \"mv /tmp/" + sqliteFileName + ".db /tmp/" + sqliteFileName + ".txt " + dropbox +"\""
1005 #    print acmd
1006 #    outcmd = subprocess.getstatusoutput(acmd)
1007 #    print outcmd[1]
1008 #    if outcmd[0]:
1009 #        print "Can't move files from tmp to dropbox!"
1010         return False
1011
1012 #    acmd = "ssh webcondvm.cern.ch \"mv /tmp/" + final_sqlite_file_name + ".txt "+dropbox +"\""
1013 #    outcmd = subprocess.getstatusoutput(acmd)
1014 #    print acmd
1015 #    print outcmd[1]
1016 #    if outcmd[0]:
1017 #        print "Can't change permission to file: " + sqliteFileName + ".txt"
1018 #        return False
1019
1020     return True
1021