scripts/fileregistration/mergeAndRegister.py

0001 #! /usr/bin/env python3
0002
0003 import os,time,sys,shutil,glob
0004 from sets import Set
0005 from datetime import datetime
0006 import smtplib
0007 from email.MIMEText import MIMEText
0008 from ROOT import TFile
0009
0010 def sendmail(EmailAddress,run):
0011     s=smtplib.SMTP("localhost")
0012     tolist=[EmailAddress]
0013     body="File merge failed by unknown reason for run"+run
0014     msg = MIMEText(body)
0015     msg['Subject'] = "File merge failed."
0016     msg['From'] = ServerMail
0017     msg['To'] = EmailAddress
0018     s.sendmail(ServerMail,tolist,msg.as_string())
0019     s.quit()
0020
0021 def filecheck(rootfile):
0022     f = TFile(rootfile)
0023     if (f.IsZombie()):
0024         #print "File corrupted"
0025         f.Close()
0026         return 0
0027     else:
0028         hist = f.FindObjectAny("reportSummaryContents")
0029         #(skip filecheck for HcalTiming files!!)
0030         if (hist == None and rootfile.rfind('HcalTiming') == -1):
0031             #print "File is incomplete"
0032             f.Close()
0033             return 0
0034         else:
0035             #print "File is OK"
0036             f.Close()
0037             return 1
0038
0039
0040
0041 TempTag = TimeTag + '-tmp'
0042 if not os.path.exists(TimeTag):
0043     os.system('touch -t 01010000 '+ TimeTag)
0044
0045
0046 #### search new files
0047 NFOUND = 0
0048 NEW = {}
0049 TAG = os.stat(TimeTag).st_mtime
0050 for dir, subdirs, files in os.walk(DIR):
0051     paths = ["%s/%s" % (dir, x) for x in files]
0052     for f in [x for x in paths if re.search(r'/DQM_.*_R[0-9]{9}\.root$', x)]:
0053         if os.stat(f).st_mtime > TAG and f.index("DQM_Reference") < 0:
0054             NEW.get(f[-14:-5], []).append(f)
0055             NFOUND += 1
0056
0057 if not NFOUND:
0058     print('Have not found new files...')
0059     os._exit(99)
0060
0061 os.system("ls -l %s" % TimeTag)
0062 os.system('touch '+ TempTag)
0063 print(datetime.now())
0064 print('Found %d new file(s).' % NFOUND)
0065
0066 #### loop for runs
0067 newFiles = []
0068 allOldFiles = []
0069 for (run, files) in NEW.items():
0070     runnr = "%09d" % long(run)
0071     destdir = "%s/%s/%s/%s" % (FILEDIR, runnr[0:3], runnr[3:6], runnr[6:9])
0072     oldfiles = []
0073     version = 1
0074     while True:
0075         destfile = "%s/DQM_V%04d_R%s.root" % (destdir, version, runnr)
0076         if not os.path.exists(destfile): break
0077         oldfiles.append(destfile)
0078         version += 1
0079
0080     if not os.path.exists(destdir):
0081         os.makedirs(destdir)
0082
0083     logfile = "%s.log" % destfile[:-4]
0084     tmpdestfile = "%s.tmp" % destfile
0085
0086     print('Run %s is being merged...' % run)
0087
0088     if os.path.exists(tmpdestfile):
0089         os.remove(tmpdestfile)
0090
0091     ntries = 0
0092     while ntries < 30:
0093         LOGFILE = file(logfile, 'a')
0094         LOGFILE.write(os.popen('DQMMergeFile %s %s' % (tmpdestfile, " ".join(files))).read())
0095         if not os.path.exists(tmpdestfile):
0096             print('Failed merging files for run %s. Try again after two minutes' % run)
0097             time.sleep(WAITTIME)
0098             ntries += 1
0099         else:
0100             break
0101
0102     os.rename(tmpdestfile, destfile)
0103     sendmail(YourEmail,run)
0104     allOldFiles.extend(oldfiles)
0105     newFiles.append(destfile)
0106
0107
0108 if os.path.exists(TMPDB):
0109     os.remove(TMPDB)
0110
0111 if os.path.exists(DB):
0112     os.rename(DB, TMPDB)
0113 else:
0114     logfile.write('*** INITIALISE DATABASE ***\n')
0115     logfile.write(os.popen('visDQMRegisterFile %s "/Global/Online/ALL" "Global run"' % TMPDB).read())
0116
0117 logfile.write('*** UNREGISTER %d OLD FILES ***\n' % len(allOldFiles))
0118 while len(allOldFiles) > 0:
0119     (slice, rest) = (allOldFiles[0:50], allOldFiles[50:])
0120     logfile.write(os.popen('visDQMUnregisterFile %s %s' % (tmpdb, " ".join(slice))).read())
0121     allOldFiles = rest
0122     for old in slice:
0123         os.remove(old)
0124
0125 for file in newFiles:
0126     print('Registering %s' % file)
0127     logfile.write('*** REGISTER FILE %s ***\n' % file)
0128     logfile.write(os.popen('visDQMRegisterFile %s "/Global/Online/ALL" "Global run" %s' % (TMPDB, file)).read())
0129     print('%s registered' % file)
0130
0131 os.rename(TMPDB, DB)
0132 os.remove(TimeTag)
0133 os.rename(TempTag, TimeTag)