scripts/fileregistration/mergeAndRegister.py

0001 #! /usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import os,time,sys,shutil,glob
0005 from sets import Set
0006 from datetime import datetime
0007 import smtplib
0008 from email.MIMEText import MIMEText
0009 from ROOT import TFile
0010
0011 def sendmail(EmailAddress,run):
0012     s=smtplib.SMTP("localhost")
0013     tolist=[EmailAddress]
0014     body="File merge failed by unknown reason for run"+run
0015     msg = MIMEText(body)
0016     msg['Subject'] = "File merge failed."
0017     msg['From'] = ServerMail
0018     msg['To'] = EmailAddress
0019     s.sendmail(ServerMail,tolist,msg.as_string())
0020     s.quit()
0021
0022 def filecheck(rootfile):
0023     f = TFile(rootfile)
0024     if (f.IsZombie()):
0025         #print "File corrupted"
0026         f.Close()
0027         return 0
0028     else:
0029         hist = f.FindObjectAny("reportSummaryContents")
0030         #(skip filecheck for HcalTiming files!!)
0031         if (hist == None and rootfile.rfind('HcalTiming') == -1):
0032             #print "File is incomplete"
0033             f.Close()
0034             return 0
0035         else:
0036             #print "File is OK"
0037             f.Close()
0038             return 1
0039
0040
0041
0042 TempTag = TimeTag + '-tmp'
0043 if not os.path.exists(TimeTag):
0044     os.system('touch -t 01010000 '+ TimeTag)
0045
0046
0047 #### search new files
0048 NFOUND = 0
0049 NEW = {}
0050 TAG = os.stat(TimeTag).st_mtime
0051 for dir, subdirs, files in os.walk(DIR):
0052     paths = ["%s/%s" % (dir, x) for x in files]
0053     for f in [x for x in paths if re.search(r'/DQM_.*_R[0-9]{9}\.root$', x)]:
0054         if os.stat(f).st_mtime > TAG and f.index("DQM_Reference") < 0:
0055             NEW.get(f[-14:-5], []).append(f)
0056             NFOUND += 1
0057
0058 if not NFOUND:
0059     print('Have not found new files...')
0060     os._exit(99)
0061
0062 os.system("ls -l %s" % TimeTag)
0063 os.system('touch '+ TempTag)
0064 print(datetime.now())
0065 print('Found %d new file(s).' % NFOUND)
0066
0067 #### loop for runs
0068 newFiles = []
0069 allOldFiles = []
0070 for (run, files) in NEW.items():
0071     runnr = "%09d" % long(run)
0072     destdir = "%s/%s/%s/%s" % (FILEDIR, runnr[0:3], runnr[3:6], runnr[6:9])
0073     oldfiles = []
0074     version = 1
0075     while True:
0076         destfile = "%s/DQM_V%04d_R%s.root" % (destdir, version, runnr)
0077         if not os.path.exists(destfile): break
0078         oldfiles.append(destfile)
0079         version += 1
0080
0081     if not os.path.exists(destdir):
0082         os.makedirs(destdir)
0083
0084     logfile = "%s.log" % destfile[:-4]
0085     tmpdestfile = "%s.tmp" % destfile
0086
0087     print('Run %s is being merged...' % run)
0088
0089     if os.path.exists(tmpdestfile):
0090         os.remove(tmpdestfile)
0091
0092     ntries = 0
0093     while ntries < 30:
0094         LOGFILE = file(logfile, 'a')
0095         LOGFILE.write(os.popen('DQMMergeFile %s %s' % (tmpdestfile, " ".join(files))).read())
0096         if not os.path.exists(tmpdestfile):
0097             print('Failed merging files for run %s. Try again after two minutes' % run)
0098             time.sleep(WAITTIME)
0099             ntries += 1
0100         else:
0101             break
0102
0103     os.rename(tmpdestfile, destfile)
0104     sendmail(YourEmail,run)
0105     allOldFiles.extend(oldfiles)
0106     newFiles.append(destfile)
0107
0108
0109 if os.path.exists(TMPDB):
0110     os.remove(TMPDB)
0111
0112 if os.path.exists(DB):
0113     os.rename(DB, TMPDB)
0114 else:
0115     logfile.write('*** INITIALISE DATABASE ***\n')
0116     logfile.write(os.popen('visDQMRegisterFile %s "/Global/Online/ALL" "Global run"' % TMPDB).read())
0117
0118 logfile.write('*** UNREGISTER %d OLD FILES ***\n' % len(allOldFiles))
0119 while len(allOldFiles) > 0:
0120     (slice, rest) = (allOldFiles[0:50], allOldFiles[50:])
0121     logfile.write(os.popen('visDQMUnregisterFile %s %s' % (tmpdb, " ".join(slice))).read())
0122     allOldFiles = rest
0123     for old in slice:
0124         os.remove(old)
0125
0126 for file in newFiles:
0127     print('Registering %s' % file)
0128     logfile.write('*** REGISTER FILE %s ***\n' % file)
0129     logfile.write(os.popen('visDQMRegisterFile %s "/Global/Online/ALL" "Global run" %s' % (TMPDB, file)).read())
0130     print('%s registered' % file)
0131
0132 os.rename(TMPDB, DB)
0133 os.remove(TimeTag)
0134 os.rename(TempTag, TimeTag)