|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
#! /usr/bin/env python3
import os,time,sys,shutil,glob
from sets import Set
from datetime import datetime
import smtplib
from email.MIMEText import MIMEText
from ROOT import TFile
def sendmail(EmailAddress,run):
s=smtplib.SMTP("localhost")
tolist=[EmailAddress]
body="File merge failed by unknown reason for run"+run
msg = MIMEText(body)
msg['Subject'] = "File merge failed."
msg['From'] = ServerMail
msg['To'] = EmailAddress
s.sendmail(ServerMail,tolist,msg.as_string())
s.quit()
def filecheck(rootfile):
f = TFile(rootfile)
if (f.IsZombie()):
#print "File corrupted"
f.Close()
return 0
else:
hist = f.FindObjectAny("reportSummaryContents")
#(skip filecheck for HcalTiming files!!)
if (hist == None and rootfile.rfind('HcalTiming') == -1):
#print "File is incomplete"
f.Close()
return 0
else:
#print "File is OK"
f.Close()
return 1
TempTag = TimeTag + '-tmp'
if not os.path.exists(TimeTag):
os.system('touch -t 01010000 '+ TimeTag)
#### search new files
NFOUND = 0
NEW = {}
TAG = os.stat(TimeTag).st_mtime
for dir, subdirs, files in os.walk(DIR):
paths = ["%s/%s" % (dir, x) for x in files]
for f in [x for x in paths if re.search(r'/DQM_.*_R[0-9]{9}\.root$', x)]:
if os.stat(f).st_mtime > TAG and f.index("DQM_Reference") < 0:
NEW.get(f[-14:-5], []).append(f)
NFOUND += 1
if not NFOUND:
print('Have not found new files...')
os._exit(99)
os.system("ls -l %s" % TimeTag)
os.system('touch '+ TempTag)
print(datetime.now())
print('Found %d new file(s).' % NFOUND)
#### loop for runs
newFiles = []
allOldFiles = []
for (run, files) in NEW.items():
runnr = "%09d" % long(run)
destdir = "%s/%s/%s/%s" % (FILEDIR, runnr[0:3], runnr[3:6], runnr[6:9])
oldfiles = []
version = 1
while True:
destfile = "%s/DQM_V%04d_R%s.root" % (destdir, version, runnr)
if not os.path.exists(destfile): break
oldfiles.append(destfile)
version += 1
if not os.path.exists(destdir):
os.makedirs(destdir)
logfile = "%s.log" % destfile[:-4]
tmpdestfile = "%s.tmp" % destfile
print('Run %s is being merged...' % run)
if os.path.exists(tmpdestfile):
os.remove(tmpdestfile)
ntries = 0
while ntries < 30:
LOGFILE = file(logfile, 'a')
LOGFILE.write(os.popen('DQMMergeFile %s %s' % (tmpdestfile, " ".join(files))).read())
if not os.path.exists(tmpdestfile):
print('Failed merging files for run %s. Try again after two minutes' % run)
time.sleep(WAITTIME)
ntries += 1
else:
break
os.rename(tmpdestfile, destfile)
sendmail(YourEmail,run)
allOldFiles.extend(oldfiles)
newFiles.append(destfile)
if os.path.exists(TMPDB):
os.remove(TMPDB)
if os.path.exists(DB):
os.rename(DB, TMPDB)
else:
logfile.write('*** INITIALISE DATABASE ***\n')
logfile.write(os.popen('visDQMRegisterFile %s "/Global/Online/ALL" "Global run"' % TMPDB).read())
logfile.write('*** UNREGISTER %d OLD FILES ***\n' % len(allOldFiles))
while len(allOldFiles) > 0:
(slice, rest) = (allOldFiles[0:50], allOldFiles[50:])
logfile.write(os.popen('visDQMUnregisterFile %s %s' % (tmpdb, " ".join(slice))).read())
allOldFiles = rest
for old in slice:
os.remove(old)
for file in newFiles:
print('Registering %s' % file)
logfile.write('*** REGISTER FILE %s ***\n' % file)
logfile.write(os.popen('visDQMRegisterFile %s "/Global/Online/ALL" "Global run" %s' % (TMPDB, file)).read())
print('%s registered' % file)
os.rename(TMPDB, DB)
os.remove(TimeTag)
os.rename(TempTag, TimeTag)
|