|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
#!/usr/bin/env python3
import argparse, contextlib, os, re, shutil, subprocess, tempfile, time
if __name__ == "__main__":
def abspath(path):
if not os.path.exists(path): raise ValueError(path+" does not exist")
return os.path.abspath(path)
p = argparse.ArgumentParser()
p.add_argument("cfgfile", type=abspath)
p.add_argument("baddatafileslist", nargs="?", default=None)
args = p.parse_args()
def runcfg(cfgfile, badfilelist):
try:
subprocess.check_output(["cmsRun", cfgfile], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if "FallbackFileOpenError" in e.output:
output = e.output.split("An exception of category 'FallbackFileOpenError' occurred while")[1]
filename = re.search("Failed to open the file '[^']*(/store/.*[.]root)", output).group(1)
with OneAtATime(badfilelist+".tmp", 2) as f:
with open(badfilelist) as f:
contents = set(f.read().split())
if filename in contents:
raise RuntimeError(filename+"\nis already in\n"+badfilelist+"\n\nExiting to avoid an infinite loop. Maybe you have this running on the same cfg file multiple times?")
contents.add(filename)
contents = sorted(contents)
with open(badfilelist, "w") as f:
f.write("\n".join(contents)+"\n")
print("found and added a bad file:\n"+filename)
else:
raise
return runcfg(cfgfile, badfilelist)
print("all files left are good")
@contextlib.contextmanager
def cd(newdir):
"""http://stackoverflow.com/a/24176022/5228524"""
prevdir = os.getcwd()
os.chdir(os.path.expanduser(newdir))
try:
yield
finally:
os.chdir(prevdir)
def cdtemp(): return cd(tempfile.mkdtemp())
class KeepWhileOpenFile(object):
def __init__(self, name, message=None):
self.filename = name
self.__message = message
self.pwd = os.getcwd()
self.fd = self.f = None
self.bool = False
@property
def wouldbevalid(self):
if self: return True
with self:
return bool(self)
def __open(self):
self.fd = os.open(self.filename, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
def __enter__(self):
with cd(self.pwd):
try:
self.__open()
except OSError:
return None
self.f = os.fdopen(self.fd, 'w')
try:
if self.__message is not None:
self.f.write(self.__message+"\n")
except IOError:
pass
try:
self.f.close()
except IOError:
pass
self.bool = True
return True
def __exit__(self, *args):
if self:
try:
with cd(self.pwd):
os.remove(self.filename)
except OSError:
pass #ignore it
self.fd = self.f = None
self.bool = False
def __nonzero__(self):
return self.bool
class OneAtATime(KeepWhileOpenFile):
def __init__(self, name, delay, message=None, printmessage=None, task="doing this"):
super(OneAtATime, self).__init__(name, message=message)
self.delay = delay
if printmessage is None:
printmessage = "Another process is already {task}! Waiting {delay} seconds."
printmessage = printmessage.format(delay=delay, task=task)
self.__printmessage = printmessage
def __enter__(self):
while True:
result = super(OneAtATime, self).__enter__()
if result:
return result
print(self.__printmessage)
time.sleep(self.delay)
if __name__ == "__main__":
with cdtemp():
shutil.copy(args.cfgfile, ".")
badfilelist = args.badfilelist
if badfilelist is None:
badfilelist = os.path.join(os.path.dirname(cfgfile, "../../../run/DataFiles/baddatafiles.txt"))
runcfg(os.path.basename(args.cfgfile), args.badfilelist)
|