File indexing completed on 2024-11-25 02:29:02
0001
0002
0003
0004 import argparse, contextlib, os, re, shutil, subprocess, tempfile, time
0005
0006 if __name__ == "__main__":
0007 def abspath(path):
0008 if not os.path.exists(path): raise ValueError(path+" does not exist")
0009 return os.path.abspath(path)
0010
0011 p = argparse.ArgumentParser()
0012 p.add_argument("cfgfile", type=abspath)
0013 p.add_argument("baddatafileslist", nargs="?", default=None)
0014 args = p.parse_args()
0015
0016 def runcfg(cfgfile, badfilelist):
0017 try:
0018 subprocess.check_output(["cmsRun", cfgfile], stderr=subprocess.STDOUT)
0019 except subprocess.CalledProcessError as e:
0020 if "FallbackFileOpenError" in e.output:
0021 output = e.output.split("An exception of category 'FallbackFileOpenError' occurred while")[1]
0022 filename = re.search("Failed to open the file '[^']*(/store/.*[.]root)", output).group(1)
0023 with OneAtATime(badfilelist+".tmp", 2) as f:
0024 with open(badfilelist) as f:
0025 contents = set(f.read().split())
0026 if filename in contents:
0027 raise RuntimeError(filename+"\nis already in\n"+badfilelist+"\n\nExiting to avoid an infinite loop. Maybe you have this running on the same cfg file multiple times?")
0028 contents.add(filename)
0029 contents = sorted(contents)
0030 with open(badfilelist, "w") as f:
0031 f.write("\n".join(contents)+"\n")
0032 print("found and added a bad file:\n"+filename)
0033 else:
0034 raise
0035 return runcfg(cfgfile, badfilelist)
0036 print("all files left are good")
0037
0038 @contextlib.contextmanager
0039 def cd(newdir):
0040 """http://stackoverflow.com/a/24176022/5228524"""
0041 prevdir = os.getcwd()
0042 os.chdir(os.path.expanduser(newdir))
0043 try:
0044 yield
0045 finally:
0046 os.chdir(prevdir)
0047
0048 def cdtemp(): return cd(tempfile.mkdtemp())
0049
0050 class KeepWhileOpenFile(object):
0051 def __init__(self, name, message=None):
0052 self.filename = name
0053 self.__message = message
0054 self.pwd = os.getcwd()
0055 self.fd = self.f = None
0056 self.bool = False
0057
0058 @property
0059 def wouldbevalid(self):
0060 if self: return True
0061 with self:
0062 return bool(self)
0063
0064 def __open(self):
0065 self.fd = os.open(self.filename, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
0066
0067 def __enter__(self):
0068 with cd(self.pwd):
0069 try:
0070 self.__open()
0071 except OSError:
0072 return None
0073
0074 self.f = os.fdopen(self.fd, 'w')
0075
0076 try:
0077 if self.__message is not None:
0078 self.f.write(self.__message+"\n")
0079 except IOError:
0080 pass
0081 try:
0082 self.f.close()
0083 except IOError:
0084 pass
0085 self.bool = True
0086 return True
0087
0088 def __exit__(self, *args):
0089 if self:
0090 try:
0091 with cd(self.pwd):
0092 os.remove(self.filename)
0093 except OSError:
0094 pass
0095 self.fd = self.f = None
0096 self.bool = False
0097
0098 def __nonzero__(self):
0099 return self.bool
0100
0101 class OneAtATime(KeepWhileOpenFile):
0102 def __init__(self, name, delay, message=None, printmessage=None, task="doing this"):
0103 super(OneAtATime, self).__init__(name, message=message)
0104 self.delay = delay
0105 if printmessage is None:
0106 printmessage = "Another process is already {task}! Waiting {delay} seconds."
0107 printmessage = printmessage.format(delay=delay, task=task)
0108 self.__printmessage = printmessage
0109
0110 def __enter__(self):
0111 while True:
0112 result = super(OneAtATime, self).__enter__()
0113 if result:
0114 return result
0115 print(self.__printmessage)
0116 time.sleep(self.delay)
0117
0118 if __name__ == "__main__":
0119 with cdtemp():
0120 shutil.copy(args.cfgfile, ".")
0121
0122 badfilelist = args.badfilelist
0123 if badfilelist is None:
0124 badfilelist = os.path.join(os.path.dirname(cfgfile, "../../../run/DataFiles/baddatafiles.txt"))
0125
0126 runcfg(os.path.basename(args.cfgfile), args.badfilelist)
