File indexing completed on 2023-03-17 10:39:13
0001
0002
0003 from __future__ import print_function
0004
0005 import argparse, contextlib, os, re, shutil, subprocess, tempfile, time
0006
0007 if __name__ == "__main__":
0008 def abspath(path):
0009 if not os.path.exists(path): raise ValueError(path+" does not exist")
0010 return os.path.abspath(path)
0011
0012 p = argparse.ArgumentParser()
0013 p.add_argument("cfgfile", type=abspath)
0014 p.add_argument("baddatafileslist", nargs="?", default=None)
0015 args = p.parse_args()
0016
0017 def runcfg(cfgfile, badfilelist):
0018 try:
0019 subprocess.check_output(["cmsRun", cfgfile], stderr=subprocess.STDOUT)
0020 except subprocess.CalledProcessError as e:
0021 if "FallbackFileOpenError" in e.output:
0022 output = e.output.split("An exception of category 'FallbackFileOpenError' occurred while")[1]
0023 filename = re.search("Failed to open the file '[^']*(/store/.*[.]root)", output).group(1)
0024 with OneAtATime(badfilelist+".tmp", 2) as f:
0025 with open(badfilelist) as f:
0026 contents = set(f.read().split())
0027 if filename in contents:
0028 raise RuntimeError(filename+"\nis already in\n"+badfilelist+"\n\nExiting to avoid an infinite loop. Maybe you have this running on the same cfg file multiple times?")
0029 contents.add(filename)
0030 contents = sorted(contents)
0031 with open(badfilelist, "w") as f:
0032 f.write("\n".join(contents)+"\n")
0033 print("found and added a bad file:\n"+filename)
0034 else:
0035 raise
0036 return runcfg(cfgfile, badfilelist)
0037 print("all files left are good")
0038
0039 @contextlib.contextmanager
0040 def cd(newdir):
0041 """http://stackoverflow.com/a/24176022/5228524"""
0042 prevdir = os.getcwd()
0043 os.chdir(os.path.expanduser(newdir))
0044 try:
0045 yield
0046 finally:
0047 os.chdir(prevdir)
0048
0049 def cdtemp(): return cd(tempfile.mkdtemp())
0050
0051 class KeepWhileOpenFile(object):
0052 def __init__(self, name, message=None):
0053 self.filename = name
0054 self.__message = message
0055 self.pwd = os.getcwd()
0056 self.fd = self.f = None
0057 self.bool = False
0058
0059 @property
0060 def wouldbevalid(self):
0061 if self: return True
0062 with self:
0063 return bool(self)
0064
0065 def __open(self):
0066 self.fd = os.open(self.filename, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
0067
0068 def __enter__(self):
0069 with cd(self.pwd):
0070 try:
0071 self.__open()
0072 except OSError:
0073 return None
0074
0075 self.f = os.fdopen(self.fd, 'w')
0076
0077 try:
0078 if self.__message is not None:
0079 self.f.write(self.__message+"\n")
0080 except IOError:
0081 pass
0082 try:
0083 self.f.close()
0084 except IOError:
0085 pass
0086 self.bool = True
0087 return True
0088
0089 def __exit__(self, *args):
0090 if self:
0091 try:
0092 with cd(self.pwd):
0093 os.remove(self.filename)
0094 except OSError:
0095 pass
0096 self.fd = self.f = None
0097 self.bool = False
0098
0099 def __nonzero__(self):
0100 return self.bool
0101
0102 class OneAtATime(KeepWhileOpenFile):
0103 def __init__(self, name, delay, message=None, printmessage=None, task="doing this"):
0104 super(OneAtATime, self).__init__(name, message=message)
0105 self.delay = delay
0106 if printmessage is None:
0107 printmessage = "Another process is already {task}! Waiting {delay} seconds."
0108 printmessage = printmessage.format(delay=delay, task=task)
0109 self.__printmessage = printmessage
0110
0111 def __enter__(self):
0112 while True:
0113 result = super(OneAtATime, self).__enter__()
0114 if result:
0115 return result
0116 print(self.__printmessage)
0117 time.sleep(self.delay)
0118
0119 if __name__ == "__main__":
0120 with cdtemp():
0121 shutil.copy(args.cfgfile, ".")
0122
0123 badfilelist = args.badfilelist
0124 if badfilelist is None:
0125 badfilelist = os.path.join(os.path.dirname(cfgfile, "../../../run/DataFiles/baddatafiles.txt"))
0126
0127 runcfg(os.path.basename(args.cfgfile), args.badfilelist)
