Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2022-03-03 02:23:11

0001 #!/usr/bin/env python3
0002 
0003 from __future__ import print_function
0004 
0005 import argparse, contextlib, os, re, shutil, subprocess, tempfile, time
0006 
0007 if __name__ == "__main__":
0008   def abspath(path):
0009     if not os.path.exists(path): raise ValueError(path+" does not exist")
0010     return os.path.abspath(path)
0011 
0012   p = argparse.ArgumentParser()
0013   p.add_argument("cfgfile", type=abspath)
0014   p.add_argument("baddatafileslist", nargs="?", default=None)
0015   args = p.parse_args()
0016 
0017 def runcfg(cfgfile, badfilelist):
0018   try:
0019     subprocess.check_output(["cmsRun", cfgfile], stderr=subprocess.STDOUT)
0020   except subprocess.CalledProcessError as e:
0021     if "FallbackFileOpenError" in e.output:
0022       output = e.output.split("An exception of category 'FallbackFileOpenError' occurred while")[1]
0023       filename = re.search("Failed to open the file '[^']*(/store/.*[.]root)", output).group(1)
0024       with OneAtATime(badfilelist+".tmp", 2) as f:
0025         with open(badfilelist) as f:
0026           contents = set(f.read().split())
0027         if filename in contents:
0028           raise RuntimeError(filename+"\nis already in\n"+badfilelist+"\n\nExiting to avoid an infinite loop.  Maybe you have this running on the same cfg file multiple times?")
0029         contents.add(filename)
0030         contents = sorted(contents)
0031         with open(badfilelist, "w") as f:
0032           f.write("\n".join(contents)+"\n")
0033         print("found and added a bad file:\n"+filename)
0034     else:
0035       raise
0036     return runcfg(cfgfile, badfilelist)
0037   print("all files left are good")
0038 
0039 @contextlib.contextmanager
0040 def cd(newdir):
0041   """http://stackoverflow.com/a/24176022/5228524"""
0042   prevdir = os.getcwd()
0043   os.chdir(os.path.expanduser(newdir))
0044   try:
0045     yield
0046   finally:
0047     os.chdir(prevdir)
0048 
0049 def cdtemp(): return cd(tempfile.mkdtemp())
0050 
0051 class KeepWhileOpenFile(object):
0052   def __init__(self, name, message=None):
0053     self.filename = name
0054     self.__message = message
0055     self.pwd = os.getcwd()
0056     self.fd = self.f = None
0057     self.bool = False
0058 
0059   @property
0060   def wouldbevalid(self):
0061     if self: return True
0062     with self:
0063       return bool(self)
0064 
0065   def __open(self):
0066     self.fd = os.open(self.filename, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
0067 
0068   def __enter__(self):
0069     with cd(self.pwd):
0070       try:
0071         self.__open()
0072       except OSError:
0073         return None
0074 
0075       self.f = os.fdopen(self.fd, 'w')
0076 
0077       try:
0078         if self.__message is not None:
0079           self.f.write(self.__message+"\n")
0080       except IOError:
0081         pass
0082       try:
0083         self.f.close()
0084       except IOError:
0085         pass
0086       self.bool = True
0087       return True
0088 
0089   def __exit__(self, *args):
0090     if self:
0091       try:
0092         with cd(self.pwd):
0093           os.remove(self.filename)
0094       except OSError:
0095         pass #ignore it
0096     self.fd = self.f = None
0097     self.bool = False
0098 
0099   def __nonzero__(self):
0100     return self.bool
0101 
0102 class OneAtATime(KeepWhileOpenFile):
0103   def __init__(self, name, delay, message=None, printmessage=None, task="doing this"):
0104     super(OneAtATime, self).__init__(name, message=message)
0105     self.delay = delay
0106     if printmessage is None:
0107       printmessage = "Another process is already {task}!  Waiting {delay} seconds."
0108     printmessage = printmessage.format(delay=delay, task=task)
0109     self.__printmessage = printmessage
0110 
0111   def __enter__(self):
0112     while True:
0113       result = super(OneAtATime, self).__enter__()
0114       if result:
0115         return result
0116       print(self.__printmessage)
0117       time.sleep(self.delay)
0118 
0119 if __name__ == "__main__":
0120   with cdtemp():
0121     shutil.copy(args.cfgfile, ".")
0122 
0123     badfilelist = args.badfilelist
0124     if badfilelist is None:
0125       badfilelist = os.path.join(os.path.dirname(cfgfile, "../../../run/DataFiles/baddatafiles.txt"))
0126 
0127     runcfg(os.path.basename(args.cfgfile), args.badfilelist)