HIPAlignmentAlgorithm/scripts/makeHippyCampaign.py

0001 #!/usr/bin/env python3
0002
0003 import argparse
0004 import contextlib
0005 import errno
0006 import glob
0007 import os
0008 import re
0009 import shutil
0010 import stat
0011 import subprocess
0012 import sys
0013
0014 thisfile = os.path.abspath(__file__)
0015
0016 def main():
0017   parser = argparse.ArgumentParser()
0018   parser.add_argument("foldername", help="folder name for the campaign.  Example: CRUZET20xy")
0019   parser.add_argument("--cmssw", default=os.environ["CMSSW_VERSION"])
0020   parser.add_argument("--scram-arch", default=os.environ["SCRAM_ARCH"])
0021   parser.add_argument("--subfolder", default="", help="subfolder within basedir to make 'foldername' in.")
0022   parser.add_argument("--merge-topic", action="append", help="things to cms-merge-topic within the CMSSW release created", default=[])
0023   parser.add_argument("--print-sys-path", action="store_true", help=argparse.SUPPRESS) #internal, don't use this
0024   parser.add_argument('--basedir', default="/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy")
0025   args = parser.parse_args()
0026
0027   basedir = args.basedir
0028   if not os.path.exists(basedir):
0029     raise FileExistsError("Base Directory does not exist!")
0030
0031   if basedir[-1] == '/':
0032     basedir = basedir[:-1] #No trailing slashed allowed
0033
0034   if args.print_sys_path:
0035     print(repr(sys.path))
0036     return
0037
0038   folder = os.path.join(basedir, args.subfolder, args.foldername)
0039
0040   mkdir_p(folder)
0041
0042   with cd(folder):
0043     if not os.path.exists(args.cmssw):
0044       os.environ["SCRAM_ARCH"] = args.scram_arch
0045       subprocess.check_call(["scram", "p", "CMSSW", args.cmssw])
0046     with cd(args.cmssw):
0047        cmsenv()
0048        for _ in args.merge_topic:
0049          subprocess.check_call(["git", "cms-merge-topic", _])
0050        os.system("eval $(scram ru -sh) && scram b -j 10")  #my cmsenv function isn't quite good enough for scram b purposes.  Also, http://stackoverflow.com/a/38792806/5228524
0051
0052        if os.path.exists("src/Alignment/HIPAlignmentAlgorithm"):
0053          HIPAlignmentAlgorithm = os.path.abspath("src/Alignment/HIPAlignmentAlgorithm")
0054        else:
0055          with cd(os.environ["CMSSW_RELEASE_BASE"]):
0056            HIPAlignmentAlgorithm = os.path.abspath("src/Alignment/HIPAlignmentAlgorithm")
0057
0058        assert os.path.exists(HIPAlignmentAlgorithm), HIPAlignmentAlgorithm
0059
0060     mkdir_p("Jobs")
0061     mkdir_p("run")
0062
0063     with cd("run"):
0064       subprocess.check_call(["git", "init"])
0065
0066       mkdir_p("Configurations")
0067       with cd("Configurations"):
0068         if not os.path.exists("align_tpl_py.txt"):
0069           shutil.copy(os.path.join(HIPAlignmentAlgorithm, "python", "align_tpl_py.txt"), ".")
0070           subprocess.check_call(["git", "add", "align_tpl_py.txt"])
0071         if not os.path.exists("common_cff_py_TEMPLATE.txt"):
0072           shutil.copy(os.path.join(HIPAlignmentAlgorithm, "python", "common_cff_py.txt"), "common_cff_py_TEMPLATE.txt")
0073           subprocess.check_call(["git", "add", "common_cff_py_TEMPLATE.txt"])
0074         mkdir_p("TrackSelection")
0075         with cd("TrackSelection"):
0076           for _ in glob.iglob(os.path.join(HIPAlignmentAlgorithm, "python", "*TrackSelection_cff_py.txt")):
0077             if not os.path.exists(os.path.basename(_)):
0078               shutil.copy(_, ".")
0079               subprocess.check_call(["git", "add", os.path.basename(_)])
0080
0081       mkdir_p("DataFiles")
0082       with cd("DataFiles"):
0083         if not os.path.exists("data_example.lst"):
0084           with open("data_example.lst", "w") as f:
0085             f.write(os.path.join(os.getcwd(), "minbias.txt") + ",,MBVertex,Datatype:0\n")
0086             f.write(os.path.join(os.getcwd(), "cosmics.txt") + ",,COSMICS,Datatype:1 APVMode:deco Bfield:3.8T\n")
0087             f.write(os.path.join(os.getcwd(), "CDCs.txt") + ",,CDCS,Datatype:1 APVMode:deco Bfield:3.8T\n")
0088           subprocess.check_call(["git", "add", "data_example.lst"])
0089         if not os.path.exists("baddatafiles.txt"):
0090           with open("baddatafiles.txt", "w") as f:
0091             f.write("If any data files are bad (e.g. not at CERN), put them here,\n")
0092             f.write("separated by newlines or spaces or nothing or whatever you like.\n")
0093             f.write("Anything else in this file, like these lines, will be ignored.\n")
0094             f.write("You can also run hippyaddtobaddatafiles.py .../align_cfg.py to automatically\n")
0095             f.write("find bad data files.\n")
0096             f.write("Running jobs will automatically pick up changes here next time they resubmit.")
0097
0098       mkdir_p("IOV")
0099       with cd("IOV"):
0100         if not os.path.exists("RunXXXXXX"):
0101           with open("RunXXXXXX", "w") as f:
0102             f.write("XXXXXX")
0103           subprocess.check_call(["git", "add", "RunXXXXXX"])
0104
0105       if not os.path.exists("submit_template.sh"):
0106         shutil.copy(os.path.join(HIPAlignmentAlgorithm, "test", "hippysubmittertemplate.sh"), "submit_template.sh")
0107         os.chmod("submit_template.sh", os.stat("submit_template.sh").st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
0108         subprocess.check_call(["git", "add", "submit_template.sh"])
0109
0110       if not os.path.exists("submit_script.sh"):
0111         shutil.copy(os.path.join(HIPAlignmentAlgorithm, "test", "hippysubmitterscript.sh"), "submit_script.sh")
0112         os.chmod("submit_script.sh", os.stat("submit_script.sh").st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
0113         subprocess.check_call(["git", "add", "submit_script.sh"])
0114
0115       print("Dumped files into ", folder)
0116
0117       try:
0118         subprocess.check_output(["git", "diff", "--staged", "--quiet"])
0119       except subprocess.CalledProcessError:
0120         subprocess.check_call(["git", "commit", "-m", "commit templates"])
0121
0122 def mkdir_p(path):
0123   """http://stackoverflow.com/a/600612/5228524"""
0124   try:
0125     os.makedirs(path)
0126   except OSError as exc:
0127     if exc.errno == errno.EEXIST and os.path.isdir(path):
0128       pass
0129     else:
0130       raise
0131
0132 @contextlib.contextmanager
0133 def cd(newdir):
0134   """http://stackoverflow.com/a/24176022/5228524"""
0135   prevdir = os.getcwd()
0136   os.chdir(os.path.expanduser(newdir))
0137   try:
0138     yield
0139   finally:
0140     os.chdir(prevdir)
0141
0142 def cmsenv():
0143   output = subprocess.check_output(["scram", "ru", "-sh"])
0144   for line in output.decode('utf8').split(";\n"):
0145     if not line.strip(): continue
0146     match1 = re.match(r'^export (\w*)="([^"]*)"$', line)
0147     match2 = re.match(r'^unset *((\w* *)*)$', line)
0148     if match1:
0149       variable, value = match1.groups()
0150       os.environ[variable] = value
0151     elif match2:
0152       for variable in match2.group(1).split():
0153         del os.environ[variable]
0154     else:
0155       raise ValueError("Bad scram ru -sh line:\n"+line)
0156   sys.path[:] = eval(subprocess.check_output([thisfile, "dummy", "--print-sys-path"]))
0157
0158 if __name__ == "__main__":
0159   main()