CommonAlignmentProducer/scripts/AlCaHLTBitMon_ParallelJobs.py

0001 #!/usr/bin/env python3
0002
0003 #-----------------------------------------------------
0004 # original author: Andrea Lucaroni
0005 # Revision:        $Revision: 1.53 $
0006 # Last update:     $Date: 2011/06/28 19:01:36 $
0007 # by:              $Author: mussgill $
0008 #-----------------------------------------------------
0009
0010 import re
0011 import json
0012 import os
0013 import stat
0014 import sys
0015
0016 import array
0017 import pickle as pk
0018
0019 from optparse import OptionParser
0020 #####DEBUG
0021 DEBUG = 0
0022
0023 def getConfigTemplateFilename():
0024     template = os.path.expandvars('$CMSSW_BASE/src/Alignment/CommonAlignmentProducer/data/AlCaHLTBitMon_cfg_template_py')
0025     if os.path.exists(template):
0026         return template
0027     template = os.path.expandvars('$CMSSW_RELEASE_BASE/src/Alignment/CommonAlignmentProducer/data/AlCaHLTBitMon_cfg_template_py')
0028     if os.path.exists(template):
0029         return template
0030     return 'None'
0031
0032 def mkHLTKeyListList(hltkeylistfile):
0033     keylistlist = []
0034     f = open(hltkeylistfile, 'r')
0035     for line in f:
0036         keylistlist.append(line.replace('\n',''))
0037     f.close()
0038     return keylistlist
0039
0040 def parallelJobs(hltkeylistfile,jsonDir,globalTag,templateName,queue,cafsetup):
0041     PWD = os.path.abspath('.')
0042
0043     if templateName == 'default':
0044         templateFile = getConfigTemplateFilename()
0045     else:
0046         templateFile = os.path.abspath(os.path.expandvars(templateName))
0047
0048     tfile = open(templateFile, 'r')
0049     template = tfile.read()
0050     tfile.close()
0051     template = template.replace('%%%GLOBALTAG%%%', globalTag)
0052
0053     keylistlist = mkHLTKeyListList(hltkeylistfile)
0054     index = 0
0055     for keylist in keylistlist:
0056
0057         configName = 'AlCaHLTBitMon_%d_cfg.py'%index
0058         jobName = 'AlCaHLTBitMon_%d_job.csh'%index
0059         jsonFile = os.path.abspath('%(dir)s/%(index)d.json' % {'dir' : jsonDir, 'index' : index})
0060         dataFile = os.path.abspath('%(dir)s/%(index)d.data' % {'dir' : jsonDir, 'index' : index})
0061         logFile = 'AlCaHLTBitMon_%d'%index
0062
0063         dfile = open(dataFile, 'r')
0064         data = dfile.read()
0065         dfile.close()
0066
0067         config = template.replace('%%%JSON%%%', jsonFile);
0068         config = config.replace('%%%DATA%%%', data);
0069         config = config.replace('%%%KEYNAME%%%', keylist);
0070         config = config.replace('%%%LOGFILE%%%', logFile);
0071
0072         cfile = open(configName, 'w')
0073         for line in config:
0074             cfile.write(line)
0075         cfile.close()
0076
0077         jfile = open(jobName, 'w')
0078         jfile.write('#!/bin/tcsh\n\n')
0079         if cafsetup == True:
0080             jfile.write('source /afs/cern.ch/cms/caf/setup.csh\n\n')
0081
0082         jfile.write('cd $1\n\n')
0083         jfile.write('eval `scramv1 run -csh`\n\n')
0084         jfile.write('cmsRun %s\n\n'%configName)
0085         jfile.write('cp %s.log $2'%logFile)
0086         jfile.close()
0087
0088         if os.path.exists('./%s'%keylist) == False:
0089             os.system('mkdir ./%s'%keylist)
0090
0091         os.system('chmod u+x %s'%jobName)
0092         #print('bsub -q %(queue)s %(jobname)s %(pwd)s %(pwd)s/%(outdir)s' % {'queue' : queue, 'jobname' : jobName, 'pwd' : PWD, 'outdir' : keylist})
0093         os.system('bsub -q %(queue)s %(jobname)s %(pwd)s %(pwd)s/%(outdir)s' % {'queue' : queue, 'jobname' : jobName, 'pwd' : PWD, 'outdir' : keylist})
0094
0095         index = index + 1
0096
0097 def defineOptions():
0098     parser = OptionParser()
0099
0100     parser.add_option("-k", "--keylist",
0101                       dest="hltKeyListFile",
0102                       default="lista_key.txt",
0103                       help="text file with HLT keys")
0104
0105     parser.add_option("-j", "--json",
0106                       dest="jsonDir",
0107                       help="directory with the corresponding json files")
0108
0109     parser.add_option("-g", "--globalTag",
0110                       dest="globalTag",
0111                       help="the global tag to use in the config files")
0112
0113     parser.add_option("-t", "--template",
0114                       dest="template",
0115                       default="default",
0116                       help="the template to use for the config files")
0117
0118     parser.add_option("-q", "--queue",
0119                       dest="queue",
0120                       default="cmscaf1nd",
0121                       help="the queue to use (default=cmscaf1nd)")
0122
0123     parser.add_option("-c", "--cafsetup", action="store_true",
0124                       dest="cafsetup",
0125                       default=False,
0126                       help="wether the caf setup is sourced in the scripts")
0127
0128     (options, args) = parser.parse_args()
0129
0130     if len(sys.argv) == 1:
0131         print("\nUsage: %s --help"%sys.argv[0])
0132         sys.exit(0)
0133
0134     if str(options.hltKeyListFile) == 'None':
0135         print("Please provide a file with HLT keys")
0136         sys.exit(0)
0137
0138     if str(options.jsonDir) == 'None':
0139         print("Please provide a directory containing the json files")
0140         sys.exit(0)
0141
0142     if str(options.globalTag) == 'None':
0143         print("Please provide a global tag")
0144         sys.exit(0)
0145
0146     return options
0147
0148
0149 #---------------------------------------------MAIN
0150
0151 options = defineOptions()
0152 p = parallelJobs(options.hltKeyListFile,
0153                  options.jsonDir,
0154                  options.globalTag,
0155                  options.template,
0156                  options.queue,
0157                  options.cafsetup)