CommonAlignmentProducer/scripts/AlCaHLTBitMon_ParallelJobs.py

0001 #!/usr/bin/env python3
0002
0003 #-----------------------------------------------------
0004 # original author: Andrea Lucaroni
0005 # Revision:        $Revision: 1.53 $
0006 # Last update:     $Date: 2011/06/28 19:01:36 $
0007 # by:              $Author: mussgill $
0008 #-----------------------------------------------------
0009
0010 from __future__ import print_function
0011 import re
0012 import json
0013 import os
0014 import stat
0015 import sys
0016
0017 import array
0018 import pickle as pk
0019
0020 from optparse import OptionParser
0021 #####DEBUG
0022 DEBUG = 0
0023
0024 def getConfigTemplateFilename():
0025     template = os.path.expandvars('$CMSSW_BASE/src/Alignment/CommonAlignmentProducer/data/AlCaHLTBitMon_cfg_template_py')
0026     if os.path.exists(template):
0027         return template
0028     template = os.path.expandvars('$CMSSW_RELEASE_BASE/src/Alignment/CommonAlignmentProducer/data/AlCaHLTBitMon_cfg_template_py')
0029     if os.path.exists(template):
0030         return template
0031     return 'None'
0032
0033 def mkHLTKeyListList(hltkeylistfile):
0034     keylistlist = []
0035     f = open(hltkeylistfile, 'r')
0036     for line in f:
0037         keylistlist.append(line.replace('\n',''))
0038     f.close()
0039     return keylistlist
0040
0041 def parallelJobs(hltkeylistfile,jsonDir,globalTag,templateName,queue,cafsetup):
0042     PWD = os.path.abspath('.')
0043
0044     if templateName == 'default':
0045         templateFile = getConfigTemplateFilename()
0046     else:
0047         templateFile = os.path.abspath(os.path.expandvars(templateName))
0048
0049     tfile = open(templateFile, 'r')
0050     template = tfile.read()
0051     tfile.close()
0052     template = template.replace('%%%GLOBALTAG%%%', globalTag)
0053
0054     keylistlist = mkHLTKeyListList(hltkeylistfile)
0055     index = 0
0056     for keylist in keylistlist:
0057
0058         configName = 'AlCaHLTBitMon_%d_cfg.py'%index
0059         jobName = 'AlCaHLTBitMon_%d_job.csh'%index
0060         jsonFile = os.path.abspath('%(dir)s/%(index)d.json' % {'dir' : jsonDir, 'index' : index})
0061         dataFile = os.path.abspath('%(dir)s/%(index)d.data' % {'dir' : jsonDir, 'index' : index})
0062         logFile = 'AlCaHLTBitMon_%d'%index
0063
0064         dfile = open(dataFile, 'r')
0065         data = dfile.read()
0066         dfile.close()
0067
0068         config = template.replace('%%%JSON%%%', jsonFile);
0069         config = config.replace('%%%DATA%%%', data);
0070         config = config.replace('%%%KEYNAME%%%', keylist);
0071         config = config.replace('%%%LOGFILE%%%', logFile);
0072
0073         cfile = open(configName, 'w')
0074         for line in config:
0075             cfile.write(line)
0076         cfile.close()
0077
0078         jfile = open(jobName, 'w')
0079         jfile.write('#!/bin/tcsh\n\n')
0080         if cafsetup == True:
0081             jfile.write('source /afs/cern.ch/cms/caf/setup.csh\n\n')
0082
0083         jfile.write('cd $1\n\n')
0084         jfile.write('eval `scramv1 run -csh`\n\n')
0085         jfile.write('cmsRun %s\n\n'%configName)
0086         jfile.write('cp %s.log $2'%logFile)
0087         jfile.close()
0088
0089         if os.path.exists('./%s'%keylist) == False:
0090             os.system('mkdir ./%s'%keylist)
0091
0092         os.system('chmod u+x %s'%jobName)
0093         #print('bsub -q %(queue)s %(jobname)s %(pwd)s %(pwd)s/%(outdir)s' % {'queue' : queue, 'jobname' : jobName, 'pwd' : PWD, 'outdir' : keylist})
0094         os.system('bsub -q %(queue)s %(jobname)s %(pwd)s %(pwd)s/%(outdir)s' % {'queue' : queue, 'jobname' : jobName, 'pwd' : PWD, 'outdir' : keylist})
0095
0096         index = index + 1
0097
0098 def defineOptions():
0099     parser = OptionParser()
0100
0101     parser.add_option("-k", "--keylist",
0102                       dest="hltKeyListFile",
0103                       default="lista_key.txt",
0104                       help="text file with HLT keys")
0105
0106     parser.add_option("-j", "--json",
0107                       dest="jsonDir",
0108                       help="directory with the corresponding json files")
0109
0110     parser.add_option("-g", "--globalTag",
0111                       dest="globalTag",
0112                       help="the global tag to use in the config files")
0113
0114     parser.add_option("-t", "--template",
0115                       dest="template",
0116                       default="default",
0117                       help="the template to use for the config files")
0118
0119     parser.add_option("-q", "--queue",
0120                       dest="queue",
0121                       default="cmscaf1nd",
0122                       help="the queue to use (default=cmscaf1nd)")
0123
0124     parser.add_option("-c", "--cafsetup", action="store_true",
0125                       dest="cafsetup",
0126                       default=False,
0127                       help="wether the caf setup is sourced in the scripts")
0128
0129     (options, args) = parser.parse_args()
0130
0131     if len(sys.argv) == 1:
0132         print("\nUsage: %s --help"%sys.argv[0])
0133         sys.exit(0)
0134
0135     if str(options.hltKeyListFile) == 'None':
0136         print("Please provide a file with HLT keys")
0137         sys.exit(0)
0138
0139     if str(options.jsonDir) == 'None':
0140         print("Please provide a directory containing the json files")
0141         sys.exit(0)
0142
0143     if str(options.globalTag) == 'None':
0144         print("Please provide a global tag")
0145         sys.exit(0)
0146
0147     return options
0148
0149
0150 #---------------------------------------------MAIN
0151
0152 options = defineOptions()
0153 p = parallelJobs(options.hltKeyListFile,
0154                  options.jsonDir,
0155                  options.globalTag,
0156                  options.template,
0157                  options.queue,
0158                  options.cafsetup)