File indexing completed on 2024-11-26 02:34:10
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013 import sys
0014 import os
0015 import os.path
0016 import subprocess
0017 import shutil
0018 import string
0019 import math
0020 import urllib
0021 import time
0022 import datetime
0023 import smtplib
0024
0025
0026
0027
0028 OCT_rwx_r_r = 0o744
0029 LFLOAT_valueMagField = [0.0,2.0,3.0,3.5,3.8,4.0]
0030 TD_shiftUTC = datetime.timedelta(hours = 2)
0031
0032 LSTR_true = ['1','TRUE' ,'True' ,'true' ]
0033 LSTR_false = ['0','FALSE','False','false']
0034 STR_default = 'DEFAULT'
0035 STR_none = 'None'
0036 LSTR_auto = ['AUTO','Auto','auto']
0037 STR_nameCmsswPackage = 'DQM/SiStripMonitorClient'
0038 STR_textUsage = """ CMSSW/DQM/SiStripMonitorClient/scripts/submitDQMOfflineCAF.py
0039
0040 This script submits batch jobs to the CAF in order to process the full
0041 granularity SiStrip offline DQM.
0042 Questions and comments to: volker.adler@cern.ch
0043
0044 Func_Usage(): submitDQMOfflineCAF.py (-s, --submit | -c, --create |
0045 -h, --help)
0046 [-r, --run]
0047 [-C, --CRAB]
0048 [-S, --server]
0049 [-e, --email]
0050 [-j, --jobs]
0051 [-f, --filter]
0052 [-d, --dataset]
0053 [-o, --outpath]
0054 [-m, --mergepath]
0055
0056 Function letters: One of the following options m u s t be used.
0057
0058 -s, --submit
0059 create jobs and submit them to CAF;
0060 requires option '-r'
0061
0062 -c, --create
0063 create jobs, but do not submit them;
0064 requires option '-r'
0065
0066 -h, --help
0067 print this message
0068
0069 Other options:
0070
0071 -r, --run RUNNUMBER
0072 number of run to process;
0073 required by funtion letters '-s' and '-c'
0074
0075 -d, --dataset PRIMARY_DATASET
0076 specify dataset for DBS query;
0077 required by funtion letters '-s' and '-c'
0078
0079 -C, --CRAB TRUE/FALSE
0080 submit or submit not using CRAB;
0081 default: TRUE
0082
0083 NOTE: This script runs only with CRAB 2.4.0 or higher.
0084
0085 -S, --server CRAB_SERVER
0086 CRAB server to use;
0087 available: None (default)
0088 caf (works, but slow)
0089 bari (CRAB version >= 2.4.1,
0090 s. https://twiki.cern.ch/twiki/bin/view/CMS/CrabServer#Server_available_for_users)
0091
0092 NOTE: CRAB server submission is disabled at the moment.
0093
0094 -e, --email EMAIL_ADDRESS
0095 where the CRAB server should send its messages;
0096 default: volker.adler@cern.ch
0097
0098 -j, --jobs NUMBER
0099 number of jobs to create;
0100 default: 10
0101
0102 -g, --global-tag GLOBAL_TAG
0103 global tag to be used;
0104 default: CRAFT_V4P::All
0105
0106 -M, --magnetic-field FIELD
0107 specification of field to be used;
0108 can be:
0109 - average field during run in Tesla, given as float (e.g.: 3.8),
0110 - specification referring to existing configuration files (e.g.: 38T)
0111 (s. Configuration/StandardSequences/python/MagneticField_[FIELD]_cff.py),
0112 - usage of automatic average field determination from CMS WBM by using "auto"
0113 default: 38T
0114
0115 NOTE: "auto" is disabled for the moment due to unavailablity of CMS WBM outside '.cms' network.
0116
0117 -f, --filter TRUE/FALSE
0118 use or use not HLT filters to select events to process;
0119 default: FALSE
0120
0121 -o, --outpath PATH
0122 path to copy job output *.root files to;
0123 currently (almost) no check performed;
0124 must be in AFS or CASTOR
0125 default: /castor/cern.ch/user/c/cctrack/DQM
0126
0127 -m, --mergepath PATH
0128 path to merge the job output *.root files;
0129 currently (almost) no check performed;
0130 must be in AFS or on local computer (e.g. /tmp/[user])
0131 default: /afs/cern.ch/cms/CAF/CMSCOMM/COMM_TRACKER/DQM/SiStrip/jobs/merged
0132 """
0133 LSTR_datatiers = ['RECO','RAW']
0134
0135 LSTR_wordArgument = sys.argv[1:]
0136
0137 BOOL_CRAB = True
0138 LSTR_server = [STR_none,'caf','bari']
0139 STR_server = LSTR_server[0]
0140 STR_email = 'volker.adler@cern.ch'
0141 INT_jobs = 10
0142 STR_globalTag = 'CRAFT_V4P::All'
0143 STR_magField = '38T'
0144 BOOL_magFieldAuto = False
0145 BOOL_filter = False
0146 STR_outpath = '/castor/cern.ch/user/c/cctrack/DQM'
0147 BOOL_useCastor = True
0148 STR_mergepath = '/afs/cern.ch/cms/CAF/CMSCOMM/COMM_TRACKER/DQM/SiStrip/jobs/merged'
0149
0150 LSTR_functionLetters = ['-s','-c','-h']
0151 DICT_functionLetters = {'--submit':LSTR_functionLetters[0],
0152 '--create':LSTR_functionLetters[1],
0153 '--help' :LSTR_functionLetters[2]}
0154 LSTR_optionLetters = ['-r','-C','-S','-e','-j','-M','-g','-f','-d','-o','-m']
0155 DICT_optionLetters = {'--run' :LSTR_optionLetters[0],
0156 '--CRAB' :LSTR_optionLetters[1],
0157 '--server' :LSTR_optionLetters[2],
0158 '--email' :LSTR_optionLetters[3],
0159 '--jobs' :LSTR_optionLetters[4],
0160 '--magnetic-field':LSTR_optionLetters[5],
0161 '--global-tag' :LSTR_optionLetters[6],
0162 '--filter' :LSTR_optionLetters[7],
0163 '--dataset' :LSTR_optionLetters[8],
0164 '--outpath' :LSTR_optionLetters[9],
0165 '--mergepath' :LSTR_optionLetters[10]}
0166 STR_mailSmtp = 'localhost'
0167 STR_mailServer = '@mail.cern.ch'
0168 STR_mailTextOpener = """Dear """ + os.getenv('USER').capitalize() + """,
0169
0170 on """ + str(time.ctime()) + """, you have submitted run """
0171 STR_mailText = """
0172 for SiStrip offline DQM at the CAF.
0173 Unfortunately, this needed to be done from your private account. So, only you
0174 are able to finalize this submission -- even after the end of your shift.
0175 To do so, please forward all emails from the LSF batch system referring to the
0176 respective jobs to the list t h i s message was sent to.
0177 -- and then your shift is r e a l l y done :-)
0178
0179 We are very sorry for the inconvenience.
0180 Thanks a lot!
0181
0182 Best regards,
0183 your SiStrip DQM team
0184
0185 P.S.:
0186 To reply to this email, simply use the "Reply to all" function of your email
0187 client.
0188 """
0189
0190
0191
0192
0193 global Dict_arguments
0194 global Str_run
0195 global Bool_CRAB
0196 global Str_server
0197 global Str_email
0198 global Int_jobs
0199 global Str_globalTag
0200 global Bool_filter
0201 global Str_dataset
0202 global Str_datatier
0203 global Str_magField
0204 global Float_magField
0205 global Bool_magFieldAuto
0206 global Str_outpath
0207 global Bool_useCastor
0208 global Str_mergepath
0209
0210 global Str_pathCurrentDir
0211 global Str_pathCmsswBase
0212 global Str_nameCmsswRel
0213 global Str_pathCmsswBasePackage
0214 global Str_nameRun
0215 global Str_pathRunIncludeDir
0216 global Str_pathInputFilesCAFCff
0217 global Int_jobsNew
0218 global Str_magField
0219
0220 Dict_arguments = {}
0221 Bool_CRAB = BOOL_CRAB
0222 Str_server = STR_server
0223 Str_email = STR_email
0224 Int_jobs = INT_jobs
0225 Str_globalTag = STR_globalTag
0226 Bool_filter = BOOL_filter
0227 Str_magField = STR_magField
0228 Float_magField = float(Str_magField[:-1])/10.
0229 Bool_magFieldAuto = BOOL_magFieldAuto
0230 Str_outpath = STR_outpath
0231 Bool_useCastor = BOOL_useCastor
0232 Str_mergepath = STR_mergepath
0233
0234
0235
0236
0237 def Func_Usage():
0238 """ Function Func_Usage():
0239 Displays usage of the script
0240 """
0241 print(STR_textUsage)
0242
0243
0244
0245
0246 def Func_Exit():
0247 """ Function Func_Exit():
0248 Exit after error
0249 """
0250 print(' exit')
0251 print()
0252 sys.exit(1)
0253
0254
0255
0256
0257 def Func_ExitUsage():
0258 """ Function Func_ExitUsage():
0259 Exit after wrong invocation of script
0260 """
0261 print(' exit')
0262 print()
0263 Func_Usage()
0264 sys.exit(1)
0265
0266
0267
0268
0269 def Func_ExitBool(int_index):
0270 """ Function Func_ExitBool():
0271 Exit after wrong assignment of bool option
0272 """
0273 print('> submitDQMOfflineCAF.py > option %s expects 0/1, FALSE/TRUE, False/True or false/true' %(DICT_optionLetters.items()[int_index]))
0274 Func_Exit()
0275
0276
0277
0278
0279 def Func_MkDir(str_path):
0280 """ Function Func_MkDir():
0281 Create new directory
0282 """
0283 shutil.rmtree(str_path, True)
0284 os.mkdir(str_path)
0285
0286
0287
0288
0289 def Func_MagConfig(float_magFieldMeasured):
0290 """ Func_MagConfig(float_magFieldMeasured):
0291 Determine configuration to be used for a given magnetic field
0292 """
0293 float_magField = 0.0
0294 for float_valueMagField in LFLOAT_valueMagField:
0295 if math.fabs(float_valueMagField-float_magFieldMeasured) < math.fabs(float_magField-float_magFieldMeasured):
0296 float_magField = float_valueMagField
0297 return float_magField
0298
0299
0300
0301 print()
0302
0303
0304
0305 Str_pathCurrentDir = os.getcwd()
0306 Str_pathCmsswBase = os.getenv('CMSSW_BASE')
0307 if not Str_pathCmsswBase:
0308 print('> submitDQMOfflineCAF.py > CMSSW environment not set properly;')
0309 print(' first do')
0310 print()
0311 print(' $ cd [your/CMSSW/release/area]/src')
0312 print(' $ cmsenv')
0313 print()
0314 Func_Exit()
0315 Str_nameCmsswRel = os.getenv('CMSSW_VERSION')
0316 Str_pathCmsswBasePackage = Str_pathCmsswBase + '/src/' + STR_nameCmsswPackage
0317 str_suffixShell = 'csh'
0318 if not os.getenv('SHELL')[-3:] == str_suffixShell:
0319 str_suffixShell = 'sh'
0320
0321
0322
0323 if len(LSTR_wordArgument) == 0:
0324 Func_ExitUsage()
0325 int_nFunctionLetters = 0
0326 for str_argument in LSTR_wordArgument:
0327 if str_argument in LSTR_functionLetters or\
0328 str_argument in DICT_functionLetters :
0329 int_nFunctionLetters += 1
0330 if int_nFunctionLetters == 0:
0331 print('> submitDQMOfflineCAF.py > no or unknown function letter used')
0332 Func_ExitUsage()
0333 elif int_nFunctionLetters > 1:
0334 print('> submitDQMOfflineCAF.py > too many function letter used')
0335 Func_ExitUsage()
0336
0337
0338
0339 str_argumentFormer = ''
0340 bool_standBy = False
0341 for str_argument in LSTR_wordArgument:
0342 if not ( str_argument in LSTR_functionLetters or str_argument in DICT_functionLetters or\
0343 str_argument in LSTR_optionLetters or str_argument in DICT_optionLetters ):
0344 if str_argument[0] == '-':
0345 print('> submitDQMOfflineCAF.py > unknown option used')
0346 Func_ExitUsage()
0347 if not bool_standBy:
0348 print('> submitDQMOfflineCAF.py > value without option used')
0349 Func_ExitUsage()
0350 Dict_arguments[str_argumentFormer] = str_argument
0351 bool_standBy = False
0352 else:
0353 if bool_standBy:
0354 Dict_arguments[str_argumentFormer] = STR_default
0355 if str_argumentFormer in LSTR_optionLetters or\
0356 str_argumentFormer in DICT_optionLetters:
0357 print('> submitDQMOfflineCAF.py > option "%s" w/o value' %(str_argumentFormer))
0358 print(' default used')
0359 print()
0360 bool_standBy = not ( str_argument in LSTR_functionLetters or\
0361 str_argument in DICT_functionLetters )
0362 if not bool_standBy:
0363 Dict_arguments[str_argument] = STR_default
0364 str_argumentFormer = str_argument
0365 if bool_standBy:
0366 Dict_arguments[str_argumentFormer] = STR_default
0367 if str_argumentFormer in LSTR_optionLetters or\
0368 str_argumentFormer in DICT_optionLetters :
0369 print('> submitDQMOfflineCAF.py > option "%s" w/o value' %(str_argumentFormer))
0370 print(' default used')
0371 print()
0372
0373
0374
0375 dict_arguments = Dict_arguments
0376 for str_key, str_value in dict_arguments.items():
0377 if str_key in DICT_functionLetters.keys():
0378 del Dict_arguments[str_key]
0379 Dict_arguments[DICT_functionLetters[str_key]] = str_value
0380 if str_key in DICT_optionLetters.keys():
0381 del Dict_arguments[str_key]
0382 Dict_arguments[DICT_optionLetters[str_key]] = str_value
0383
0384
0385
0386 if LSTR_functionLetters[2] in Dict_arguments:
0387 Func_Usage()
0388 sys.exit(0)
0389
0390
0391
0392
0393 if LSTR_optionLetters[0] in Dict_arguments and\
0394 Dict_arguments[LSTR_optionLetters[0]] != STR_default :
0395 Str_run = Dict_arguments[LSTR_optionLetters[0]]
0396 else:
0397 print('> submitDQMOfflineCAF.py > no run number given')
0398 Func_Exit()
0399
0400 if LSTR_optionLetters[1] in Dict_arguments and\
0401 Dict_arguments[LSTR_optionLetters[1]] != STR_default :
0402 if Dict_arguments[LSTR_optionLetters[1]] in LSTR_true:
0403 Bool_CRAB = True
0404 elif Dict_arguments[LSTR_optionLetters[1]] in LSTR_false:
0405 Bool_CRAB = False
0406 else:
0407 Func_ExitBool(1)
0408
0409 if LSTR_optionLetters[2] in Dict_arguments and\
0410 Dict_arguments[LSTR_optionLetters[2]] != STR_default :
0411 Str_server = Dict_arguments[LSTR_optionLetters[2]]
0412
0413 if LSTR_optionLetters[3] in Dict_arguments and\
0414 Dict_arguments[LSTR_optionLetters[3]] != STR_default :
0415 Str_email = Dict_arguments[LSTR_optionLetters[3]]
0416
0417 if LSTR_optionLetters[4] in Dict_arguments and\
0418 Dict_arguments[LSTR_optionLetters[4]] != STR_default :
0419 Int_jobs = int(Dict_arguments[LSTR_optionLetters[4]])
0420
0421 if LSTR_optionLetters[5] in Dict_arguments and\
0422 Dict_arguments[LSTR_optionLetters[5]] != STR_default :
0423 Str_magField = Dict_arguments[LSTR_optionLetters[5]]
0424 if Str_magField in LSTR_auto:
0425
0426 print('> submitDQMOfflineCAF.py > automatic determination of magnetic field disabled at the moment')
0427 Func_Exit()
0428
0429 if LSTR_optionLetters[6] in Dict_arguments and\
0430 Dict_arguments[LSTR_optionLetters[6]] != STR_default :
0431 Str_globalTag = Dict_arguments[LSTR_optionLetters[6]]
0432
0433 if LSTR_optionLetters[7] in Dict_arguments and\
0434 Dict_arguments[LSTR_optionLetters[7]] != STR_default :
0435 if Dict_arguments[LSTR_optionLetters[7]] in LSTR_true:
0436 Bool_filter = True
0437 elif Dict_arguments[LSTR_optionLetters[7]] in LSTR_false:
0438 Bool_filter = False
0439 else:
0440 Func_ExitBool(7)
0441
0442 if LSTR_optionLetters[8] in Dict_arguments and\
0443 Dict_arguments[LSTR_optionLetters[8]] != STR_default :
0444 Str_dataset = Dict_arguments[LSTR_optionLetters[8]]
0445 else:
0446 print('> submitDQMOfflineCAF.py > no primary dataset given')
0447 Func_Exit()
0448
0449 if LSTR_optionLetters[9] in Dict_arguments and\
0450 Dict_arguments[LSTR_optionLetters[9]] != STR_default :
0451 Str_outpath = Dict_arguments[LSTR_optionLetters[9]]
0452
0453 if LSTR_optionLetters[10] in Dict_arguments and\
0454 Dict_arguments[LSTR_optionLetters[10]] != STR_default :
0455 Str_mergepath = Dict_arguments[LSTR_optionLetters[10]]
0456
0457
0458
0459
0460 if Bool_CRAB:
0461 str_buffer = subprocess.getoutput('which crab')
0462 if str_buffer.find('which: no crab in') >= 0:
0463 str_suffixShell = 'csh'
0464 if not os.getenv('SHELL')[-3:] == str_suffixShell:
0465 str_suffixShell = 'sh'
0466 print('> submitDQMOfflineCAF.py > CRAB environment not set properly;')
0467 print(' please use')
0468 print()
0469 print(' $ source /afs/cern.ch/cms/ccs/wm/scripts/Crab/crab.%s' %(str_suffixShell))
0470 print()
0471 Func_Exit()
0472
0473 if not Str_server in LSTR_server:
0474 print('> submitDQMOfflineCAF.py > CRAB server "%s" not available' %(Str_server))
0475 Func_Exit()
0476
0477 if Int_jobs == 0:
0478 Int_jobs = 1
0479 print('> submitDQMOfflineCAF.py > number of requested jobs was 0')
0480 print(' set to 1')
0481
0482 if Str_magField in LSTR_auto:
0483
0484 print('> submitDQMOfflineCAF.py > automatic determination of magnetic field disabled at the moment')
0485 Func_Exit()
0486 elif Str_magField[-1] == 'T':
0487 bool_foundField = False
0488 for float_valueMagField in LFLOAT_valueMagField:
0489 if str(int(float_valueMagField*10)) == Str_magField[:-1]:
0490 Float_magField = float_valueMagField
0491 bool_foundField = True
0492 break
0493 if not bool_foundField:
0494 print('> submitDQMOfflineCAF.py > no magnet configuration for \'%s\' available' %(Str_magField))
0495 Func_Exit()
0496 else:
0497 Float_magField = float(Str_magField)
0498
0499
0500 str_datatier = Str_dataset.split('/')[-1]
0501 if str_datatier == 'RAW-RECO':
0502 Str_datatier = LSTR_datatiers[1]
0503 else:
0504 Str_datatier = str_datatier
0505 Str_datatier = Str_dataset.split('/')[-1]
0506 if not Str_datatier in LSTR_datatiers:
0507 print('> submitDQMOfflineCAF.py > datatier "%s" not processable' %(Str_datatier))
0508 Func_Exit()
0509
0510
0511 if Str_outpath.split('/')[1] == 'afs':
0512 Bool_useCastor = False
0513 elif Str_outpath.split('/')[1] != 'castor':
0514 print('> submitDQMOfflineCAF.py > output path not accepted')
0515 Func_ExitUsage()
0516 str_castorCp = 'cp'
0517 if Bool_useCastor:
0518 str_castorCp = 'rfcp'
0519
0520 if Str_mergepath.split('/')[1] != 'afs':
0521 print('> submitDQMOfflineCAF.py > merge path not accepted')
0522 Func_ExitUsage()
0523
0524
0525
0526
0527 Str_nameRun = 'R' + Str_run.zfill(9)
0528
0529 Func_MkDir(Str_nameRun)
0530 Str_pathRunIncludeDir = Str_pathCmsswBasePackage + '/python/' + Str_nameRun
0531 Func_MkDir(Str_pathRunIncludeDir)
0532 str_nameInputFilesFile = Str_nameRun + '/' + Str_nameRun + '.txt'
0533 str_nameRunIncludeDir = STR_nameCmsswPackage.replace('/','.') + '.' + Str_nameRun
0534
0535
0536
0537
0538 int_nInputFiles = 0
0539 file_inputFilesCff = file(str_nameInputFilesFile, 'w')
0540
0541 str_dbsParams = urllib.urlencode({'dbsInst':'cms_dbs_prod_global', 'blockName':'*', 'dataset':Str_dataset, 'userMode':'user', 'run':Str_run, 'what':'py'})
0542 file_dbsOutput = urllib.urlopen("https://cmsweb.cern.ch/dbs_discovery/getLFN_txt", str_dbsParams)
0543 for str_iLine in file_dbsOutput.readlines():
0544 lstr_wordsLine = str_iLine.split("/")
0545 if len(lstr_wordsLine) >= 5:
0546 if lstr_wordsLine[1] == 'store' and\
0547 lstr_wordsLine[2] == 'data' and\
0548 Str_dataset.find(lstr_wordsLine[3]) >= 0 and\
0549 Str_dataset.find(lstr_wordsLine[4]) >= 0 and\
0550 lstr_wordsLine[5] == str_datatier :
0551 int_nInputFiles += 1
0552 file_inputFilesCff.write(str_iLine)
0553 if int_nInputFiles == 0:
0554 print('> submitDQMOfflineCAF.py > no input files found in DBS for run %s in dataset %s' %(Str_run,Str_dataset))
0555 Func_Exit()
0556 file_inputFilesCff.close()
0557 print('> submitDQMOfflineCAF.py > input files for run %s: %i' %(Str_run,int_nInputFiles))
0558 if int_nInputFiles < Int_jobs:
0559 Int_jobs = int_nInputFiles
0560 print(' number of requested jobs reduced accordingly')
0561 Int_jobsNew = Int_jobs
0562 print()
0563
0564
0565 if Bool_magFieldAuto:
0566
0567 str_cmsmonParams = urllib.urlencode({'RUN':Str_run})
0568 file_cmsmonOutput = urllib.urlopen("http://cmsmon.cern.ch/cmsdb/servlet/RunSummary", str_cmsmonParams)
0569 str_timeBegin = ''
0570 str_timeEnd = ''
0571 for str_cmsmonOutput in file_cmsmonOutput.readlines():
0572 if str_cmsmonOutput.find('HREF=Component?RUN=' + Str_run + '&NAME=TRACKER') >= 0:
0573 lstr_timeQuery = str_cmsmonOutput.split('HREF=Component?RUN=' + Str_run + '&NAME=TRACKER&')[1].split('>TRACKER')[0].split('&')
0574 for str_timeQuery in lstr_timeQuery:
0575 str_nameStamp = str_timeQuery.split('=')[0]
0576 lstr_timeDate = str_timeQuery.split('=')[1].split('_')[0].split('.')
0577 lstr_timeTime = str_timeQuery.split('=')[1].split('_')[1].split(':')
0578 dt_stampOld = datetime.datetime(int(lstr_timeDate[0]),int(lstr_timeDate[1]),int(lstr_timeDate[2]),int(lstr_timeTime[0]),int(lstr_timeTime[1]),int(lstr_timeTime[2]))
0579 dt_stampNew = dt_stampOld - TD_shiftUTC
0580 str_timeStamp = str(dt_stampNew).replace('-','.')
0581 if str_nameStamp == 'TIME_BEGIN':
0582 str_timeBegin = str_timeStamp
0583 elif str_nameStamp == 'TIME_END':
0584 str_timeEnd = str_timeStamp
0585
0586 str_cmsmonParams = urllib.urlencode({'TIME_BEGIN':str_timeBegin, 'TIME_END':str_timeEnd})
0587 file_cmsmonOutput = urllib.urlopen("http://cmsmon.cern.ch/cmsdb/servlet/MagnetHistory", str_cmsmonParams)
0588 bool_foundField = False
0589 for str_cmsmonOutput in file_cmsmonOutput.readlines():
0590 if str_cmsmonOutput.find('BFIELD, Tesla') >= 0:
0591 Float_magField = float(str_cmsmonOutput.split('</A>')[0].split('>')[-1])
0592 bool_foundField = True
0593 break
0594 if not bool_foundField:
0595 print('> submitDQMOfflineCAF.py > could not extract magnetic field')
0596 print(' please provide value')
0597 Func_Exit()
0598
0599 float_magField = Func_MagConfig(Float_magField)
0600 Str_magField = str(int(float_magField*10)) + 'T'
0601 print('> submitDQMOfflineCAF.py > (average) magnetic field in run %s: %s T' %(Str_run,Float_magField))
0602 print(' using %s T for configuration' %(float_magField))
0603 print()
0604
0605
0606
0607 int_nLinesRead = 0
0608 file_inputFilesCff = file(str_nameInputFilesFile)
0609 lstr_linesInput = file_inputFilesCff.readlines()
0610 file_inputFilesCff.close()
0611
0612
0613 str_sedCommand = 'sed '
0614 str_sedCommand += '-e \"s#xMAG_FIELDx#' + Str_magField + '#g\" '
0615 str_sedCommand += '-e \"s#xGLOBAL_TAGx#' + Str_globalTag + '#g\" '
0616 str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameRunIncludeDir + '#g\" '
0617 str_sedCommand += '-e \"s#xMERGE_PATHx#' + Str_mergepath + '#g\" '
0618 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripCAFHarvest_template_cfg.py > ' + Str_nameRun + '/SiStripCAFHarvest_cfg.py'
0619 os.system(str_sedCommand)
0620 str_sedCommand = 'sed '
0621 str_sedCommand += '-e \"s#xCMSSW_BASEx#' + Str_pathCmsswBase + '#g\" '
0622 str_sedCommand += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
0623 str_sedCommand += '-e \"s#xMERGE_PATHx#' + Str_mergepath + '#g\" '
0624 str_sedCommand += '-e \"s#xCURRENT_DIRx#' + Str_pathCurrentDir + '#g\" '
0625 str_sedCommand += '-e \"s#xDATA_TIERx#' + Str_datatier + '#g\" '
0626 str_sedCommand += '-e \"s#xCMSSW_VERSIONx#' + Str_nameCmsswRel + '#g\" '
0627 str_sedCommand += Str_pathCmsswBasePackage + '/scripts/SiStripDQMCAFHarvest_template.job > ' + Str_nameRun + '/SiStripCAFHarvest.job'
0628 os.system(str_sedCommand)
0629
0630 Str_pathInputFilesCAFCff = Str_pathRunIncludeDir + '/inputFilesCAF_cff.py'
0631 file_inputFilesCAFCff = file(Str_pathInputFilesCAFCff, 'w')
0632 file_inputFilesCAFCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n processingMode = cms.untracked.string( \'Runs\' ),\n fileNames = cms.untracked.vstring(\n')
0633 for int_iJob in range(1,Int_jobs+1):
0634 str_lineInput = Str_outpath + '/SiStripDQMOfflineGlobalRunCAF-' + Str_nameRun + '_' + str(int_iJob) + '.root'
0635 if Bool_useCastor:
0636 str_lineInput = 'rfio:' + str_lineInput
0637 str_lineInput = ' \'' + str_lineInput + '\''
0638 if int_iJob == Int_jobs:
0639 str_lineInput += '\n'
0640 file_inputFilesCAFCff.write(str_lineInput)
0641 break
0642 str_lineInput += ',\n'
0643 file_inputFilesCAFCff.write(str_lineInput)
0644 file_inputFilesCAFCff.write(' )\n)\n')
0645 file_inputFilesCAFCff.close()
0646
0647 str_sedCommandCommon = 'sed '
0648 if Bool_filter:
0649 str_sedCommandCommon += '-e \"s#xHLT_FILTERx# #g\" '
0650 else:
0651 str_sedCommandCommon += '-e \"s#xHLT_FILTERx#\# #g\" '
0652 if Str_datatier == 'RECO':
0653 str_sedCommandCommon += '-e \"s#xRECO_FROM_RAWx#\# #g\" '
0654 str_sedCommandCommon += '-e \"s#xDQM_FROM_RAWx#\# #g\" '
0655 else:
0656 str_sedCommandCommon += '-e \"s#xRECO_FROM_RAWx# #g\" '
0657 str_sedCommandCommon += '-e \"s#xDQM_FROM_RAWx# #g\" '
0658 str_sedCommandCommon += '-e \"s#xMAG_FIELDx#' + Str_magField + '#g\" '
0659 str_sedCommandCommon += '-e \"s#xGLOBAL_TAGx#' + Str_globalTag + '#g\" '
0660 str_sedCommandCommon += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
0661
0662 if Bool_CRAB:
0663 os.chdir(Str_nameRun)
0664 str_outputDir = '.'
0665
0666 str_sedCommand = str_sedCommandCommon
0667 str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameRunIncludeDir + '#g\" '
0668 str_sedCommand += '-e \"s#xOUTPUT_DIRECTORYx#' + str_outputDir + '#g\" '
0669 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineGlobalRunCAF_template_cfg.py > SiStripDQMOfflineGlobalRunCAF_cfg.py'
0670 os.system(str_sedCommand)
0671
0672 str_pathInputFilesJobCff = Str_pathRunIncludeDir + '/inputFiles_cff.py'
0673 file_inputFilesJobCff = file(str_pathInputFilesJobCff, 'w')
0674 file_inputFilesJobCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n fileNames = cms.untracked.vstring (\n')
0675 nLines = 0
0676 for str_linesInput in lstr_linesInput:
0677 nLines += 1
0678 str_correctedLine1 = str_linesInput.replace(') );',',')
0679 str_correctedLine = str_correctedLine1.replace(' ] );',',')
0680 if nLines == len(lstr_linesInput):
0681 str_actualLine = str_correctedLine.replace(',','\n )\n)\n')
0682 elif nLines%255 == 0:
0683 str_actualLine = str_correctedLine.replace(',','\n )\n)\nsource.fileNames.extend(\n (')
0684 else:
0685 str_actualLine = str_correctedLine
0686 file_inputFilesJobCff.write(str_actualLine)
0687 file_inputFilesJobCff.close()
0688
0689 lstr_outpath = Str_outpath.split('/', 3)
0690 str_outpath = lstr_outpath[0] + '/' + lstr_outpath[1] + '/' + lstr_outpath[2]
0691 str_sedCommand = 'sed '
0692 str_sedCommand += '-e \"s#xSERVER_NAMEx#' + Str_server + '#g\" '
0693 str_sedCommand += '-e \"s#xDATASETPATHx#' + Str_dataset + '#g\" '
0694 str_sedCommand += '-e \"s#xRUNSELECTIONx#' + Str_run + '#g\" '
0695 str_sedCommand += '-e \"s#xNUMBER_OF_JOBSx#' + str(Int_jobs) + '#g\" '
0696 str_sedCommand += '-e \"s#xEMAILx#' + Str_email + '#g\" '
0697 str_sedCommand += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
0698 str_sedCommand += '-e \"s#xSTORAGE_PATHx#' + str_outpath + '#g\" '
0699 str_sedCommand += '-e \"s#xLFNx#' + lstr_outpath[3] + '#g\" '
0700 str_sedCommand += '-e \"s#xCOPY_DATAx#' + str(int(Bool_useCastor)) + '#g\" '
0701 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineCAF_template.crab > crab.cfg'
0702 os.system(str_sedCommand)
0703 os.chdir(Str_pathCurrentDir)
0704 else:
0705
0706 int_nInputFilesJob = int(int_nInputFiles/Int_jobs) + 1
0707 if int_nInputFiles%Int_jobs == 0:
0708 int_nInputFilesJob -= 1
0709 if int_nInputFiles == int_nInputFilesJob*(Int_jobs-1) and Int_jobs > 1:
0710 Int_jobs -= 1
0711
0712 for int_iJob in range(1,Int_jobs+1):
0713 str_nameJob = Str_nameRun + '_' + str(int_iJob).zfill(4)
0714
0715 str_nameJobDir = Str_nameRun + "/" + str_nameJob
0716 str_outputDir = '/tmp/' + os.getenv('USER') + '/' + str_nameJobDir
0717 str_pathJobIncludeDir = Str_pathRunIncludeDir + '/' + str_nameJob
0718 str_nameJobIncludeDir = STR_nameCmsswPackage.replace('/','.') + '.' + str_nameJobDir.replace('/','.')
0719 os.mkdir(str_nameJobDir)
0720 os.chdir(str_nameJobDir)
0721
0722 str_sedCommand = 'sed '
0723 str_sedCommand += '-e \"s#xCMSSW_BASEx#' + Str_pathCmsswBase + '#g\" '
0724 str_sedCommand += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
0725 str_sedCommand += '-e \"s#xJOB_NAMEx#' + str_nameJob + '#g\" '
0726 str_sedCommand += '-e \"s#xCURRENT_DIRx#' + Str_pathCurrentDir + '#g\" '
0727 str_sedCommand += '-e \"s#xCOPYx#' + str_castorCp + '#g\" '
0728 str_sedCommand += '-e \"s#xOUTPUT_DIRx#' + Str_outpath + '#g\" '
0729 str_sedCommand += Str_pathCmsswBasePackage + '/scripts/SiStripDQMOfflineCAF_template.job > SiStripDQMOfflineCAF.job'
0730 os.system(str_sedCommand)
0731
0732 str_sedCommand = str_sedCommandCommon
0733 str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameJobIncludeDir + '#g\" '
0734 str_sedCommand += '-e \"s#xOUTPUT_DIRECTORYx#' + str_outputDir + '#g\" '
0735 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineGlobalRunCAF_template_cfg.py > SiStripDQMOfflineGlobalRunCAF_cfg.py'
0736 os.system(str_sedCommand)
0737
0738 os.mkdir(str_pathJobIncludeDir)
0739
0740 str_pathInputFilesJobCff = str_pathJobIncludeDir + '/inputFiles_cff.py'
0741 file_inputFilesJobCff = file(str_pathInputFilesJobCff, 'w')
0742 file_inputFilesJobCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n fileNames = cms.untracked.vstring (\n')
0743 for n_iActualLine in range(int_nLinesRead, min(int_nLinesRead+int_nInputFilesJob, int_nInputFiles)):
0744 str_linesInput = lstr_linesInput[n_iActualLine].replace(') );',',')
0745
0746 str_actualLine = str_linesInput
0747 if (n_iActualLine+1)%int_nInputFilesJob == 0 or int_nLinesRead == int_nInputFiles-1:
0748 str_actualLine = str_linesInput.split(',')[0] + '\n'
0749 file_inputFilesJobCff.write(str_actualLine)
0750 int_nLinesRead += 1
0751 file_inputFilesJobCff.write(' )\n)\n')
0752 file_inputFilesJobCff.close()
0753
0754 os.chdir(Str_pathCurrentDir)
0755
0756 if int_nLinesRead >= int_nInputFiles:
0757 Int_jobsNew = int_iJob
0758 break
0759
0760
0761
0762 os.chdir(Str_pathRunIncludeDir+'/..')
0763 os.system('scramv1 b python')
0764 os.chdir(Str_pathCurrentDir)
0765 print()
0766
0767
0768
0769 if Bool_CRAB:
0770 os.chdir(Str_nameRun)
0771 os.system('crab -create')
0772 print()
0773
0774 bool_found = False
0775 file_logCRAB = file('crab'+Str_nameRun+'/log/crab.log')
0776 for str_iLine in file_logCRAB.readlines():
0777 if str_iLine.startswith('Total of ') and str_iLine.endswith(' jobs created.\n'):
0778 bool_found = True
0779 Int_jobsNew = int(str_iLine.split()[2])
0780 break
0781 file_logCRAB.close()
0782 os.chdir(Str_pathCurrentDir)
0783 if not bool_found:
0784 print('> submitDQMOfflineCAF.py > could not extract number of jobs created by CRAB; check')
0785 print(' %s' %(Str_pathInputFilesCAFCff))
0786 print(' and modify manually, if necessary')
0787 print()
0788
0789
0790
0791 print('> submitDQMOfflineCAF.py > number of created jobs: %i' %(Int_jobsNew))
0792 print()
0793 if Int_jobsNew != Int_jobs:
0794 file_inputFilesCAFCff = file(Str_pathInputFilesCAFCff, 'w')
0795 file_inputFilesCAFCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n processingMode = cms.untracked.string( \'Runs\' ),\n fileNames = cms.untracked.vstring(\n')
0796 for int_iJob in range(1,Int_jobsNew+1):
0797 str_lineInput = Str_outpath + '/SiStripDQMOfflineGlobalRunCAF-' + Str_nameRun + '_' + str(int_iJob) + '.root'
0798 if Bool_useCastor:
0799 str_lineInput = 'rfio:' + str_lineInput
0800 str_lineInput = ' \'' + str_lineInput + '\''
0801 if int_iJob == Int_jobsNew:
0802 str_lineInput += '\n'
0803 file_inputFilesCAFCff.write(str_lineInput)
0804 break
0805 str_lineInput += ',\n'
0806 file_inputFilesCAFCff.write(str_lineInput)
0807 file_inputFilesCAFCff.write(' )\n)\n')
0808 file_inputFilesCAFCff.close()
0809
0810
0811
0812 if LSTR_functionLetters[0] in Dict_arguments:
0813 os.chdir(Str_nameRun)
0814 if Bool_CRAB:
0815 str_crabCommand = 'crab -submit -c crab' + Str_nameRun
0816 print('> submitDQMOfflineCAF.py >')
0817 print(' %s : %s' %(os.getcwd(),str_crabCommand))
0818 os.system(str_crabCommand)
0819 print()
0820 time.sleep(5)
0821 os.system('crab -status -c crab' + Str_nameRun)
0822 else:
0823 for int_iJob in range(Int_jobs):
0824 str_nameJobDir = Str_nameRun + '_' + str(int_iJob).zfill(4)
0825 os.chdir(str_nameJobDir)
0826 os.chmod('SiStripDQMOfflineCAF.job',OCT_rwx_r_r)
0827 str_batchCommand = 'bsub -q cmscaf SiStripDQMOfflineCAF.job'
0828 print('> submitDQMOfflineCAF.py >')
0829 print(' %s : %s' %(os.getcwd(),str_batchCommand))
0830 os.system(str_batchCommand)
0831 print()
0832 os.chdir('../')
0833 time.sleep(5)
0834 os.system('bjobs -q cmscaf')
0835 os.chmod('SiStripCAFHarvest.job',OCT_rwx_r_r)
0836 os.chdir(Str_pathCurrentDir)
0837
0838
0839
0840 if LSTR_functionLetters[0] in Dict_arguments and not Bool_CRAB:
0841 str_mailFrom = os.getenv('USER') + STR_mailServer
0842 str_mailTo = [str_mailFrom,
0843 'volker.adler@cern.ch',
0844 'suchandra.dutta@cern.ch',
0845 'domenico.giordano@cern.ch',
0846 'vitaliano.ciulli@cern.ch']
0847 str_mailSubject = 'Your SiStrip offline DQM shift on ' + str(datetime.date.today()) + ', run ' + Str_run
0848 str_mailText = STR_mailTextOpener + Str_run + STR_mailText
0849 str_mailMessage = """From: %s
0850 To: %s
0851 Subject: %s
0852
0853 %s
0854 """ % (str_mailFrom, ", ".join(str_mailTo), str_mailSubject, str_mailText)
0855 server = smtplib.SMTP(STR_mailSmtp)
0856 server.sendmail(str_mailFrom, str_mailTo, str_mailMessage)
0857 server.quit()
