File indexing completed on 2024-04-06 12:08:42
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013 from __future__ import print_function
0014 import sys
0015 import os
0016 import os.path
0017 import subprocess
0018 import shutil
0019 import string
0020 import math
0021 import urllib
0022 import time
0023 import datetime
0024 import smtplib
0025
0026
0027
0028
0029 OCT_rwx_r_r = 0o744
0030 LFLOAT_valueMagField = [0.0,2.0,3.0,3.5,3.8,4.0]
0031 TD_shiftUTC = datetime.timedelta(hours = 2)
0032
0033 LSTR_true = ['1','TRUE' ,'True' ,'true' ]
0034 LSTR_false = ['0','FALSE','False','false']
0035 STR_default = 'DEFAULT'
0036 STR_none = 'None'
0037 LSTR_auto = ['AUTO','Auto','auto']
0038 STR_nameCmsswPackage = 'DQM/SiStripMonitorClient'
0039 STR_textUsage = """ CMSSW/DQM/SiStripMonitorClient/scripts/submitDQMOfflineCAF.py
0040
0041 This script submits batch jobs to the CAF in order to process the full
0042 granularity SiStrip offline DQM.
0043 Questions and comments to: volker.adler@cern.ch
0044
0045 Func_Usage(): submitDQMOfflineCAF.py (-s, --submit | -c, --create |
0046 -h, --help)
0047 [-r, --run]
0048 [-C, --CRAB]
0049 [-S, --server]
0050 [-e, --email]
0051 [-j, --jobs]
0052 [-f, --filter]
0053 [-d, --dataset]
0054 [-o, --outpath]
0055 [-m, --mergepath]
0056
0057 Function letters: One of the following options m u s t be used.
0058
0059 -s, --submit
0060 create jobs and submit them to CAF;
0061 requires option '-r'
0062
0063 -c, --create
0064 create jobs, but do not submit them;
0065 requires option '-r'
0066
0067 -h, --help
0068 print this message
0069
0070 Other options:
0071
0072 -r, --run RUNNUMBER
0073 number of run to process;
0074 required by funtion letters '-s' and '-c'
0075
0076 -d, --dataset PRIMARY_DATASET
0077 specify dataset for DBS query;
0078 required by funtion letters '-s' and '-c'
0079
0080 -C, --CRAB TRUE/FALSE
0081 submit or submit not using CRAB;
0082 default: TRUE
0083
0084 NOTE: This script runs only with CRAB 2.4.0 or higher.
0085
0086 -S, --server CRAB_SERVER
0087 CRAB server to use;
0088 available: None (default)
0089 caf (works, but slow)
0090 bari (CRAB version >= 2.4.1,
0091 s. https://twiki.cern.ch/twiki/bin/view/CMS/CrabServer#Server_available_for_users)
0092
0093 NOTE: CRAB server submission is disabled at the moment.
0094
0095 -e, --email EMAIL_ADDRESS
0096 where the CRAB server should send its messages;
0097 default: volker.adler@cern.ch
0098
0099 -j, --jobs NUMBER
0100 number of jobs to create;
0101 default: 10
0102
0103 -g, --global-tag GLOBAL_TAG
0104 global tag to be used;
0105 default: CRAFT_V4P::All
0106
0107 -M, --magnetic-field FIELD
0108 specification of field to be used;
0109 can be:
0110 - average field during run in Tesla, given as float (e.g.: 3.8),
0111 - specification referring to existing configuration files (e.g.: 38T)
0112 (s. Configuration/StandardSequences/python/MagneticField_[FIELD]_cff.py),
0113 - usage of automatic average field determination from CMS WBM by using "auto"
0114 default: 38T
0115
0116 NOTE: "auto" is disabled for the moment due to unavailablity of CMS WBM outside '.cms' network.
0117
0118 -f, --filter TRUE/FALSE
0119 use or use not HLT filters to select events to process;
0120 default: FALSE
0121
0122 -o, --outpath PATH
0123 path to copy job output *.root files to;
0124 currently (almost) no check performed;
0125 must be in AFS or CASTOR
0126 default: /castor/cern.ch/user/c/cctrack/DQM
0127
0128 -m, --mergepath PATH
0129 path to merge the job output *.root files;
0130 currently (almost) no check performed;
0131 must be in AFS or on local computer (e.g. /tmp/[user])
0132 default: /afs/cern.ch/cms/CAF/CMSCOMM/COMM_TRACKER/DQM/SiStrip/jobs/merged
0133 """
0134 LSTR_datatiers = ['RECO','RAW']
0135
0136 LSTR_wordArgument = sys.argv[1:]
0137
0138 BOOL_CRAB = True
0139 LSTR_server = [STR_none,'caf','bari']
0140 STR_server = LSTR_server[0]
0141 STR_email = 'volker.adler@cern.ch'
0142 INT_jobs = 10
0143 STR_globalTag = 'CRAFT_V4P::All'
0144 STR_magField = '38T'
0145 BOOL_magFieldAuto = False
0146 BOOL_filter = False
0147 STR_outpath = '/castor/cern.ch/user/c/cctrack/DQM'
0148 BOOL_useCastor = True
0149 STR_mergepath = '/afs/cern.ch/cms/CAF/CMSCOMM/COMM_TRACKER/DQM/SiStrip/jobs/merged'
0150
0151 LSTR_functionLetters = ['-s','-c','-h']
0152 DICT_functionLetters = {'--submit':LSTR_functionLetters[0],
0153 '--create':LSTR_functionLetters[1],
0154 '--help' :LSTR_functionLetters[2]}
0155 LSTR_optionLetters = ['-r','-C','-S','-e','-j','-M','-g','-f','-d','-o','-m']
0156 DICT_optionLetters = {'--run' :LSTR_optionLetters[0],
0157 '--CRAB' :LSTR_optionLetters[1],
0158 '--server' :LSTR_optionLetters[2],
0159 '--email' :LSTR_optionLetters[3],
0160 '--jobs' :LSTR_optionLetters[4],
0161 '--magnetic-field':LSTR_optionLetters[5],
0162 '--global-tag' :LSTR_optionLetters[6],
0163 '--filter' :LSTR_optionLetters[7],
0164 '--dataset' :LSTR_optionLetters[8],
0165 '--outpath' :LSTR_optionLetters[9],
0166 '--mergepath' :LSTR_optionLetters[10]}
0167 STR_mailSmtp = 'localhost'
0168 STR_mailServer = '@mail.cern.ch'
0169 STR_mailTextOpener = """Dear """ + os.getenv('USER').capitalize() + """,
0170
0171 on """ + str(time.ctime()) + """, you have submitted run """
0172 STR_mailText = """
0173 for SiStrip offline DQM at the CAF.
0174 Unfortunately, this needed to be done from your private account. So, only you
0175 are able to finalize this submission -- even after the end of your shift.
0176 To do so, please forward all emails from the LSF batch system referring to the
0177 respective jobs to the list t h i s message was sent to.
0178 -- and then your shift is r e a l l y done :-)
0179
0180 We are very sorry for the inconvenience.
0181 Thanks a lot!
0182
0183 Best regards,
0184 your SiStrip DQM team
0185
0186 P.S.:
0187 To reply to this email, simply use the "Reply to all" function of your email
0188 client.
0189 """
0190
0191
0192
0193
0194 global Dict_arguments
0195 global Str_run
0196 global Bool_CRAB
0197 global Str_server
0198 global Str_email
0199 global Int_jobs
0200 global Str_globalTag
0201 global Bool_filter
0202 global Str_dataset
0203 global Str_datatier
0204 global Str_magField
0205 global Float_magField
0206 global Bool_magFieldAuto
0207 global Str_outpath
0208 global Bool_useCastor
0209 global Str_mergepath
0210
0211 global Str_pathCurrentDir
0212 global Str_pathCmsswBase
0213 global Str_nameCmsswRel
0214 global Str_pathCmsswBasePackage
0215 global Str_nameRun
0216 global Str_pathRunIncludeDir
0217 global Str_pathInputFilesCAFCff
0218 global Int_jobsNew
0219 global Str_magField
0220
0221 Dict_arguments = {}
0222 Bool_CRAB = BOOL_CRAB
0223 Str_server = STR_server
0224 Str_email = STR_email
0225 Int_jobs = INT_jobs
0226 Str_globalTag = STR_globalTag
0227 Bool_filter = BOOL_filter
0228 Str_magField = STR_magField
0229 Float_magField = float(Str_magField[:-1])/10.
0230 Bool_magFieldAuto = BOOL_magFieldAuto
0231 Str_outpath = STR_outpath
0232 Bool_useCastor = BOOL_useCastor
0233 Str_mergepath = STR_mergepath
0234
0235
0236
0237
0238 def Func_Usage():
0239 """ Function Func_Usage():
0240 Displays usage of the script
0241 """
0242 print(STR_textUsage)
0243
0244
0245
0246
0247 def Func_Exit():
0248 """ Function Func_Exit():
0249 Exit after error
0250 """
0251 print(' exit')
0252 print()
0253 sys.exit(1)
0254
0255
0256
0257
0258 def Func_ExitUsage():
0259 """ Function Func_ExitUsage():
0260 Exit after wrong invocation of script
0261 """
0262 print(' exit')
0263 print()
0264 Func_Usage()
0265 sys.exit(1)
0266
0267
0268
0269
0270 def Func_ExitBool(int_index):
0271 """ Function Func_ExitBool():
0272 Exit after wrong assignment of bool option
0273 """
0274 print('> submitDQMOfflineCAF.py > option %s expects 0/1, FALSE/TRUE, False/True or false/true' %(DICT_optionLetters.items()[int_index]))
0275 Func_Exit()
0276
0277
0278
0279
0280 def Func_MkDir(str_path):
0281 """ Function Func_MkDir():
0282 Create new directory
0283 """
0284 shutil.rmtree(str_path, True)
0285 os.mkdir(str_path)
0286
0287
0288
0289
0290 def Func_MagConfig(float_magFieldMeasured):
0291 """ Func_MagConfig(float_magFieldMeasured):
0292 Determine configuration to be used for a given magnetic field
0293 """
0294 float_magField = 0.0
0295 for float_valueMagField in LFLOAT_valueMagField:
0296 if math.fabs(float_valueMagField-float_magFieldMeasured) < math.fabs(float_magField-float_magFieldMeasured):
0297 float_magField = float_valueMagField
0298 return float_magField
0299
0300
0301
0302 print()
0303
0304
0305
0306 Str_pathCurrentDir = os.getcwd()
0307 Str_pathCmsswBase = os.getenv('CMSSW_BASE')
0308 if not Str_pathCmsswBase:
0309 print('> submitDQMOfflineCAF.py > CMSSW environment not set properly;')
0310 print(' first do')
0311 print()
0312 print(' $ cd [your/CMSSW/release/area]/src')
0313 print(' $ cmsenv')
0314 print()
0315 Func_Exit()
0316 Str_nameCmsswRel = os.getenv('CMSSW_VERSION')
0317 Str_pathCmsswBasePackage = Str_pathCmsswBase + '/src/' + STR_nameCmsswPackage
0318 str_suffixShell = 'csh'
0319 if not os.getenv('SHELL')[-3:] == str_suffixShell:
0320 str_suffixShell = 'sh'
0321
0322
0323
0324 if len(LSTR_wordArgument) == 0:
0325 Func_ExitUsage()
0326 int_nFunctionLetters = 0
0327 for str_argument in LSTR_wordArgument:
0328 if str_argument in LSTR_functionLetters or\
0329 str_argument in DICT_functionLetters :
0330 int_nFunctionLetters += 1
0331 if int_nFunctionLetters == 0:
0332 print('> submitDQMOfflineCAF.py > no or unknown function letter used')
0333 Func_ExitUsage()
0334 elif int_nFunctionLetters > 1:
0335 print('> submitDQMOfflineCAF.py > too many function letter used')
0336 Func_ExitUsage()
0337
0338
0339
0340 str_argumentFormer = ''
0341 bool_standBy = False
0342 for str_argument in LSTR_wordArgument:
0343 if not ( str_argument in LSTR_functionLetters or str_argument in DICT_functionLetters or\
0344 str_argument in LSTR_optionLetters or str_argument in DICT_optionLetters ):
0345 if str_argument[0] == '-':
0346 print('> submitDQMOfflineCAF.py > unknown option used')
0347 Func_ExitUsage()
0348 if not bool_standBy:
0349 print('> submitDQMOfflineCAF.py > value without option used')
0350 Func_ExitUsage()
0351 Dict_arguments[str_argumentFormer] = str_argument
0352 bool_standBy = False
0353 else:
0354 if bool_standBy:
0355 Dict_arguments[str_argumentFormer] = STR_default
0356 if str_argumentFormer in LSTR_optionLetters or\
0357 str_argumentFormer in DICT_optionLetters:
0358 print('> submitDQMOfflineCAF.py > option "%s" w/o value' %(str_argumentFormer))
0359 print(' default used')
0360 print()
0361 bool_standBy = not ( str_argument in LSTR_functionLetters or\
0362 str_argument in DICT_functionLetters )
0363 if not bool_standBy:
0364 Dict_arguments[str_argument] = STR_default
0365 str_argumentFormer = str_argument
0366 if bool_standBy:
0367 Dict_arguments[str_argumentFormer] = STR_default
0368 if str_argumentFormer in LSTR_optionLetters or\
0369 str_argumentFormer in DICT_optionLetters :
0370 print('> submitDQMOfflineCAF.py > option "%s" w/o value' %(str_argumentFormer))
0371 print(' default used')
0372 print()
0373
0374
0375
0376 dict_arguments = Dict_arguments
0377 for str_key, str_value in dict_arguments.items():
0378 if str_key in DICT_functionLetters.keys():
0379 del Dict_arguments[str_key]
0380 Dict_arguments[DICT_functionLetters[str_key]] = str_value
0381 if str_key in DICT_optionLetters.keys():
0382 del Dict_arguments[str_key]
0383 Dict_arguments[DICT_optionLetters[str_key]] = str_value
0384
0385
0386
0387 if LSTR_functionLetters[2] in Dict_arguments:
0388 Func_Usage()
0389 sys.exit(0)
0390
0391
0392
0393
0394 if LSTR_optionLetters[0] in Dict_arguments and\
0395 Dict_arguments[LSTR_optionLetters[0]] != STR_default :
0396 Str_run = Dict_arguments[LSTR_optionLetters[0]]
0397 else:
0398 print('> submitDQMOfflineCAF.py > no run number given')
0399 Func_Exit()
0400
0401 if LSTR_optionLetters[1] in Dict_arguments and\
0402 Dict_arguments[LSTR_optionLetters[1]] != STR_default :
0403 if Dict_arguments[LSTR_optionLetters[1]] in LSTR_true:
0404 Bool_CRAB = True
0405 elif Dict_arguments[LSTR_optionLetters[1]] in LSTR_false:
0406 Bool_CRAB = False
0407 else:
0408 Func_ExitBool(1)
0409
0410 if LSTR_optionLetters[2] in Dict_arguments and\
0411 Dict_arguments[LSTR_optionLetters[2]] != STR_default :
0412 Str_server = Dict_arguments[LSTR_optionLetters[2]]
0413
0414 if LSTR_optionLetters[3] in Dict_arguments and\
0415 Dict_arguments[LSTR_optionLetters[3]] != STR_default :
0416 Str_email = Dict_arguments[LSTR_optionLetters[3]]
0417
0418 if LSTR_optionLetters[4] in Dict_arguments and\
0419 Dict_arguments[LSTR_optionLetters[4]] != STR_default :
0420 Int_jobs = int(Dict_arguments[LSTR_optionLetters[4]])
0421
0422 if LSTR_optionLetters[5] in Dict_arguments and\
0423 Dict_arguments[LSTR_optionLetters[5]] != STR_default :
0424 Str_magField = Dict_arguments[LSTR_optionLetters[5]]
0425 if Str_magField in LSTR_auto:
0426
0427 print('> submitDQMOfflineCAF.py > automatic determination of magnetic field disabled at the moment')
0428 Func_Exit()
0429
0430 if LSTR_optionLetters[6] in Dict_arguments and\
0431 Dict_arguments[LSTR_optionLetters[6]] != STR_default :
0432 Str_globalTag = Dict_arguments[LSTR_optionLetters[6]]
0433
0434 if LSTR_optionLetters[7] in Dict_arguments and\
0435 Dict_arguments[LSTR_optionLetters[7]] != STR_default :
0436 if Dict_arguments[LSTR_optionLetters[7]] in LSTR_true:
0437 Bool_filter = True
0438 elif Dict_arguments[LSTR_optionLetters[7]] in LSTR_false:
0439 Bool_filter = False
0440 else:
0441 Func_ExitBool(7)
0442
0443 if LSTR_optionLetters[8] in Dict_arguments and\
0444 Dict_arguments[LSTR_optionLetters[8]] != STR_default :
0445 Str_dataset = Dict_arguments[LSTR_optionLetters[8]]
0446 else:
0447 print('> submitDQMOfflineCAF.py > no primary dataset given')
0448 Func_Exit()
0449
0450 if LSTR_optionLetters[9] in Dict_arguments and\
0451 Dict_arguments[LSTR_optionLetters[9]] != STR_default :
0452 Str_outpath = Dict_arguments[LSTR_optionLetters[9]]
0453
0454 if LSTR_optionLetters[10] in Dict_arguments and\
0455 Dict_arguments[LSTR_optionLetters[10]] != STR_default :
0456 Str_mergepath = Dict_arguments[LSTR_optionLetters[10]]
0457
0458
0459
0460
0461 if Bool_CRAB:
0462 str_buffer = subprocess.getoutput('which crab')
0463 if str_buffer.find('which: no crab in') >= 0:
0464 str_suffixShell = 'csh'
0465 if not os.getenv('SHELL')[-3:] == str_suffixShell:
0466 str_suffixShell = 'sh'
0467 print('> submitDQMOfflineCAF.py > CRAB environment not set properly;')
0468 print(' please use')
0469 print()
0470 print(' $ source /afs/cern.ch/cms/ccs/wm/scripts/Crab/crab.%s' %(str_suffixShell))
0471 print()
0472 Func_Exit()
0473
0474 if not Str_server in LSTR_server:
0475 print('> submitDQMOfflineCAF.py > CRAB server "%s" not available' %(Str_server))
0476 Func_Exit()
0477
0478 if Int_jobs == 0:
0479 Int_jobs = 1
0480 print('> submitDQMOfflineCAF.py > number of requested jobs was 0')
0481 print(' set to 1')
0482
0483 if Str_magField in LSTR_auto:
0484
0485 print('> submitDQMOfflineCAF.py > automatic determination of magnetic field disabled at the moment')
0486 Func_Exit()
0487 elif Str_magField[-1] == 'T':
0488 bool_foundField = False
0489 for float_valueMagField in LFLOAT_valueMagField:
0490 if str(int(float_valueMagField*10)) == Str_magField[:-1]:
0491 Float_magField = float_valueMagField
0492 bool_foundField = True
0493 break
0494 if not bool_foundField:
0495 print('> submitDQMOfflineCAF.py > no magnet configuration for \'%s\' available' %(Str_magField))
0496 Func_Exit()
0497 else:
0498 Float_magField = float(Str_magField)
0499
0500
0501 str_datatier = Str_dataset.split('/')[-1]
0502 if str_datatier == 'RAW-RECO':
0503 Str_datatier = LSTR_datatiers[1]
0504 else:
0505 Str_datatier = str_datatier
0506 Str_datatier = Str_dataset.split('/')[-1]
0507 if not Str_datatier in LSTR_datatiers:
0508 print('> submitDQMOfflineCAF.py > datatier "%s" not processable' %(Str_datatier))
0509 Func_Exit()
0510
0511
0512 if Str_outpath.split('/')[1] == 'afs':
0513 Bool_useCastor = False
0514 elif Str_outpath.split('/')[1] != 'castor':
0515 print('> submitDQMOfflineCAF.py > output path not accepted')
0516 Func_ExitUsage()
0517 str_castorCp = 'cp'
0518 if Bool_useCastor:
0519 str_castorCp = 'rfcp'
0520
0521 if Str_mergepath.split('/')[1] != 'afs':
0522 print('> submitDQMOfflineCAF.py > merge path not accepted')
0523 Func_ExitUsage()
0524
0525
0526
0527
0528 Str_nameRun = 'R' + Str_run.zfill(9)
0529
0530 Func_MkDir(Str_nameRun)
0531 Str_pathRunIncludeDir = Str_pathCmsswBasePackage + '/python/' + Str_nameRun
0532 Func_MkDir(Str_pathRunIncludeDir)
0533 str_nameInputFilesFile = Str_nameRun + '/' + Str_nameRun + '.txt'
0534 str_nameRunIncludeDir = STR_nameCmsswPackage.replace('/','.') + '.' + Str_nameRun
0535
0536
0537
0538
0539 int_nInputFiles = 0
0540 file_inputFilesCff = file(str_nameInputFilesFile, 'w')
0541
0542 str_dbsParams = urllib.urlencode({'dbsInst':'cms_dbs_prod_global', 'blockName':'*', 'dataset':Str_dataset, 'userMode':'user', 'run':Str_run, 'what':'py'})
0543 file_dbsOutput = urllib.urlopen("https://cmsweb.cern.ch/dbs_discovery/getLFN_txt", str_dbsParams)
0544 for str_iLine in file_dbsOutput.readlines():
0545 lstr_wordsLine = str_iLine.split("/")
0546 if len(lstr_wordsLine) >= 5:
0547 if lstr_wordsLine[1] == 'store' and\
0548 lstr_wordsLine[2] == 'data' and\
0549 Str_dataset.find(lstr_wordsLine[3]) >= 0 and\
0550 Str_dataset.find(lstr_wordsLine[4]) >= 0 and\
0551 lstr_wordsLine[5] == str_datatier :
0552 int_nInputFiles += 1
0553 file_inputFilesCff.write(str_iLine)
0554 if int_nInputFiles == 0:
0555 print('> submitDQMOfflineCAF.py > no input files found in DBS for run %s in dataset %s' %(Str_run,Str_dataset))
0556 Func_Exit()
0557 file_inputFilesCff.close()
0558 print('> submitDQMOfflineCAF.py > input files for run %s: %i' %(Str_run,int_nInputFiles))
0559 if int_nInputFiles < Int_jobs:
0560 Int_jobs = int_nInputFiles
0561 print(' number of requested jobs reduced accordingly')
0562 Int_jobsNew = Int_jobs
0563 print()
0564
0565
0566 if Bool_magFieldAuto:
0567
0568 str_cmsmonParams = urllib.urlencode({'RUN':Str_run})
0569 file_cmsmonOutput = urllib.urlopen("http://cmsmon.cern.ch/cmsdb/servlet/RunSummary", str_cmsmonParams)
0570 str_timeBegin = ''
0571 str_timeEnd = ''
0572 for str_cmsmonOutput in file_cmsmonOutput.readlines():
0573 if str_cmsmonOutput.find('HREF=Component?RUN=' + Str_run + '&NAME=TRACKER') >= 0:
0574 lstr_timeQuery = str_cmsmonOutput.split('HREF=Component?RUN=' + Str_run + '&NAME=TRACKER&')[1].split('>TRACKER')[0].split('&')
0575 for str_timeQuery in lstr_timeQuery:
0576 str_nameStamp = str_timeQuery.split('=')[0]
0577 lstr_timeDate = str_timeQuery.split('=')[1].split('_')[0].split('.')
0578 lstr_timeTime = str_timeQuery.split('=')[1].split('_')[1].split(':')
0579 dt_stampOld = datetime.datetime(int(lstr_timeDate[0]),int(lstr_timeDate[1]),int(lstr_timeDate[2]),int(lstr_timeTime[0]),int(lstr_timeTime[1]),int(lstr_timeTime[2]))
0580 dt_stampNew = dt_stampOld - TD_shiftUTC
0581 str_timeStamp = str(dt_stampNew).replace('-','.')
0582 if str_nameStamp == 'TIME_BEGIN':
0583 str_timeBegin = str_timeStamp
0584 elif str_nameStamp == 'TIME_END':
0585 str_timeEnd = str_timeStamp
0586
0587 str_cmsmonParams = urllib.urlencode({'TIME_BEGIN':str_timeBegin, 'TIME_END':str_timeEnd})
0588 file_cmsmonOutput = urllib.urlopen("http://cmsmon.cern.ch/cmsdb/servlet/MagnetHistory", str_cmsmonParams)
0589 bool_foundField = False
0590 for str_cmsmonOutput in file_cmsmonOutput.readlines():
0591 if str_cmsmonOutput.find('BFIELD, Tesla') >= 0:
0592 Float_magField = float(str_cmsmonOutput.split('</A>')[0].split('>')[-1])
0593 bool_foundField = True
0594 break
0595 if not bool_foundField:
0596 print('> submitDQMOfflineCAF.py > could not extract magnetic field')
0597 print(' please provide value')
0598 Func_Exit()
0599
0600 float_magField = Func_MagConfig(Float_magField)
0601 Str_magField = str(int(float_magField*10)) + 'T'
0602 print('> submitDQMOfflineCAF.py > (average) magnetic field in run %s: %s T' %(Str_run,Float_magField))
0603 print(' using %s T for configuration' %(float_magField))
0604 print()
0605
0606
0607
0608 int_nLinesRead = 0
0609 file_inputFilesCff = file(str_nameInputFilesFile)
0610 lstr_linesInput = file_inputFilesCff.readlines()
0611 file_inputFilesCff.close()
0612
0613
0614 str_sedCommand = 'sed '
0615 str_sedCommand += '-e \"s#xMAG_FIELDx#' + Str_magField + '#g\" '
0616 str_sedCommand += '-e \"s#xGLOBAL_TAGx#' + Str_globalTag + '#g\" '
0617 str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameRunIncludeDir + '#g\" '
0618 str_sedCommand += '-e \"s#xMERGE_PATHx#' + Str_mergepath + '#g\" '
0619 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripCAFHarvest_template_cfg.py > ' + Str_nameRun + '/SiStripCAFHarvest_cfg.py'
0620 os.system(str_sedCommand)
0621 str_sedCommand = 'sed '
0622 str_sedCommand += '-e \"s#xCMSSW_BASEx#' + Str_pathCmsswBase + '#g\" '
0623 str_sedCommand += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
0624 str_sedCommand += '-e \"s#xMERGE_PATHx#' + Str_mergepath + '#g\" '
0625 str_sedCommand += '-e \"s#xCURRENT_DIRx#' + Str_pathCurrentDir + '#g\" '
0626 str_sedCommand += '-e \"s#xDATA_TIERx#' + Str_datatier + '#g\" '
0627 str_sedCommand += '-e \"s#xCMSSW_VERSIONx#' + Str_nameCmsswRel + '#g\" '
0628 str_sedCommand += Str_pathCmsswBasePackage + '/scripts/SiStripDQMCAFHarvest_template.job > ' + Str_nameRun + '/SiStripCAFHarvest.job'
0629 os.system(str_sedCommand)
0630
0631 Str_pathInputFilesCAFCff = Str_pathRunIncludeDir + '/inputFilesCAF_cff.py'
0632 file_inputFilesCAFCff = file(Str_pathInputFilesCAFCff, 'w')
0633 file_inputFilesCAFCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n processingMode = cms.untracked.string( \'Runs\' ),\n fileNames = cms.untracked.vstring(\n')
0634 for int_iJob in range(1,Int_jobs+1):
0635 str_lineInput = Str_outpath + '/SiStripDQMOfflineGlobalRunCAF-' + Str_nameRun + '_' + str(int_iJob) + '.root'
0636 if Bool_useCastor:
0637 str_lineInput = 'rfio:' + str_lineInput
0638 str_lineInput = ' \'' + str_lineInput + '\''
0639 if int_iJob == Int_jobs:
0640 str_lineInput += '\n'
0641 file_inputFilesCAFCff.write(str_lineInput)
0642 break
0643 str_lineInput += ',\n'
0644 file_inputFilesCAFCff.write(str_lineInput)
0645 file_inputFilesCAFCff.write(' )\n)\n')
0646 file_inputFilesCAFCff.close()
0647
0648 str_sedCommandCommon = 'sed '
0649 if Bool_filter:
0650 str_sedCommandCommon += '-e \"s#xHLT_FILTERx# #g\" '
0651 else:
0652 str_sedCommandCommon += '-e \"s#xHLT_FILTERx#\# #g\" '
0653 if Str_datatier == 'RECO':
0654 str_sedCommandCommon += '-e \"s#xRECO_FROM_RAWx#\# #g\" '
0655 str_sedCommandCommon += '-e \"s#xDQM_FROM_RAWx#\# #g\" '
0656 else:
0657 str_sedCommandCommon += '-e \"s#xRECO_FROM_RAWx# #g\" '
0658 str_sedCommandCommon += '-e \"s#xDQM_FROM_RAWx# #g\" '
0659 str_sedCommandCommon += '-e \"s#xMAG_FIELDx#' + Str_magField + '#g\" '
0660 str_sedCommandCommon += '-e \"s#xGLOBAL_TAGx#' + Str_globalTag + '#g\" '
0661 str_sedCommandCommon += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
0662
0663 if Bool_CRAB:
0664 os.chdir(Str_nameRun)
0665 str_outputDir = '.'
0666
0667 str_sedCommand = str_sedCommandCommon
0668 str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameRunIncludeDir + '#g\" '
0669 str_sedCommand += '-e \"s#xOUTPUT_DIRECTORYx#' + str_outputDir + '#g\" '
0670 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineGlobalRunCAF_template_cfg.py > SiStripDQMOfflineGlobalRunCAF_cfg.py'
0671 os.system(str_sedCommand)
0672
0673 str_pathInputFilesJobCff = Str_pathRunIncludeDir + '/inputFiles_cff.py'
0674 file_inputFilesJobCff = file(str_pathInputFilesJobCff, 'w')
0675 file_inputFilesJobCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n fileNames = cms.untracked.vstring (\n')
0676 nLines = 0
0677 for str_linesInput in lstr_linesInput:
0678 nLines += 1
0679 str_correctedLine1 = str_linesInput.replace(') );',',')
0680 str_correctedLine = str_correctedLine1.replace(' ] );',',')
0681 if nLines == len(lstr_linesInput):
0682 str_actualLine = str_correctedLine.replace(',','\n )\n)\n')
0683 elif nLines%255 == 0:
0684 str_actualLine = str_correctedLine.replace(',','\n )\n)\nsource.fileNames.extend(\n (')
0685 else:
0686 str_actualLine = str_correctedLine
0687 file_inputFilesJobCff.write(str_actualLine)
0688 file_inputFilesJobCff.close()
0689
0690 lstr_outpath = Str_outpath.split('/', 3)
0691 str_outpath = lstr_outpath[0] + '/' + lstr_outpath[1] + '/' + lstr_outpath[2]
0692 str_sedCommand = 'sed '
0693 str_sedCommand += '-e \"s#xSERVER_NAMEx#' + Str_server + '#g\" '
0694 str_sedCommand += '-e \"s#xDATASETPATHx#' + Str_dataset + '#g\" '
0695 str_sedCommand += '-e \"s#xRUNSELECTIONx#' + Str_run + '#g\" '
0696 str_sedCommand += '-e \"s#xNUMBER_OF_JOBSx#' + str(Int_jobs) + '#g\" '
0697 str_sedCommand += '-e \"s#xEMAILx#' + Str_email + '#g\" '
0698 str_sedCommand += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
0699 str_sedCommand += '-e \"s#xSTORAGE_PATHx#' + str_outpath + '#g\" '
0700 str_sedCommand += '-e \"s#xLFNx#' + lstr_outpath[3] + '#g\" '
0701 str_sedCommand += '-e \"s#xCOPY_DATAx#' + str(int(Bool_useCastor)) + '#g\" '
0702 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineCAF_template.crab > crab.cfg'
0703 os.system(str_sedCommand)
0704 os.chdir(Str_pathCurrentDir)
0705 else:
0706
0707 int_nInputFilesJob = int(int_nInputFiles/Int_jobs) + 1
0708 if int_nInputFiles%Int_jobs == 0:
0709 int_nInputFilesJob -= 1
0710 if int_nInputFiles == int_nInputFilesJob*(Int_jobs-1) and Int_jobs > 1:
0711 Int_jobs -= 1
0712
0713 for int_iJob in range(1,Int_jobs+1):
0714 str_nameJob = Str_nameRun + '_' + str(int_iJob).zfill(4)
0715
0716 str_nameJobDir = Str_nameRun + "/" + str_nameJob
0717 str_outputDir = '/tmp/' + os.getenv('USER') + '/' + str_nameJobDir
0718 str_pathJobIncludeDir = Str_pathRunIncludeDir + '/' + str_nameJob
0719 str_nameJobIncludeDir = STR_nameCmsswPackage.replace('/','.') + '.' + str_nameJobDir.replace('/','.')
0720 os.mkdir(str_nameJobDir)
0721 os.chdir(str_nameJobDir)
0722
0723 str_sedCommand = 'sed '
0724 str_sedCommand += '-e \"s#xCMSSW_BASEx#' + Str_pathCmsswBase + '#g\" '
0725 str_sedCommand += '-e \"s#xRUN_NAMEx#' + Str_nameRun + '#g\" '
0726 str_sedCommand += '-e \"s#xJOB_NAMEx#' + str_nameJob + '#g\" '
0727 str_sedCommand += '-e \"s#xCURRENT_DIRx#' + Str_pathCurrentDir + '#g\" '
0728 str_sedCommand += '-e \"s#xCOPYx#' + str_castorCp + '#g\" '
0729 str_sedCommand += '-e \"s#xOUTPUT_DIRx#' + Str_outpath + '#g\" '
0730 str_sedCommand += Str_pathCmsswBasePackage + '/scripts/SiStripDQMOfflineCAF_template.job > SiStripDQMOfflineCAF.job'
0731 os.system(str_sedCommand)
0732
0733 str_sedCommand = str_sedCommandCommon
0734 str_sedCommand += '-e \"s#xINCLUDE_DIRECTORYx#' + str_nameJobIncludeDir + '#g\" '
0735 str_sedCommand += '-e \"s#xOUTPUT_DIRECTORYx#' + str_outputDir + '#g\" '
0736 str_sedCommand += Str_pathCmsswBasePackage + '/test/SiStripDQMOfflineGlobalRunCAF_template_cfg.py > SiStripDQMOfflineGlobalRunCAF_cfg.py'
0737 os.system(str_sedCommand)
0738
0739 os.mkdir(str_pathJobIncludeDir)
0740
0741 str_pathInputFilesJobCff = str_pathJobIncludeDir + '/inputFiles_cff.py'
0742 file_inputFilesJobCff = file(str_pathInputFilesJobCff, 'w')
0743 file_inputFilesJobCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n fileNames = cms.untracked.vstring (\n')
0744 for n_iActualLine in range(int_nLinesRead, min(int_nLinesRead+int_nInputFilesJob, int_nInputFiles)):
0745 str_linesInput = lstr_linesInput[n_iActualLine].replace(') );',',')
0746
0747 str_actualLine = str_linesInput
0748 if (n_iActualLine+1)%int_nInputFilesJob == 0 or int_nLinesRead == int_nInputFiles-1:
0749 str_actualLine = str_linesInput.split(',')[0] + '\n'
0750 file_inputFilesJobCff.write(str_actualLine)
0751 int_nLinesRead += 1
0752 file_inputFilesJobCff.write(' )\n)\n')
0753 file_inputFilesJobCff.close()
0754
0755 os.chdir(Str_pathCurrentDir)
0756
0757 if int_nLinesRead >= int_nInputFiles:
0758 Int_jobsNew = int_iJob
0759 break
0760
0761
0762
0763 os.chdir(Str_pathRunIncludeDir+'/..')
0764 os.system('scramv1 b python')
0765 os.chdir(Str_pathCurrentDir)
0766 print()
0767
0768
0769
0770 if Bool_CRAB:
0771 os.chdir(Str_nameRun)
0772 os.system('crab -create')
0773 print()
0774
0775 bool_found = False
0776 file_logCRAB = file('crab'+Str_nameRun+'/log/crab.log')
0777 for str_iLine in file_logCRAB.readlines():
0778 if str_iLine.startswith('Total of ') and str_iLine.endswith(' jobs created.\n'):
0779 bool_found = True
0780 Int_jobsNew = int(str_iLine.split()[2])
0781 break
0782 file_logCRAB.close()
0783 os.chdir(Str_pathCurrentDir)
0784 if not bool_found:
0785 print('> submitDQMOfflineCAF.py > could not extract number of jobs created by CRAB; check')
0786 print(' %s' %(Str_pathInputFilesCAFCff))
0787 print(' and modify manually, if necessary')
0788 print()
0789
0790
0791
0792 print('> submitDQMOfflineCAF.py > number of created jobs: %i' %(Int_jobsNew))
0793 print()
0794 if Int_jobsNew != Int_jobs:
0795 file_inputFilesCAFCff = file(Str_pathInputFilesCAFCff, 'w')
0796 file_inputFilesCAFCff.write('import FWCore.ParameterSet.Config as cms\n\nsource = cms.Source ("PoolSource",\n processingMode = cms.untracked.string( \'Runs\' ),\n fileNames = cms.untracked.vstring(\n')
0797 for int_iJob in range(1,Int_jobsNew+1):
0798 str_lineInput = Str_outpath + '/SiStripDQMOfflineGlobalRunCAF-' + Str_nameRun + '_' + str(int_iJob) + '.root'
0799 if Bool_useCastor:
0800 str_lineInput = 'rfio:' + str_lineInput
0801 str_lineInput = ' \'' + str_lineInput + '\''
0802 if int_iJob == Int_jobsNew:
0803 str_lineInput += '\n'
0804 file_inputFilesCAFCff.write(str_lineInput)
0805 break
0806 str_lineInput += ',\n'
0807 file_inputFilesCAFCff.write(str_lineInput)
0808 file_inputFilesCAFCff.write(' )\n)\n')
0809 file_inputFilesCAFCff.close()
0810
0811
0812
0813 if LSTR_functionLetters[0] in Dict_arguments:
0814 os.chdir(Str_nameRun)
0815 if Bool_CRAB:
0816 str_crabCommand = 'crab -submit -c crab' + Str_nameRun
0817 print('> submitDQMOfflineCAF.py >')
0818 print(' %s : %s' %(os.getcwd(),str_crabCommand))
0819 os.system(str_crabCommand)
0820 print()
0821 time.sleep(5)
0822 os.system('crab -status -c crab' + Str_nameRun)
0823 else:
0824 for int_iJob in range(Int_jobs):
0825 str_nameJobDir = Str_nameRun + '_' + str(int_iJob).zfill(4)
0826 os.chdir(str_nameJobDir)
0827 os.chmod('SiStripDQMOfflineCAF.job',OCT_rwx_r_r)
0828 str_batchCommand = 'bsub -q cmscaf SiStripDQMOfflineCAF.job'
0829 print('> submitDQMOfflineCAF.py >')
0830 print(' %s : %s' %(os.getcwd(),str_batchCommand))
0831 os.system(str_batchCommand)
0832 print()
0833 os.chdir('../')
0834 time.sleep(5)
0835 os.system('bjobs -q cmscaf')
0836 os.chmod('SiStripCAFHarvest.job',OCT_rwx_r_r)
0837 os.chdir(Str_pathCurrentDir)
0838
0839
0840
0841 if LSTR_functionLetters[0] in Dict_arguments and not Bool_CRAB:
0842 str_mailFrom = os.getenv('USER') + STR_mailServer
0843 str_mailTo = [str_mailFrom,
0844 'volker.adler@cern.ch',
0845 'suchandra.dutta@cern.ch',
0846 'domenico.giordano@cern.ch',
0847 'vitaliano.ciulli@cern.ch']
0848 str_mailSubject = 'Your SiStrip offline DQM shift on ' + str(datetime.date.today()) + ', run ' + Str_run
0849 str_mailText = STR_mailTextOpener + Str_run + STR_mailText
0850 str_mailMessage = """From: %s
0851 To: %s
0852 Subject: %s
0853
0854 %s
0855 """ % (str_mailFrom, ", ".join(str_mailTo), str_mailSubject, str_mailText)
0856 server = smtplib.SMTP(STR_mailSmtp)
0857 server.sendmail(str_mailFrom, str_mailTo, str_mailMessage)
0858 server.quit()
