Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-26 02:34:38

0001 #! /usr/bin/env python3
0002 
0003 import os
0004 import sys
0005 import fileinput
0006 import string
0007 
0008 ##########################################################################
0009 ##########################################################################
0010 ######### User variables
0011 
0012 #Reference release
0013 NewRelease='CMSSW_3_8_0_pre2'
0014 
0015 # startup and ideal sample list
0016 #startupsamples= ['RelValSingleMuPt10', 'RelValSingleMuPt100', 'RelValSingleMuPt1000', 'RelValTTbar','RelValZMM']
0017 startupsamples= []
0018 
0019 #idealsamples= ['RelValSingleMuPt10', 'RelValSingleMuPt100', 'RelValSingleMuPt1000', 'RelValTTbar','RelValZMM']
0020 idealsamples= ['RelValSingleMuPt100']
0021 
0022 # track algorithm name and quality. Can be a list.
0023 Algos= ['']
0024 Qualities=['']
0025 #Qualities=['', 'highPurity']
0026 
0027 #Leave unchanged unless the track collection name changed
0028 Tracksname=''
0029 
0030 # Sequence. Possible values:
0031 #   -harvesting
0032 #   -reco_and_val
0033 #   -only_val
0034 
0035 Sequence='reco_and_val'
0036 
0037 SearchContent="*GEN-SIM-RECO*"
0038 DriverSteps="HARVESTING:validationHarvesting"
0039 if(Sequence=="harvesting"):
0040     SearchContent="*GEN-SIM-RECO*"
0041     DriverSteps="HARVESTING:validationHarvesting+dqmHarvesting"
0042 if(Sequence=="reco_and_val"):
0043     SearchContent="*GEN-SIM*HLTDEBUG*"
0044     DriverSteps="RAW2DIGI,RECO,VALIDATION"
0045 if(Sequence=="only_val"):
0046     SearchContent="*GEN-SIM-RECO*"
0047     DriverSteps="POSTRECO,VALIDATION"
0048 
0049 print(Sequence+' '+SearchContent)
0050 
0051 Submit=False
0052 DBS=True
0053 OneAtATime=False
0054 
0055 # Ideal and Statup tags
0056 IdealTag='MC_38Y_V1'
0057 StartupTag='STARTUP_31X_v1'
0058 
0059 # Default label is GlobalTag_noPU__Quality_Algo. Change this variable if you want to append an additional string.
0060 NewSelectionLabel='_test'
0061 
0062 WorkDirBase = '/tmp/'
0063 #WorkDirBase = '/tmp/aperrott'
0064 #WorkDirBase = '/afs/cern.ch/user/a/aeverett/scratch0'
0065 
0066 #Default Nevents
0067 defaultNevents ='-1'
0068 
0069 #Put here the number of event to be processed for specific samples (numbers must be strings) if not specified is -1:
0070 Events={} #{ 'RelValZMM':'5000', 'RelValTTbar':'5000'}
0071 
0072 
0073 
0074 #########################################################################
0075 #########################################################################
0076 ############ Functions
0077 
0078 def replace(map, filein, fileout):
0079     replace_items = map.items()
0080     while True:
0081         line = filein.readline()
0082         if not line: break
0083         for old, new in replace_items:
0084             line = string.replace(line, old, new)
0085         fileout.write(line)
0086     fileout.close()
0087     filein.close()
0088     
0089 ############################################
0090 
0091     
0092 def do_validation(samples, GlobalTagUse, trackquality, trackalgorithm):
0093     global Sequence, NewSelection, defaultNevents, Events
0094     global Tracksname,SearchContent,DriverSteps
0095     splitResult = GlobalTagUse.split('_')
0096     GlobalTag = splitResult[0]
0097     print('Search Tag: ' + GlobalTag)
0098     print('Tag to use: ' + GlobalTagUse)
0099 
0100     #build the New Selection name
0101     NewSelection=GlobalTag +'_noPU'
0102     if( trackquality !=''):
0103         NewSelection+='_'+trackquality
0104     if(trackalgorithm!=''):
0105         NewSelection+='_'+trackalgorithm
0106     if(trackquality =='') and (trackalgorithm==''):
0107         if(Tracksname==''):
0108             NewSelection+='_ootb'
0109             Tracks='generalTracks'
0110         else:
0111            NewSelection+= Tracks
0112     elif(Tracksname==''):
0113         Tracks='cutsRecoTracks'
0114     else:
0115         Tracks=Tracksname
0116     NewSelection+=NewSelectionLabel
0117 
0118     #loop on all the requested samples
0119     for sample in samples :
0120         print('Get information from DBS for sample', sample)
0121 
0122         WorkDir = WorkDirBase+'/'+NewRelease+'/'+NewSelection+'/'+sample
0123 
0124         if(os.path.exists(WorkDir)==False):
0125             os.makedirs(WorkDir)
0126 
0127         
0128         #chech if the sample is already done
0129 
0130         if(True):
0131             ## start new
0132             cmd='dbsql "find  dataset where dataset like *'
0133             cmd+=sample+'/'+NewRelease+'_'+GlobalTag+SearchContent+' "'
0134             cmd+='|grep '+sample+'|grep -v test|sort|tail -1|cut -f2 '
0135             print(cmd)
0136             dataset= os.popen(cmd).readline().strip()
0137             print('DataSet:  ', dataset, '\n')
0138             #Check if a dataset is found
0139             if(dataset!="" or DBS==False):
0140         
0141                 #Find and format the list of files
0142                 cmd2='dbsql "find file where dataset like '+ dataset +'"|grep ' + sample
0143                 thisFile=0
0144                 filenames='import FWCore.ParameterSet.Config as cms\n'
0145                 filenames+='readFiles = cms.untracked.vstring()\n'
0146                 filenames+='secFiles = cms.untracked.vstring()\n'
0147                 filenames+='source = cms.Source ("PoolSource",fileNames = readFiles, secondaryFileNames = secFiles)\n'
0148                 filenames+='readFiles.extend( [\n'
0149                 first=True
0150                 print(cmd2)
0151                 for line in os.popen(cmd2).readlines():
0152                     filename=line.strip()
0153                     thisFile=thisFile+1
0154                     if first==True:
0155                         filenames+="'"
0156                         filenames+=filename
0157                         filenames+="'"
0158                         first=False
0159                     else :
0160                         filenames+=",\n'"
0161                         filenames+=filename
0162                         filenames+="'"
0163                 filenames+='\n]);\n'
0164                 
0165                 # if not harvesting find secondary file names
0166                 if(Sequence!="harvesting" and Sequence!="reco_and_val"):
0167                         cmd3='dbsql  "find dataset.parent where dataset like '+ dataset +'"|grep ' + sample
0168                         parentdataset=os.popen(cmd3).readline()
0169                         print('Parent DataSet:  ', parentdataset, '\n')
0170                     
0171                     #Check if a dataset is found
0172                         if parentdataset!="":
0173                             cmd4='dbsql  "find file where dataset like '+ parentdataset +'"|grep ' + sample 
0174                             filenames+='secFiles.extend( [\n'
0175                             first=True
0176         
0177                             for line in os.popen(cmd4).readlines():
0178                                 secfilename=line.strip()
0179                                 if first==True:
0180                                     filenames+="'"
0181                                     filenames+=secfilename
0182                                     filenames+="'"
0183                                     first=False
0184                                 else :
0185                                     filenames+=",\n'"
0186                                     filenames+=secfilename
0187                                     filenames+="'"
0188                             filenames+='\n]);\n'
0189                         else :
0190                             print("No parent dataset found skipping sample: ", sample)
0191                             filenames+='secFiles.extend( (               ) )\n'
0192 
0193                             #continue
0194                 else :
0195                         filenames+='secFiles.extend( (               ) )\n'
0196 
0197             ## end new
0198 
0199 
0200                 cfgFileName=('%s_%s_%d') % (sample,Sequence,thisFile)
0201                 print('cfgFileName ' + cfgFileName)
0202                 
0203                 #sampleFileName=('sample_%s_%d') % (sample,thisFile)
0204                 sampleFileName='sample_'+Sequence
0205                 sampleFile = open(sampleFileName+'.py','w' )
0206                 sampleFile.write(filenames)
0207                 sampleFile.close()
0208                 #print filenames
0209                 
0210                 if ((sample in Events)!=True):
0211                     Nevents=defaultNevents
0212                 else:
0213                     Nevents=Events[sample]
0214                 
0215                 symbol_map = { 'NEVENT':Nevents, 'GLOBALTAG':GlobalTagUse, 'SEQUENCE':Sequence, 'SAMPLE': sample, 'ALGORITHM':trackalgorithm, 'QUALITY':trackquality, 'TRACKS':Tracks}
0216 
0217                 cmdrun='cmsRun ' + WorkDir +'/'+cfgFileName+ '.py >&  ' + cfgFileName + '.log < /dev/zero '
0218 
0219                 print(cmdrun)
0220                 
0221                 lancialines='#!/bin/bash \n'
0222                 lancialines+='cd '+ProjectBase+'/src \n'
0223                 lancialines+='eval `scramv1 run -sh` \n\n'
0224                 lancialines+='export PYTHONPATH=.:$PYTHONPATH \n'
0225                 lancialines+='cd '+WorkDir+'\n'
0226                 lancialines+='cmsRun '+cfgFileName+'.py  >&  ' + cfgFileName + '.log < /dev/zero \n'
0227 #                lancialines+='mv  DQM_V0001_R000000001__' + GlobalTagUse+ '__' + sample + '__Validation.root' + ' ' + 'val.' +sample+'.root \n'
0228                 lancialines+='mv  DQM_V0001_R000000001__Global__CMSSW_X_Y_Z__RECO.root' + ' ' + 'val.' +sample+'.root \n'
0229                 
0230                 lanciaName=('lancia_%s_%s_%s_%d') % (GlobalTag,sample,Sequence,thisFile)
0231                 lanciaFile = open(lanciaName,'w')
0232                 lanciaFile.write(lancialines)
0233                 lanciaFile.close()
0234 
0235                 previousFileOut=''
0236                 command=''
0237                 nextFileOut=''
0238                 specialCommand=''
0239                 processName=''
0240                 if(Sequence=="reco_and_val"):
0241                     nextFileOut+='step2_RECO_VALIDATION.root'
0242                     command+='step2'
0243                     previousFileOut+='step1.root'
0244                     processName+='RecVal'
0245                 if(Sequence=="only_val"):
0246                     nextFileOut+='step2_VALIDATION.root'
0247                     command+='step2'
0248                     previousFileOut+='step2.root'
0249                     processName+='VAL'
0250                 if(Sequence=="harvesting"):
0251                     if(previousFileOut==''):
0252                         previousFileOut+='step2.root'
0253                     command+='harvest'
0254                     nextFileOut+='harvest_out.root'
0255                     specialCommand+=' --harvesting AtJobEnd'
0256                     processName+='HARV'
0257 
0258                 driverCmdNew='cmsDriver.py '+ command
0259                 driverCmdNew+=' -s '+DriverSteps
0260                 driverCmdNew+=' '+processName+' '
0261                 driverCmdNew+=' --mc '
0262                 driverCmdNew+=' -n'+Nevents
0263                 driverCmdNew+=' --conditions FrontierConditions_GlobalTag,'+GlobalTagUse+'::All'
0264                 driverCmdNew+=' --filein file:'+previousFileOut
0265                 driverCmdNew+=' --fileout '+nextFileOut
0266                 driverCmdNew+=' --python_filename='+cfgFileName+'.py '
0267                 driverCmdNew+=' --no_exec '
0268                 driverCmdNew+=' --customise=Validation/RecoMuon/customise.py '
0269                 driverCmdNew+=' '+specialCommand+' ' 
0270 
0271                 print(driverCmdNew)
0272                 iii=os.system(driverCmdNew)
0273                     
0274                 print(("copying py file for sample: %s %s") % (sample,lanciaName))
0275                 iii = os.system('mv '+lanciaName+' '+WorkDir+'/.')
0276                 print(iii)
0277                 iii = os.system('mv '+sampleFileName+'.py '+WorkDir)
0278                 iii = os.system('mv '+cfgFileName+'.py '+WorkDir)
0279                 #iii = os.system('mv harvest_'+cfgFileName+'.py '+WorkDir)
0280                 #if(OneAtATime==False):
0281                 #    break
0282                 
0283                 retcode=0
0284 
0285                 if(Submit):
0286                     retcode=os.system(cmdrun)
0287                     if os.path.isfile(sample+'.log'):
0288                         os.system('mv '+sample+'.log ' + WorkDir)
0289                 else:
0290                     continue
0291 
0292                 if (retcode!=0):
0293                     print('Job for sample '+ sample + ' failed. \n')
0294                 else:
0295                     if Sequence=="harvesting":
0296                         os.system('mv  DQM_V0001_R000000001__' + GlobalTag+ '__' + sample + '__Validation.root' + ' ' + WorkDir+'/val.' +sample+'.root')
0297                     if os.path.isfile('fullOutput.'+sample+'.root'):
0298                         os.system('mv fullOutput.'+sample+'.root ' + WorkDir)
0299                     if os.path.isfile('output.'+sample+'.root'):
0300                         os.system('mv output.'+sample+'.root ' + WorkDir)
0301                         
0302             else:
0303                 print('No dataset found skipping sample: '+ sample, '\n')
0304         else:
0305             print('Validation for sample ' + sample + ' already done. Skipping this sample. \n')
0306 
0307 
0308 
0309 
0310 ##########################################################################
0311 #########################################################################
0312 ######## This is the main program
0313             
0314 try:
0315      #Get some environment variables to use
0316      if Submit:
0317          print('Will Submit job')
0318          #NewRelease     = os.environ["CMSSW_VERSION"]
0319      ProjectBase    = os.environ["CMSSW_BASE"]
0320      #else:
0321          #NewRelease='CMSSW_2_2_3'
0322 
0323 
0324       
0325 except KeyError:
0326      print('Error: The environment variable CMSSW_VERSION is not available.', file=sys.stderr)
0327      print('       Please run eval `scramv1 runtime -csh` to set your environment variables', file=sys.stderr)
0328      sys.exit()
0329 
0330 
0331 #print 'Running validation on the following samples: ', samples
0332 
0333 if os.path.isfile( 'dbsCommandLine.py')!=True:
0334     print("Failed to find the file ($DBSCMD_HOME/dbsCommandLine.py)", file=sys.stderr)
0335 
0336 NewSelection=''
0337 
0338 for algo in Algos:
0339     print('algo ' + algo)
0340     for quality in Qualities:
0341         print('quality ' + quality)
0342         do_validation(idealsamples, IdealTag, quality , algo)
0343         do_validation(startupsamples, StartupTag, quality , algo)
0344