test/7TeVData/automatic_RunOnCalibTree.py

0001 #!/usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import os,sys
0005 import getopt
0006 import subprocess
0007 import time
0008 import ROOT
0009 import urllib
0010 import string
0011 import optparse
0012 import dataCert
0013
0014 def numberOfEvents(file,mode):
0015    if mode=='':
0016       mode="StdBunch0T"
0017    rootfile = ROOT.TFile.Open(file,'read')
0018    tree = ROOT.TTree()
0019    rootfile.GetObject("gainCalibrationTree%s/tree"%mode,tree)
0020    NEntries = tree.GetEntries()
0021    rootfile.Close()
0022    print(file +' --> '+str(NEntries))
0023    print("gainCalibrationTree%s/tree"%mode)
0024    return NEntries
0025
0026
0027 PCLDATASET = "/StreamExpress/Run2017*-PromptCalibProdSiStripGains__AAG__-Express-v*/ALCAPROMPT"
0028 CALIBTREEPATH = '/store/group/dpg_tracker_strip/comm_tracker/Strip/Calibration/calibrationtree/GR17__AAG__' #used if usePCL==False
0029
0030 runValidation = 273555
0031 runsToVeto = [272935, 273290, 273294, 273295, 273296, 273531,
0032               273537, 273526, 273523, 273514, 273589,  # Low PU
0033               273588, 273586, 273581, 273580, 273579, 273522, 273521, 273520, 273512, 273511, 273510, 273509,  #VDM
0034               275326, 275656, 276764, 275765, 275769, 275783, 275825, 275829, 275838
0035 ]
0036
0037 #read arguments to the command line
0038 #configure
0039 usage = 'usage: %prog [options]'
0040 parser = optparse.OptionParser(usage)
0041 parser.add_option('-f', '--firstRun'   ,    dest='firstRun'           , help='first run to process (-1 --> automatic)'  , default='-1')
0042 parser.add_option('-l', '--lastRun'    ,    dest='lastRun'            , help='last run to process (-1 --> automatic)'   , default='-1')
0043 parser.add_option('-P', '--publish'    ,    dest='publish'            , help='publish the results'                      , default='True')
0044 parser.add_option('-p', '--pcl'        ,    dest='usePCL'             , help='use PCL output instead of calibTree'      , default='False')
0045 parser.add_option('-m', '--mode'       ,    dest='calMode'            , help='select the statistics type'      , default='AagBunch')
0046 (opt, args) = parser.parse_args()
0047
0048 scriptDir = os.getcwd()
0049 globaltag = "92X_dataRun2_Express_v2"
0050 firstRun = int(opt.firstRun)
0051 lastRun  = int(opt.lastRun)
0052 calMode  = str(opt.calMode) if not str(opt.calMode)=='' else "AagBunch" # Set default to AAG
0053 PCLDATASET = PCLDATASET.replace("__AAG__","") if calMode.lower()=="stdbunch" else PCLDATASET.replace("__AAG__","AAG")
0054 CALIBTREEPATH = CALIBTREEPATH.replace("__AAG__","") if calMode.lower()=="stdbunch" else CALIBTREEPATH.replace("__AAG__","_Aag")
0055 MC=""
0056 publish = (opt.publish=='True')
0057 mail = ""
0058 automatic = True;
0059 usePCL = (opt.usePCL=='True')
0060 minNEvents = 3000      # minimum events for a run to be accepted for the gain payload computation
0061 maxNEvents = 3000000   # maximum events allowed in a gain payload computation
0062
0063 if(firstRun!=-1 or lastRun!=-1): automatic = False
0064
0065
0066 DQM_dir = "AlCaReco/SiStripGains" if "AagBunch" not in opt.calMode else "AlCaReco/SiStripGainsAAG"
0067
0068 print()
0069 print()
0070 print("Gain payload computing configuration")
0071 print("  firstRun = " +str(firstRun))
0072 print("  lastRun  = " +str(lastRun))
0073 print("  publish  = " +str(publish))
0074 print("  usePCL   = " +str(usePCL))
0075 print("  calMode  = " + calMode)
0076 print("  DQM_dir  = " + DQM_dir)
0077 print()
0078
0079
0080 #go to parent directory = test directory
0081 os.chdir("..");
0082
0083 #identify last run of the previous calibration
0084 if(firstRun<=0):
0085    out = subprocess.getstatusoutput("ls /afs/cern.ch/cms/tracker/sistrvalidation/WWW/CalibrationValidation/ParticleGain/ | grep Run_ | tail -n 1");
0086    firstRun = int(out[1].split('_')[3])+1
0087    print("firstRun set to " +str(firstRun))
0088    print()
0089
0090 initEnv='cd ' + os.getcwd() + ';'
0091 initEnv+='source /afs/cern.ch/cms/cmsset_default.sh' + ';'
0092 initEnv+='eval `scramv1 runtime -sh`' + ';'
0093
0094
0095 #Get List of Files to process:
0096 NTotalEvents = 0;
0097 run = 0
0098 FileList = ""
0099
0100 dataCertInfo = dataCert.get()
0101 print("Loaded certification info. Last update : %s"%dataCertInfo["Last update"])
0102
0103 lastGoodRun = -1
0104 if(usePCL==True):
0105    print("Get the list of PCL output files from DAS")
0106    print(initEnv+"das_client.py  --limit=9999 --query='dataset=%s'"%PCLDATASET)
0107    dasOutput = subprocess.getstatusoutput(initEnv+"das_client.py  --limit=9999 --query='dataset=%s'"%PCLDATASET)[1]
0108    datasets = [ line for line in dasOutput.splitlines()
0109                 if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ]
0110    print(datasets)
0111    if len( datasets)==0 or 'Error' in " ".join(datasets):
0112        print("Issues in gathering the dataset names, please check the command and the query")
0113        print("***** DAS OUTPUT *****")
0114        print(dasOutput)
0115        print("**********************")
0116        exit (0)
0117
0118    runs = []
0119    for dataset in datasets:
0120        runs += [ (dataset, line) for line in subprocess.getstatusoutput(initEnv+
0121                                          "das_client.py  --limit=9999 --query='run dataset=%s'"%dataset)[1].splitlines()
0122                       if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ]
0123    if len( runs )==0 or 'Error' in " ".join(datasets):
0124        print("Issues in gathering the run numbers, please check the command and the query")
0125        exit (0)
0126    sorted( runs, key=lambda x: x[1])
0127
0128    for dataset, run_number in runs:
0129       run  = int(run_number)
0130       if(run<firstRun or run in runsToVeto):continue
0131       if(lastRun>0 and run>lastRun):continue
0132       if not dataCert.checkRun(run,dataCertInfo):
0133          print("Skipping...")
0134          continue
0135       lastGoodRun = run
0136       sys.stdout.write( 'Gathering infos for RUN %i:  ' % run )
0137
0138       #check the events available for this run
0139       NEventsDasOut = [ line for line in subprocess.getstatusoutput(initEnv+
0140        "das_client.py  --limit=9999 --query='summary dataset=%s run=%i | grep summary.nevents'"%(dataset,run))[1].splitlines()
0141                         if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ][-1]
0142       if(not NEventsDasOut.isdigit() ):
0143          print ("cannot retrieve the number of events from das, SKIPPING")
0144          #print NEventsDasOut
0145          continue
0146
0147       if(FileList==""):firstRun=run;
0148       NEvents = int(NEventsDasOut)
0149
0150       if(NEvents<=minNEvents):
0151          print ("only %i events in this run, SKIPPING" % NEvents)
0152          continue
0153
0154       FileList+="#run=" + str(run) + " -->  NEvents="+str(NEvents/1000).rjust(8)+"K\n"
0155       resultsFiles = [ line for line in subprocess.getstatusoutput(initEnv+
0156                        "das_client.py  --limit=9999 --query='file dataset=%s run=%i'"%(dataset,run))[1].splitlines()
0157                        if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ]
0158       if len(resultsFiles)==0 or 'Error' in " ".join(resultsFiles):
0159          print ("cannot retrieve the list of files from das, SKIPPING")
0160          #print resultsFiles
0161          continue
0162
0163       for file in resultsFiles: FileList+='calibTreeList.extend(["'+file+'"])\n'
0164       NTotalEvents += NEvents;
0165
0166       print("including %s events in gain processing, cumulative total=%i" % (str(NEvents).rjust(7),NTotalEvents))
0167       if(automatic==True and NTotalEvents >= maxNEvents):break;
0168
0169 else:
0170    print("Get the list of calibTree from castor (eos ls " + CALIBTREEPATH + ")")
0171    calibTreeInfo = subprocess.getstatusoutput("eos ls -l "+CALIBTREEPATH)[1].split('\n');
0172    print(calibTreeInfo)
0173    # collect the list of runs and file size
0174    # calibTreeInfo.split()[8] - file name
0175    # calibTreeInfo.split()[4] - file size
0176    info_list = [(i.split()[8],
0177                  int( i.split()[8].replace("calibTree_","").replace(".root","").split('_')[0] ),
0178                  int( i.split()[4] )/1048576 ) for i in calibTreeInfo]
0179    info_list.sort( key=lambda tup: tup[1] )
0180
0181    print("Check the number of events available")
0182    for info in info_list:
0183       if(len(info)<1):continue;
0184       size = info[2]
0185       if(size < 10): continue   #skip file<10MB
0186       run = info[1]
0187       if(run<firstRun or run in runsToVeto):continue
0188       if(lastRun>0 and run>lastRun):continue
0189       if not dataCert.checkRun(run,dataCertInfo):
0190          print("Skipping...")
0191          continue
0192       if run<295310:
0193          print("Skipping...")
0194          continue
0195       lastGoodRun = run
0196       NEvents = numberOfEvents("root://eoscms//eos/cms"+CALIBTREEPATH+'/'+info[0],calMode);
0197       if(NEvents<=3000):continue #only keep runs with at least 3K events
0198       if(FileList==""):firstRun=run;
0199       FileList += 'calibTreeList.extend(["root://eoscms//eos/cms'+CALIBTREEPATH+'/'+info[0]+'"]) #' + str(size).rjust(6)+'MB  NEvents='+str(NEvents/1000).rjust(8)+'K\n'
0200       NTotalEvents += NEvents;
0201       print("Current number of events to process is " + str(NTotalEvents))
0202       if(automatic==True and NTotalEvents >= maxNEvents):break;
0203
0204 if lastGoodRun < 0:
0205    print("No good run to process.")
0206    sys.exit()
0207 if(lastRun<=0):lastRun = lastGoodRun
0208
0209 print("RunRange=[" + str(firstRun) + "," + str(lastRun) + "] --> NEvents=" + str(NTotalEvents/1000)+"K")
0210
0211 if(automatic==True and NTotalEvents<2e6):   #ask at least 2M events to perform the calibration
0212     print('Not Enough events to run the calibration')
0213         os.system('echo "Gain calibration postponed" | mail -s "Gain calibration postponed ('+str(firstRun)+' to '+str(lastRun)+') NEvents=' + str(NTotalEvents/1000)+'K" ' + mail)
0214     exit(0);
0215
0216 name = "Run_"+str(firstRun)+"_to_"+str(lastRun)
0217 if len(calMode)>0:  name = name+"_"+calMode
0218 if(usePCL==True):   name = name+"_PCL"
0219 else:               name = name+"_CalibTree"
0220 print(name)
0221
0222 oldDirectory = "7TeVData"
0223 newDirectory = "Data_"+name;
0224 os.system("mkdir -p " + newDirectory);
0225 os.system("cp " + oldDirectory + "/* " + newDirectory+"/.");
0226 file = open(newDirectory+"/FileList_cfg.py", "w")
0227 file.write("import FWCore.ParameterSet.Config as cms\n")
0228 file.write("calibTreeList = cms.untracked.vstring()\n")
0229 file.write("#TotalNumberOfEvent considered is %i\n" % NTotalEvents)
0230 file.write(FileList)
0231 file.close()
0232 os.system("cat " + newDirectory + "/FileList_cfg.py")
0233 os.system("sed -i 's|XXX_FIRSTRUN_XXX|"+str(firstRun)+"|g' "+newDirectory+"/*_cfg.py")
0234 os.system("sed -i 's|XXX_LASTRUN_XXX|"+str(lastRun)+"|g' "+newDirectory+"/*_cfg.py")
0235 os.system("sed -i 's|XXX_GT_XXX|"+globaltag+"|g' "+newDirectory+"/*_cfg.py")
0236 os.system("sed -i 's|XXX_PCL_XXX|"+str(usePCL)+"|g' "+newDirectory+"/*_cfg.py")
0237 os.system("sed -i 's|XXX_CALMODE_XXX|"+calMode+"|g' "+newDirectory+"/*_cfg.py")
0238 os.system("sed -i 's|XXX_DQMDIR_XXX|"+DQM_dir+"|g' "+newDirectory+"/*_cfg.py")
0239 os.chdir(newDirectory);
0240
0241 job = initEnv
0242 job+= "\ncd %s; \npwd; \nls; \npython submitJob.py -f %s -l %s -p %s -P %s"%(os.getcwd(),firstRun,lastRun,usePCL,publish)
0243 job+= " -m %s -s %s -a %s"%(calMode,scriptDir,automatic)
0244
0245 print("*** JOB : ***")
0246 print(job)
0247 print("cwd = %s"%(os.getcwd()))
0248 with open("job.sh","w") as f:
0249    f.write(job)
0250 os.system("chmod +x job.sh")
0251 submitCMD =  'bsub  -q 2nd -J G2prod -R "type == SLC6_64 && pool > 30000" "job.sh"'
0252 print(submitCMD)
0253 os.system(submitCMD)
0254
0255 #if(os.system("sh sequence.sh \"" + name + "\" \"" + calMode + "\" \"CMS Preliminary  -  Run " + str(firstRun) + " to " + str(lastRun) + "\"")!=0):
0256 #   os.system('echo "Gain calibration failed" | mail -s "Gain calibration failed ('+name+')" ' + mail)
0257 #else:
0258 #   if(publish==True):os.system("sh sequence.sh " + name);
0259 #   os.system('echo "Gain calibration done\nhttps://test-stripcalibvalidation.web.cern.ch/test-stripcalibvalidation/CalibrationValidation/ParticleGain/" | mail -s "Gain calibration done ('+name+')" ' + mail)
0260 #
0261 #if(False and usePCL==True):
0262 #   #Make the same results using the calibTrees for comparisons
0263 #   os.chdir(scriptDir); #go back to initial location
0264 #   os.system('python automatic_RunOnCalibTree.py --firstRun ' + str(firstRun) + ' --lastRun ' + str(lastRun) + ' --publish False --pcl False')
0265 #
0266 #if(automatic==True):
0267 #   #call the script one more time to make sure that we do not have a new run to process
0268 #   os.chdir(scriptDir); #go back to initial location
0269 #   os.system('python automatic_RunOnCalibTree.py')
0270 #