File indexing completed on 2024-11-25 02:29:10
0001
0002
0003 import os,sys
0004 import getopt
0005 import subprocess
0006 import time
0007 import ROOT
0008 import urllib
0009 import string
0010 import optparse
0011 import dataCert
0012
0013 def numberOfEvents(file,mode):
0014 if mode=='':
0015 mode="StdBunch0T"
0016 rootfile = ROOT.TFile.Open(file,'read')
0017 tree = ROOT.TTree()
0018 rootfile.GetObject("gainCalibrationTree%s/tree"%mode,tree)
0019 NEntries = tree.GetEntries()
0020 rootfile.Close()
0021 print(file +' --> '+str(NEntries))
0022 print("gainCalibrationTree%s/tree"%mode)
0023 return NEntries
0024
0025
0026 PCLDATASET = "/StreamExpress/Run2017*-PromptCalibProdSiStripGains__AAG__-Express-v*/ALCAPROMPT"
0027 CALIBTREEPATH = '/store/group/dpg_tracker_strip/comm_tracker/Strip/Calibration/calibrationtree/GR17__AAG__'
0028
0029 runValidation = 273555
0030 runsToVeto = [272935, 273290, 273294, 273295, 273296, 273531,
0031 273537, 273526, 273523, 273514, 273589,
0032 273588, 273586, 273581, 273580, 273579, 273522, 273521, 273520, 273512, 273511, 273510, 273509,
0033 275326, 275656, 276764, 275765, 275769, 275783, 275825, 275829, 275838
0034 ]
0035
0036
0037
0038 usage = 'usage: %prog [options]'
0039 parser = optparse.OptionParser(usage)
0040 parser.add_option('-f', '--firstRun' , dest='firstRun' , help='first run to process (-1 --> automatic)' , default='-1')
0041 parser.add_option('-l', '--lastRun' , dest='lastRun' , help='last run to process (-1 --> automatic)' , default='-1')
0042 parser.add_option('-P', '--publish' , dest='publish' , help='publish the results' , default='True')
0043 parser.add_option('-p', '--pcl' , dest='usePCL' , help='use PCL output instead of calibTree' , default='False')
0044 parser.add_option('-m', '--mode' , dest='calMode' , help='select the statistics type' , default='AagBunch')
0045 (opt, args) = parser.parse_args()
0046
0047 scriptDir = os.getcwd()
0048 globaltag = "92X_dataRun2_Express_v2"
0049 firstRun = int(opt.firstRun)
0050 lastRun = int(opt.lastRun)
0051 calMode = str(opt.calMode) if not str(opt.calMode)=='' else "AagBunch"
0052 PCLDATASET = PCLDATASET.replace("__AAG__","") if calMode.lower()=="stdbunch" else PCLDATASET.replace("__AAG__","AAG")
0053 CALIBTREEPATH = CALIBTREEPATH.replace("__AAG__","") if calMode.lower()=="stdbunch" else CALIBTREEPATH.replace("__AAG__","_Aag")
0054 MC=""
0055 publish = (opt.publish=='True')
0056 mail = ""
0057 automatic = True;
0058 usePCL = (opt.usePCL=='True')
0059 minNEvents = 3000
0060 maxNEvents = 3000000
0061
0062 if(firstRun!=-1 or lastRun!=-1): automatic = False
0063
0064
0065 DQM_dir = "AlCaReco/SiStripGains" if "AagBunch" not in opt.calMode else "AlCaReco/SiStripGainsAAG"
0066
0067 print()
0068 print()
0069 print("Gain payload computing configuration")
0070 print(" firstRun = " +str(firstRun))
0071 print(" lastRun = " +str(lastRun))
0072 print(" publish = " +str(publish))
0073 print(" usePCL = " +str(usePCL))
0074 print(" calMode = " + calMode)
0075 print(" DQM_dir = " + DQM_dir)
0076 print()
0077
0078
0079
0080 os.chdir("..");
0081
0082
0083 if(firstRun<=0):
0084 out = subprocess.getstatusoutput("ls /afs/cern.ch/cms/tracker/sistrvalidation/WWW/CalibrationValidation/ParticleGain/ | grep Run_ | tail -n 1");
0085 firstRun = int(out[1].split('_')[3])+1
0086 print("firstRun set to " +str(firstRun))
0087 print()
0088
0089 initEnv='cd ' + os.getcwd() + ';'
0090 initEnv+='source /afs/cern.ch/cms/cmsset_default.sh' + ';'
0091 initEnv+='eval `scramv1 runtime -sh`' + ';'
0092
0093
0094
0095 NTotalEvents = 0;
0096 run = 0
0097 FileList = ""
0098
0099 dataCertInfo = dataCert.get()
0100 print("Loaded certification info. Last update : %s"%dataCertInfo["Last update"])
0101
0102 lastGoodRun = -1
0103 if(usePCL==True):
0104 print("Get the list of PCL output files from DAS")
0105 print(initEnv+"das_client.py --limit=9999 --query='dataset=%s'"%PCLDATASET)
0106 dasOutput = subprocess.getstatusoutput(initEnv+"das_client.py --limit=9999 --query='dataset=%s'"%PCLDATASET)[1]
0107 datasets = [ line for line in dasOutput.splitlines()
0108 if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ]
0109 print(datasets)
0110 if len( datasets)==0 or 'Error' in " ".join(datasets):
0111 print("Issues in gathering the dataset names, please check the command and the query")
0112 print("***** DAS OUTPUT *****")
0113 print(dasOutput)
0114 print("**********************")
0115 exit (0)
0116
0117 runs = []
0118 for dataset in datasets:
0119 runs += [ (dataset, line) for line in subprocess.getstatusoutput(initEnv+
0120 "das_client.py --limit=9999 --query='run dataset=%s'"%dataset)[1].splitlines()
0121 if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ]
0122 if len( runs )==0 or 'Error' in " ".join(datasets):
0123 print("Issues in gathering the run numbers, please check the command and the query")
0124 exit (0)
0125 sorted( runs, key=lambda x: x[1])
0126
0127 for dataset, run_number in runs:
0128 run = int(run_number)
0129 if(run<firstRun or run in runsToVeto):continue
0130 if(lastRun>0 and run>lastRun):continue
0131 if not dataCert.checkRun(run,dataCertInfo):
0132 print("Skipping...")
0133 continue
0134 lastGoodRun = run
0135 sys.stdout.write( 'Gathering infos for RUN %i: ' % run )
0136
0137
0138 NEventsDasOut = [ line for line in subprocess.getstatusoutput(initEnv+
0139 "das_client.py --limit=9999 --query='summary dataset=%s run=%i | grep summary.nevents'"%(dataset,run))[1].splitlines()
0140 if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ][-1]
0141 if(not NEventsDasOut.isdigit() ):
0142 print ("cannot retrieve the number of events from das, SKIPPING")
0143
0144 continue
0145
0146 if(FileList==""):firstRun=run;
0147 NEvents = int(NEventsDasOut)
0148
0149 if(NEvents<=minNEvents):
0150 print ("only %i events in this run, SKIPPING" % NEvents)
0151 continue
0152
0153 FileList+="#run=" + str(run) + " --> NEvents="+str(NEvents/1000).rjust(8)+"K\n"
0154 resultsFiles = [ line for line in subprocess.getstatusoutput(initEnv+
0155 "das_client.py --limit=9999 --query='file dataset=%s run=%i'"%(dataset,run))[1].splitlines()
0156 if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ]
0157 if len(resultsFiles)==0 or 'Error' in " ".join(resultsFiles):
0158 print ("cannot retrieve the list of files from das, SKIPPING")
0159
0160 continue
0161
0162 for file in resultsFiles: FileList+='calibTreeList.extend(["'+file+'"])\n'
0163 NTotalEvents += NEvents;
0164
0165 print("including %s events in gain processing, cumulative total=%i" % (str(NEvents).rjust(7),NTotalEvents))
0166 if(automatic==True and NTotalEvents >= maxNEvents):break;
0167
0168 else:
0169 print("Get the list of calibTree from castor (eos ls " + CALIBTREEPATH + ")")
0170 calibTreeInfo = subprocess.getstatusoutput("eos ls -l "+CALIBTREEPATH)[1].split('\n');
0171 print(calibTreeInfo)
0172
0173
0174
0175 info_list = [(i.split()[8],
0176 int( i.split()[8].replace("calibTree_","").replace(".root","").split('_')[0] ),
0177 int( i.split()[4] )/1048576 ) for i in calibTreeInfo]
0178 info_list.sort( key=lambda tup: tup[1] )
0179
0180 print("Check the number of events available")
0181 for info in info_list:
0182 if(len(info)<1):continue;
0183 size = info[2]
0184 if(size < 10): continue
0185 run = info[1]
0186 if(run<firstRun or run in runsToVeto):continue
0187 if(lastRun>0 and run>lastRun):continue
0188 if not dataCert.checkRun(run,dataCertInfo):
0189 print("Skipping...")
0190 continue
0191 if run<295310:
0192 print("Skipping...")
0193 continue
0194 lastGoodRun = run
0195 NEvents = numberOfEvents("root://eoscms//eos/cms"+CALIBTREEPATH+'/'+info[0],calMode);
0196 if(NEvents<=3000):continue
0197 if(FileList==""):firstRun=run;
0198 FileList += 'calibTreeList.extend(["root://eoscms//eos/cms'+CALIBTREEPATH+'/'+info[0]+'"]) #' + str(size).rjust(6)+'MB NEvents='+str(NEvents/1000).rjust(8)+'K\n'
0199 NTotalEvents += NEvents;
0200 print("Current number of events to process is " + str(NTotalEvents))
0201 if(automatic==True and NTotalEvents >= maxNEvents):break;
0202
0203 if lastGoodRun < 0:
0204 print("No good run to process.")
0205 sys.exit()
0206 if(lastRun<=0):lastRun = lastGoodRun
0207
0208 print("RunRange=[" + str(firstRun) + "," + str(lastRun) + "] --> NEvents=" + str(NTotalEvents/1000)+"K")
0209
0210 if(automatic==True and NTotalEvents<2e6):
0211 print('Not Enough events to run the calibration')
0212 os.system('echo "Gain calibration postponed" | mail -s "Gain calibration postponed ('+str(firstRun)+' to '+str(lastRun)+') NEvents=' + str(NTotalEvents/1000)+'K" ' + mail)
0213 exit(0)
0214
0215 name = "Run_"+str(firstRun)+"_to_"+str(lastRun)
0216 if len(calMode)>0: name = name+"_"+calMode
0217 if(usePCL==True): name = name+"_PCL"
0218 else: name = name+"_CalibTree"
0219 print(name)
0220
0221 oldDirectory = "7TeVData"
0222 newDirectory = "Data_"+name;
0223 os.system("mkdir -p " + newDirectory);
0224 os.system("cp " + oldDirectory + "/* " + newDirectory+"/.");
0225 file = open(newDirectory+"/FileList_cfg.py", "w")
0226 file.write("import FWCore.ParameterSet.Config as cms\n")
0227 file.write("calibTreeList = cms.untracked.vstring()\n")
0228 file.write("#TotalNumberOfEvent considered is %i\n" % NTotalEvents)
0229 file.write(FileList)
0230 file.close()
0231 os.system("cat " + newDirectory + "/FileList_cfg.py")
0232 os.system("sed -i 's|XXX_FIRSTRUN_XXX|"+str(firstRun)+"|g' "+newDirectory+"/*_cfg.py")
0233 os.system("sed -i 's|XXX_LASTRUN_XXX|"+str(lastRun)+"|g' "+newDirectory+"/*_cfg.py")
0234 os.system("sed -i 's|XXX_GT_XXX|"+globaltag+"|g' "+newDirectory+"/*_cfg.py")
0235 os.system("sed -i 's|XXX_PCL_XXX|"+str(usePCL)+"|g' "+newDirectory+"/*_cfg.py")
0236 os.system("sed -i 's|XXX_CALMODE_XXX|"+calMode+"|g' "+newDirectory+"/*_cfg.py")
0237 os.system("sed -i 's|XXX_DQMDIR_XXX|"+DQM_dir+"|g' "+newDirectory+"/*_cfg.py")
0238 os.chdir(newDirectory);
0239
0240 job = initEnv
0241 job+= "\ncd %s; \npwd; \nls; \npython submitJob.py -f %s -l %s -p %s -P %s"%(os.getcwd(),firstRun,lastRun,usePCL,publish)
0242 job+= " -m %s -s %s -a %s"%(calMode,scriptDir,automatic)
0243
0244 print("*** JOB : ***")
0245 print(job)
0246 print("cwd = %s"%(os.getcwd()))
0247 with open("job.sh","w") as f:
0248 f.write(job)
0249 os.system("chmod +x job.sh")
0250 submitCMD = 'bsub -q 2nd -J G2prod -R "type == SLC6_64 && pool > 30000" "job.sh"'
0251 print(submitCMD)
0252 os.system(submitCMD)
0253
0254
0255
0256
0257
0258
0259
0260
0261
0262
0263
0264
0265
0266
0267
0268
0269