Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-25 02:29:10

0001 #!/usr/bin/env python3
0002 
0003 import os,sys
0004 import getopt
0005 import subprocess
0006 import time
0007 import ROOT
0008 import urllib
0009 import string
0010 import optparse
0011 import dataCert
0012 
0013 def numberOfEvents(file,mode):
0014    if mode=='':
0015       mode="StdBunch0T"
0016    rootfile = ROOT.TFile.Open(file,'read')
0017    tree = ROOT.TTree()
0018    rootfile.GetObject("gainCalibrationTree%s/tree"%mode,tree)
0019    NEntries = tree.GetEntries()
0020    rootfile.Close()
0021    print(file +' --> '+str(NEntries))
0022    print("gainCalibrationTree%s/tree"%mode)
0023    return NEntries  
0024 
0025 
0026 PCLDATASET = "/StreamExpress/Run2017*-PromptCalibProdSiStripGains__AAG__-Express-v*/ALCAPROMPT"
0027 CALIBTREEPATH = '/store/group/dpg_tracker_strip/comm_tracker/Strip/Calibration/calibrationtree/GR17__AAG__' #used if usePCL==False
0028 
0029 runValidation = 273555
0030 runsToVeto = [272935, 273290, 273294, 273295, 273296, 273531,
0031               273537, 273526, 273523, 273514, 273589,  # Low PU
0032               273588, 273586, 273581, 273580, 273579, 273522, 273521, 273520, 273512, 273511, 273510, 273509,  #VDM
0033               275326, 275656, 276764, 275765, 275769, 275783, 275825, 275829, 275838
0034 ]
0035 
0036 #read arguments to the command line
0037 #configure
0038 usage = 'usage: %prog [options]'
0039 parser = optparse.OptionParser(usage)
0040 parser.add_option('-f', '--firstRun'   ,    dest='firstRun'           , help='first run to process (-1 --> automatic)'  , default='-1')
0041 parser.add_option('-l', '--lastRun'    ,    dest='lastRun'            , help='last run to process (-1 --> automatic)'   , default='-1')
0042 parser.add_option('-P', '--publish'    ,    dest='publish'            , help='publish the results'                      , default='True')
0043 parser.add_option('-p', '--pcl'        ,    dest='usePCL'             , help='use PCL output instead of calibTree'      , default='False')
0044 parser.add_option('-m', '--mode'       ,    dest='calMode'            , help='select the statistics type'      , default='AagBunch')
0045 (opt, args) = parser.parse_args()
0046 
0047 scriptDir = os.getcwd()
0048 globaltag = "92X_dataRun2_Express_v2"
0049 firstRun = int(opt.firstRun)
0050 lastRun  = int(opt.lastRun)
0051 calMode  = str(opt.calMode) if not str(opt.calMode)=='' else "AagBunch" # Set default to AAG
0052 PCLDATASET = PCLDATASET.replace("__AAG__","") if calMode.lower()=="stdbunch" else PCLDATASET.replace("__AAG__","AAG")
0053 CALIBTREEPATH = CALIBTREEPATH.replace("__AAG__","") if calMode.lower()=="stdbunch" else CALIBTREEPATH.replace("__AAG__","_Aag")
0054 MC=""
0055 publish = (opt.publish=='True')
0056 mail = ""
0057 automatic = True;
0058 usePCL = (opt.usePCL=='True')
0059 minNEvents = 3000      # minimum events for a run to be accepted for the gain payload computation
0060 maxNEvents = 3000000   # maximum events allowed in a gain payload computation
0061 
0062 if(firstRun!=-1 or lastRun!=-1): automatic = False
0063 
0064 
0065 DQM_dir = "AlCaReco/SiStripGains" if "AagBunch" not in opt.calMode else "AlCaReco/SiStripGainsAAG"
0066 
0067 print()
0068 print()
0069 print("Gain payload computing configuration")
0070 print("  firstRun = " +str(firstRun))
0071 print("  lastRun  = " +str(lastRun))
0072 print("  publish  = " +str(publish))
0073 print("  usePCL   = " +str(usePCL))
0074 print("  calMode  = " + calMode)
0075 print("  DQM_dir  = " + DQM_dir)
0076 print()
0077 
0078 
0079 #go to parent directory = test directory
0080 os.chdir("..");
0081 
0082 #identify last run of the previous calibration
0083 if(firstRun<=0):
0084    out = subprocess.getstatusoutput("ls /afs/cern.ch/cms/tracker/sistrvalidation/WWW/CalibrationValidation/ParticleGain/ | grep Run_ | tail -n 1");
0085    firstRun = int(out[1].split('_')[3])+1
0086    print("firstRun set to " +str(firstRun))
0087    print()
0088 
0089 initEnv='cd ' + os.getcwd() + ';'
0090 initEnv+='source /afs/cern.ch/cms/cmsset_default.sh' + ';'
0091 initEnv+='eval `scramv1 runtime -sh`' + ';'
0092 
0093 
0094 #Get List of Files to process:
0095 NTotalEvents = 0;
0096 run = 0
0097 FileList = ""
0098 
0099 dataCertInfo = dataCert.get()
0100 print("Loaded certification info. Last update : %s"%dataCertInfo["Last update"]) 
0101 
0102 lastGoodRun = -1
0103 if(usePCL==True):
0104    print("Get the list of PCL output files from DAS")
0105    print(initEnv+"das_client.py  --limit=9999 --query='dataset=%s'"%PCLDATASET)
0106    dasOutput = subprocess.getstatusoutput(initEnv+"das_client.py  --limit=9999 --query='dataset=%s'"%PCLDATASET)[1]
0107    datasets = [ line for line in dasOutput.splitlines()
0108                 if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ]
0109    print(datasets)
0110    if len( datasets)==0 or 'Error' in " ".join(datasets):
0111        print("Issues in gathering the dataset names, please check the command and the query")
0112        print("***** DAS OUTPUT *****")
0113        print(dasOutput)
0114        print("**********************")
0115        exit (0) 
0116 
0117    runs = []
0118    for dataset in datasets:
0119        runs += [ (dataset, line) for line in subprocess.getstatusoutput(initEnv+
0120                                          "das_client.py  --limit=9999 --query='run dataset=%s'"%dataset)[1].splitlines()
0121                       if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ]
0122    if len( runs )==0 or 'Error' in " ".join(datasets):
0123        print("Issues in gathering the run numbers, please check the command and the query")
0124        exit (0)
0125    sorted( runs, key=lambda x: x[1])
0126 
0127    for dataset, run_number in runs:
0128       run  = int(run_number)
0129       if(run<firstRun or run in runsToVeto):continue
0130       if(lastRun>0 and run>lastRun):continue      
0131       if not dataCert.checkRun(run,dataCertInfo):
0132          print("Skipping...")
0133          continue
0134       lastGoodRun = run
0135       sys.stdout.write( 'Gathering infos for RUN %i:  ' % run )
0136 
0137       #check the events available for this run
0138       NEventsDasOut = [ line for line in subprocess.getstatusoutput(initEnv+
0139        "das_client.py  --limit=9999 --query='summary dataset=%s run=%i | grep summary.nevents'"%(dataset,run))[1].splitlines()
0140                         if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ][-1]
0141       if(not NEventsDasOut.isdigit() ):
0142          print ("cannot retrieve the number of events from das, SKIPPING")
0143          #print NEventsDasOut
0144          continue
0145 
0146       if(FileList==""):firstRun=run;
0147       NEvents = int(NEventsDasOut)
0148 
0149       if(NEvents<=minNEvents):
0150          print ("only %i events in this run, SKIPPING" % NEvents)
0151          continue
0152 
0153       FileList+="#run=" + str(run) + " -->  NEvents="+str(NEvents/1000).rjust(8)+"K\n"
0154       resultsFiles = [ line for line in subprocess.getstatusoutput(initEnv+
0155                        "das_client.py  --limit=9999 --query='file dataset=%s run=%i'"%(dataset,run))[1].splitlines()
0156                        if not line.startswith('Showing') and 'SCRAM fatal' not in line and len(line)>0 ]
0157       if len(resultsFiles)==0 or 'Error' in " ".join(resultsFiles):
0158          print ("cannot retrieve the list of files from das, SKIPPING")
0159          #print resultsFiles
0160          continue
0161 
0162       for file in resultsFiles: FileList+='calibTreeList.extend(["'+file+'"])\n'
0163       NTotalEvents += NEvents;
0164 
0165       print("including %s events in gain processing, cumulative total=%i" % (str(NEvents).rjust(7),NTotalEvents))
0166       if(automatic==True and NTotalEvents >= maxNEvents):break;
0167 
0168 else:
0169    print("Get the list of calibTree from castor (eos ls " + CALIBTREEPATH + ")")
0170    calibTreeInfo = subprocess.getstatusoutput("eos ls -l "+CALIBTREEPATH)[1].split('\n');
0171    print(calibTreeInfo)
0172    # collect the list of runs and file size
0173    # calibTreeInfo.split()[8] - file name
0174    # calibTreeInfo.split()[4] - file size
0175    info_list = [(i.split()[8],
0176                  int( i.split()[8].replace("calibTree_","").replace(".root","").split('_')[0] ),
0177                  int( i.split()[4] )/1048576 ) for i in calibTreeInfo]
0178    info_list.sort( key=lambda tup: tup[1] )
0179 
0180    print("Check the number of events available")
0181    for info in info_list:
0182       if(len(info)<1):continue;
0183       size = info[2]
0184       if(size < 10): continue   #skip file<10MB
0185       run = info[1]
0186       if(run<firstRun or run in runsToVeto):continue
0187       if(lastRun>0 and run>lastRun):continue
0188       if not dataCert.checkRun(run,dataCertInfo):
0189          print("Skipping...")
0190          continue
0191       if run<295310:
0192          print("Skipping...")
0193          continue
0194       lastGoodRun = run
0195       NEvents = numberOfEvents("root://eoscms//eos/cms"+CALIBTREEPATH+'/'+info[0],calMode); 
0196       if(NEvents<=3000):continue #only keep runs with at least 3K events
0197       if(FileList==""):firstRun=run;
0198       FileList += 'calibTreeList.extend(["root://eoscms//eos/cms'+CALIBTREEPATH+'/'+info[0]+'"]) #' + str(size).rjust(6)+'MB  NEvents='+str(NEvents/1000).rjust(8)+'K\n'
0199       NTotalEvents += NEvents;
0200       print("Current number of events to process is " + str(NTotalEvents))
0201       if(automatic==True and NTotalEvents >= maxNEvents):break;
0202 
0203 if lastGoodRun < 0:
0204    print("No good run to process.")
0205    sys.exit()
0206 if(lastRun<=0):lastRun = lastGoodRun
0207 
0208 print("RunRange=[" + str(firstRun) + "," + str(lastRun) + "] --> NEvents=" + str(NTotalEvents/1000)+"K")
0209 
0210 if(automatic==True and NTotalEvents<2e6): #ask at least 2M events to perform the calibration
0211         print('Not Enough events to run the calibration')
0212         os.system('echo "Gain calibration postponed" | mail -s "Gain calibration postponed ('+str(firstRun)+' to '+str(lastRun)+') NEvents=' + str(NTotalEvents/1000)+'K" ' + mail)
0213         exit(0)
0214 
0215 name = "Run_"+str(firstRun)+"_to_"+str(lastRun)
0216 if len(calMode)>0:  name = name+"_"+calMode
0217 if(usePCL==True):   name = name+"_PCL"
0218 else:               name = name+"_CalibTree"
0219 print(name)
0220 
0221 oldDirectory = "7TeVData"
0222 newDirectory = "Data_"+name;
0223 os.system("mkdir -p " + newDirectory);
0224 os.system("cp " + oldDirectory + "/* " + newDirectory+"/.");
0225 file = open(newDirectory+"/FileList_cfg.py", "w")
0226 file.write("import FWCore.ParameterSet.Config as cms\n")
0227 file.write("calibTreeList = cms.untracked.vstring()\n")
0228 file.write("#TotalNumberOfEvent considered is %i\n" % NTotalEvents)
0229 file.write(FileList)
0230 file.close()
0231 os.system("cat " + newDirectory + "/FileList_cfg.py")
0232 os.system("sed -i 's|XXX_FIRSTRUN_XXX|"+str(firstRun)+"|g' "+newDirectory+"/*_cfg.py")
0233 os.system("sed -i 's|XXX_LASTRUN_XXX|"+str(lastRun)+"|g' "+newDirectory+"/*_cfg.py")
0234 os.system("sed -i 's|XXX_GT_XXX|"+globaltag+"|g' "+newDirectory+"/*_cfg.py")
0235 os.system("sed -i 's|XXX_PCL_XXX|"+str(usePCL)+"|g' "+newDirectory+"/*_cfg.py")
0236 os.system("sed -i 's|XXX_CALMODE_XXX|"+calMode+"|g' "+newDirectory+"/*_cfg.py")
0237 os.system("sed -i 's|XXX_DQMDIR_XXX|"+DQM_dir+"|g' "+newDirectory+"/*_cfg.py")
0238 os.chdir(newDirectory);
0239 
0240 job = initEnv
0241 job+= "\ncd %s; \npwd; \nls; \npython submitJob.py -f %s -l %s -p %s -P %s"%(os.getcwd(),firstRun,lastRun,usePCL,publish)
0242 job+= " -m %s -s %s -a %s"%(calMode,scriptDir,automatic)
0243 
0244 print("*** JOB : ***")
0245 print(job)
0246 print("cwd = %s"%(os.getcwd()))
0247 with open("job.sh","w") as f:
0248    f.write(job)
0249 os.system("chmod +x job.sh")
0250 submitCMD =  'bsub  -q 2nd -J G2prod -R "type == SLC6_64 && pool > 30000" "job.sh"'
0251 print(submitCMD)
0252 os.system(submitCMD)
0253 
0254 #if(os.system("sh sequence.sh \"" + name + "\" \"" + calMode + "\" \"CMS Preliminary  -  Run " + str(firstRun) + " to " + str(lastRun) + "\"")!=0):
0255 #   os.system('echo "Gain calibration failed" | mail -s "Gain calibration failed ('+name+')" ' + mail)        
0256 #else:
0257 #   if(publish==True):os.system("sh sequence.sh " + name);
0258 #   os.system('echo "Gain calibration done\nhttps://test-stripcalibvalidation.web.cern.ch/test-stripcalibvalidation/CalibrationValidation/ParticleGain/" | mail -s "Gain calibration done ('+name+')" ' + mail)
0259 #
0260 #if(False and usePCL==True):
0261 #   #Make the same results using the calibTrees for comparisons
0262 #   os.chdir(scriptDir); #go back to initial location
0263 #   os.system('python automatic_RunOnCalibTree.py --firstRun ' + str(firstRun) + ' --lastRun ' + str(lastRun) + ' --publish False --pcl False')
0264 #
0265 #if(automatic==True):
0266 #   #call the script one more time to make sure that we do not have a new run to process
0267 #   os.chdir(scriptDir); #go back to initial location
0268 #   os.system('python automatic_RunOnCalibTree.py')
0269 #