BeamSpotProducer/scripts/checkRuns.py

0001 #!/usr/bin/env python3
0002 from __future__ import print_function
0003 from builtins import range
0004 import sys,os,re
0005 import xmlrpclib
0006 from CommonMethods import *
0007 try: # FUTURE: Python 2.6, prior to 2.6 requires simplejson
0008     import json
0009 except:
0010     try:
0011         import simplejson as json
0012     except:
0013         print("Please set a crab environment in order to get the proper JSON lib")
0014         sys.exit(1)
0015
0016 #####################################################################################
0017 def getListOfRunsAndLumiFromFile(firstRun=-1,fileName=""):
0018     file = open(fileName);
0019     jsonFile = file.read();
0020     file.close()
0021     jsonList=json.loads(jsonFile);
0022
0023     selected_dcs = {};
0024     for element in jsonList:
0025         selected_dcs[long(element)]=jsonList[element]
0026     return selected_dcs
0027
0028 #####################################################################################
0029 def getListOfRunsAndLumiFromRR(firstRun=-1,error=""):
0030     RunReg  ="http://pccmsdqm04.cern.ch/runregistry"
0031     #RunReg  = "http://localhost:40010/runregistry"
0032     #Dataset=%Online%
0033     Group   = "Collisions10"
0034
0035     # get handler to RR XML-RPC server
0036     FULLADDRESS=RunReg + "/xmlrpc"
0037     #print "RunRegistry from: ",FULLADDRESS
0038     #firstRun = 153000
0039     server = xmlrpclib.ServerProxy(FULLADDRESS)
0040     #sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) + " and {datasetName} LIKE '" + Dataset + "'"
0041     sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun)
0042
0043     tries = 0;
0044     maxAttempts = 3
0045     while tries<maxAttempts:
0046         try:
0047             run_data = server.DataExporter.export('RUN', 'GLOBAL', 'csv_runs', sel_runtable)
0048             break
0049         except:
0050             tries += 1
0051             print("Trying to get run data. This fails only 2-3 times so don't panic yet...", tries, "/", maxAttempts)
0052             time.sleep(1)
0053             print("Exception type: ", sys.exc_info()[0])
0054         if tries==maxAttempts:
0055             error = "Ok, now panic...run registry unaccessible...I'll get the runs from a json file!"
0056             print(error);
0057             return {};
0058
0059     listOfRuns=[]
0060     runErrors = {}
0061     for line in run_data.split("\n"):
0062         run=line.split(',')[0]
0063         if run.isdigit():
0064             listOfRuns.append(run)
0065
0066     tries = 0
0067     maxAttempts = 3
0068     firstRun = listOfRuns[len(listOfRuns)-1];
0069     lastRun  = listOfRuns[0];
0070     sel_dcstable="{groupName} ='" + Group + "' and {runNumber} >= " + str(firstRun) + " and {runNumber} <= " + str(lastRun) + " and {parDcsBpix} = 1 and {parDcsFpix} = 1 and {parDcsTibtid} = 1 and {parDcsTecM} = 1 and {parDcsTecP} = 1 and {parDcsTob} = 1 and {parDcsEbminus} = 1 and {parDcsEbplus} = 1 and {parDcsEeMinus} = 1 and {parDcsEePlus} = 1 and {parDcsEsMinus} = 1 and {parDcsEsPlus} = 1 and {parDcsHbheA} = 1 and {parDcsHbheB} = 1 and {parDcsHbheC} = 1 and {parDcsH0} = 1 and {parDcsHf} = 1"
0071     while tries<maxAttempts:
0072         try:
0073             dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json'    , sel_dcstable)
0074             break
0075         except:
0076             tries += 1
0077             print("I was able to get the list of runs and now I am trying to access the detector status", tries, "/", maxAttempts)
0078             time.sleep(1)
0079             print("Exception type: ", sys.exc_info()[0])
0080
0081     if tries==maxAttempts:
0082         error = "Ok, now panic...run registry unaccessible...I'll get the runs from a json file!"
0083         print(error);
0084         return {};
0085
0086     selected_dcs={}
0087     jsonList=json.loads(dcs_data)
0088
0089     #for element in jsonList:
0090     for element in listOfRuns:
0091         #if element in listOfRuns:
0092         if element in jsonList:
0093             selected_dcs[long(element)]=jsonList[element]
0094         else:
0095             #print "WARNING: Run " + element + " is a collision10 run with 0 lumis in Run Registry!"
0096             selected_dcs[long(element)]= [[]]
0097     return selected_dcs
0098
0099 #####################################################################################
0100 def main():
0101     filesDir = "LatestRuns/Results/";
0102     fileList = ls(filesDir)
0103     listOfRunsAndLumi = {};
0104     #listOfRunsAndLumi = getListOfRunsAndLumiFromRR(-1);
0105     if(not listOfRunsAndLumi):
0106         listOfRunsAndLumi = getListOfRunsAndLumiFromFile(-1,"/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions10/7TeV/StreamExpress/Cert_132440-149442_7TeV_StreamExpress_Collisions10_JSON_v3.txt");
0107
0108     runKeys = listOfRunsAndLumi.keys();
0109     runKeys.sort();
0110     runFiles = [];
0111     for fileName in fileList:
0112         regExp = re.search('(\D+)(\d+)_(\d+)_(\d+).txt',fileName);
0113         if(not regExp):
0114             error = "Can't find reg exp";
0115             exit(error);
0116         runFiles.append(long(regExp.group(3)));
0117
0118     #for run in runKeys:
0119     #    if(run not in runFiles):
0120     #        print "Can't find run", run, "in the files!"
0121
0122     runsAndLumisInRR = {};
0123     for run in runKeys:
0124         RRList = [];
0125         for lumiRange in listOfRunsAndLumi[run]:
0126             if lumiRange != []:
0127                 for l in range(lumiRange[0],lumiRange[1]+1):
0128                     RRList.append(long(l));
0129         #print run, "->", RRList;
0130         runsAndLumisInRR[run] = RRList;
0131
0132     runsAndLumisProcessed = {}
0133     for fileName in fileList:
0134         file = open(filesDir+fileName)
0135         for line in file:
0136             if line.find("Runnumber") != -1:
0137                 run = long(line.replace('\n','').split(' ')[1])
0138             elif line.find("LumiRange") != -1:
0139                 lumiLine = line.replace('\n','').split(' ')
0140                 begLumi = long(lumiLine[1])
0141                 endLumi = long(lumiLine[3])
0142                 if begLumi != endLumi:
0143                     error = "The lumi range is greater than 1 for run " + str(run) + " " + line + " in file: " + runListDir + fileName
0144                     exit(error)
0145                 else:
0146                     if not run in runsAndLumisProcessed:
0147                         runsAndLumisProcessed[run] = []
0148                     if begLumi in runsAndLumisProcessed[run]:
0149                         print("Lumi " + str(begLumi) + " in event " + str(run) + " already exist. This MUST not happen but right now I will ignore this lumi!")
0150                     else:
0151                         runsAndLumisProcessed[run].append(begLumi)
0152         file.close()
0153         #print run, "->", runsAndLumisProcessed[run];
0154
0155     for run in runKeys:
0156         missingLumis = [];
0157         for lumi in runsAndLumisInRR[run]:
0158             #print str(counter) + "->" + str(lumi)
0159             #counter += 1
0160             if(run not in runFiles):
0161                 print("Can't find run", run, "in the files!")
0162                 break ;
0163             elif( not lumi in runsAndLumisProcessed[run]):
0164                 missingLumis.append(lumi)
0165         if(len(missingLumis) != 0):
0166             print("In run", run, "these lumis are missing ->", missingLumis)
0167
0168
0169 if __name__ == "__main__":
0170         main()
0171