BeamSpotProducer/scripts/checkRuns.py

0001 #!/usr/bin/env python3
0002 from builtins import range
0003 import sys,os,re
0004 import xmlrpclib
0005 from CommonMethods import *
0006 try: # FUTURE: Python 2.6, prior to 2.6 requires simplejson
0007     import json
0008 except:
0009     try:
0010         import simplejson as json
0011     except:
0012         print("Please set a crab environment in order to get the proper JSON lib")
0013         sys.exit(1)
0014
0015 #####################################################################################
0016 def getListOfRunsAndLumiFromFile(firstRun=-1,fileName=""):
0017     file = open(fileName);
0018     jsonFile = file.read();
0019     file.close()
0020     jsonList=json.loads(jsonFile);
0021
0022     selected_dcs = {};
0023     for element in jsonList:
0024         selected_dcs[long(element)]=jsonList[element]
0025     return selected_dcs
0026
0027 #####################################################################################
0028 def getListOfRunsAndLumiFromRR(firstRun=-1,error=""):
0029     RunReg  ="http://pccmsdqm04.cern.ch/runregistry"
0030     #RunReg  = "http://localhost:40010/runregistry"
0031     #Dataset=%Online%
0032     Group   = "Collisions10"
0033
0034     # get handler to RR XML-RPC server
0035     FULLADDRESS=RunReg + "/xmlrpc"
0036     #print "RunRegistry from: ",FULLADDRESS
0037     #firstRun = 153000
0038     server = xmlrpclib.ServerProxy(FULLADDRESS)
0039     #sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) + " and {datasetName} LIKE '" + Dataset + "'"
0040     sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun)
0041
0042     tries = 0;
0043     maxAttempts = 3
0044     while tries<maxAttempts:
0045         try:
0046             run_data = server.DataExporter.export('RUN', 'GLOBAL', 'csv_runs', sel_runtable)
0047             break
0048         except:
0049             tries += 1
0050             print("Trying to get run data. This fails only 2-3 times so don't panic yet...", tries, "/", maxAttempts)
0051             time.sleep(1)
0052             print("Exception type: ", sys.exc_info()[0])
0053         if tries==maxAttempts:
0054             error = "Ok, now panic...run registry unaccessible...I'll get the runs from a json file!"
0055             print(error);
0056             return {};
0057
0058     listOfRuns=[]
0059     runErrors = {}
0060     for line in run_data.split("\n"):
0061         run=line.split(',')[0]
0062         if run.isdigit():
0063             listOfRuns.append(run)
0064
0065     tries = 0
0066     maxAttempts = 3
0067     firstRun = listOfRuns[len(listOfRuns)-1];
0068     lastRun  = listOfRuns[0];
0069     sel_dcstable="{groupName} ='" + Group + "' and {runNumber} >= " + str(firstRun) + " and {runNumber} <= " + str(lastRun) + " and {parDcsBpix} = 1 and {parDcsFpix} = 1 and {parDcsTibtid} = 1 and {parDcsTecM} = 1 and {parDcsTecP} = 1 and {parDcsTob} = 1 and {parDcsEbminus} = 1 and {parDcsEbplus} = 1 and {parDcsEeMinus} = 1 and {parDcsEePlus} = 1 and {parDcsEsMinus} = 1 and {parDcsEsPlus} = 1 and {parDcsHbheA} = 1 and {parDcsHbheB} = 1 and {parDcsHbheC} = 1 and {parDcsH0} = 1 and {parDcsHf} = 1"
0070     while tries<maxAttempts:
0071         try:
0072             dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json'    , sel_dcstable)
0073             break
0074         except:
0075             tries += 1
0076             print("I was able to get the list of runs and now I am trying to access the detector status", tries, "/", maxAttempts)
0077             time.sleep(1)
0078             print("Exception type: ", sys.exc_info()[0])
0079
0080     if tries==maxAttempts:
0081         error = "Ok, now panic...run registry unaccessible...I'll get the runs from a json file!"
0082         print(error);
0083         return {};
0084
0085     selected_dcs={}
0086     jsonList=json.loads(dcs_data)
0087
0088     #for element in jsonList:
0089     for element in listOfRuns:
0090         #if element in listOfRuns:
0091         if element in jsonList:
0092             selected_dcs[long(element)]=jsonList[element]
0093         else:
0094             #print "WARNING: Run " + element + " is a collision10 run with 0 lumis in Run Registry!"
0095             selected_dcs[long(element)]= [[]]
0096     return selected_dcs
0097
0098 #####################################################################################
0099 def main():
0100     filesDir = "LatestRuns/Results/";
0101     fileList = ls(filesDir)
0102     listOfRunsAndLumi = {};
0103     #listOfRunsAndLumi = getListOfRunsAndLumiFromRR(-1);
0104     if(not listOfRunsAndLumi):
0105         listOfRunsAndLumi = getListOfRunsAndLumiFromFile(-1,"/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions10/7TeV/StreamExpress/Cert_132440-149442_7TeV_StreamExpress_Collisions10_JSON_v3.txt");
0106
0107     runKeys = listOfRunsAndLumi.keys();
0108     runKeys.sort();
0109     runFiles = [];
0110     for fileName in fileList:
0111         regExp = re.search('(\D+)(\d+)_(\d+)_(\d+).txt',fileName);
0112         if(not regExp):
0113             error = "Can't find reg exp";
0114             exit(error);
0115         runFiles.append(long(regExp.group(3)));
0116
0117     #for run in runKeys:
0118     #    if(run not in runFiles):
0119     #        print "Can't find run", run, "in the files!"
0120
0121     runsAndLumisInRR = {};
0122     for run in runKeys:
0123         RRList = [];
0124         for lumiRange in listOfRunsAndLumi[run]:
0125             if lumiRange != []:
0126                 for l in range(lumiRange[0],lumiRange[1]+1):
0127                     RRList.append(long(l));
0128         #print run, "->", RRList;
0129         runsAndLumisInRR[run] = RRList;
0130
0131     runsAndLumisProcessed = {}
0132     for fileName in fileList:
0133         file = open(filesDir+fileName)
0134         for line in file:
0135             if line.find("Runnumber") != -1:
0136                 run = long(line.replace('\n','').split(' ')[1])
0137             elif line.find("LumiRange") != -1:
0138                 lumiLine = line.replace('\n','').split(' ')
0139                 begLumi = long(lumiLine[1])
0140                 endLumi = long(lumiLine[3])
0141                 if begLumi != endLumi:
0142                     error = "The lumi range is greater than 1 for run " + str(run) + " " + line + " in file: " + runListDir + fileName
0143                     exit(error)
0144                 else:
0145                     if not run in runsAndLumisProcessed:
0146                         runsAndLumisProcessed[run] = []
0147                     if begLumi in runsAndLumisProcessed[run]:
0148                         print("Lumi " + str(begLumi) + " in event " + str(run) + " already exist. This MUST not happen but right now I will ignore this lumi!")
0149                     else:
0150                         runsAndLumisProcessed[run].append(begLumi)
0151         file.close()
0152         #print run, "->", runsAndLumisProcessed[run];
0153
0154     for run in runKeys:
0155         missingLumis = [];
0156         for lumi in runsAndLumisInRR[run]:
0157             #print str(counter) + "->" + str(lumi)
0158             #counter += 1
0159             if(run not in runFiles):
0160                 print("Can't find run", run, "in the files!")
0161                 break ;
0162             elif( not lumi in runsAndLumisProcessed[run]):
0163                 missingLumis.append(lumi)
0164         if(len(missingLumis) != 0):
0165             print("In run", run, "these lumis are missing ->", missingLumis)
0166
0167
0168 if __name__ == "__main__":
0169         main()
0170