BeamSpotProducer/scripts/checkPayloads.py

0001 #!/usr/bin/env python3
0002 from __future__ import print_function
0003 import sys,os,subprocess,re
0004 import xmlrpclib
0005 from CommonMethods import *
0006
0007 try: # FUTURE: Python 2.6, prior to 2.6 requires simplejson
0008     import json
0009 except:
0010     try:
0011         import simplejson as json
0012     except:
0013         print("Please set a crab environment in order to get the proper JSON lib")
0014         sys.exit(1)
0015
0016 #####################################################################################
0017 def getUploadedIOVs(tagName,destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"):
0018     listIOVCommand = "cmscond_list_iov -c " + destDB + " -P /afs/cern.ch/cms/DB/conddb -t " + tagName
0019     dbError = subprocess.getstatusoutput( listIOVCommand )
0020     if dbError[0] != 0 :
0021         if dbError[1].find("metadata entry \"" + tagName + "\" does not exist") != -1:
0022             exit(dbError[1])
0023         else:
0024             exit("ERROR: Can\'t connect to db because:\n" + dbError[1])
0025
0026
0027     aCommand = listIOVCommand + " | grep DB= | awk \'{print $1}\'"
0028     #print aCommand
0029     output = subprocess.getstatusoutput( aCommand )
0030
0031     #WARNING when we pass to lumi IOV this should be long long
0032     if output[1] == '':
0033         exit("ERROR: The tag " + tagName + " exists but I can't get the value of the last IOV")
0034
0035     runs = []
0036     for run in output[1].split('\n'):
0037         runs.append(long(run))
0038
0039     return runs
0040
0041 #####################################################################################
0042 def getListOfRunsAndLumiFromFile(firstRun=-1,fileName=""):
0043     file = open(fileName);
0044     jsonFile = file.read();
0045     file.close()
0046     jsonList=json.loads(jsonFile);
0047
0048     selected_dcs = {};
0049     for element in jsonList:
0050         selected_dcs[long(element)]=jsonList[element]
0051     return selected_dcs
0052
0053 #####################################################################################
0054 def getListOfRunsAndLumiFromRR(firstRun=-1,error=""):
0055     RunReg  ="http://pccmsdqm04.cern.ch/runregistry"
0056     #RunReg  = "http://localhost:40010/runregistry"
0057     #Dataset=%Online%
0058     Group   = "Collisions10"
0059
0060     # get handler to RR XML-RPC server
0061     FULLADDRESS=RunReg + "/xmlrpc"
0062     #print "RunRegistry from: ",FULLADDRESS
0063     #firstRun = 153000
0064     server = xmlrpclib.ServerProxy(FULLADDRESS)
0065     #sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) + " and {datasetName} LIKE '" + Dataset + "'"
0066     sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun)
0067
0068     tries = 0;
0069     maxAttempts = 3
0070     while tries<maxAttempts:
0071         try:
0072             run_data = server.DataExporter.export('RUN', 'GLOBAL', 'csv_runs', sel_runtable)
0073             break
0074         except:
0075             tries += 1
0076             print("Trying to get run data. This fails only 2-3 times so don't panic yet...", tries, "/", maxAttempts)
0077             time.sleep(1)
0078             print("Exception type: ", sys.exc_info()[0])
0079         if tries==maxAttempts:
0080             error = "Ok, now panic...run registry unaccessible...I'll get the runs from a json file!"
0081             print(error);
0082             return {};
0083
0084     listOfRuns=[]
0085     runErrors = {}
0086     for line in run_data.split("\n"):
0087         run=line.split(',')[0]
0088         if run.isdigit():
0089             listOfRuns.append(run)
0090
0091     tries = 0
0092     maxAttempts = 3
0093     firstRun = listOfRuns[len(listOfRuns)-1];
0094     lastRun  = listOfRuns[0];
0095     sel_dcstable="{groupName} ='" + Group + "' and {runNumber} >= " + str(firstRun) + " and {runNumber} <= " + str(lastRun) + " and {parDcsBpix} = 1 and {parDcsFpix} = 1 and {parDcsTibtid} = 1 and {parDcsTecM} = 1 and {parDcsTecP} = 1 and {parDcsTob} = 1 and {parDcsEbminus} = 1 and {parDcsEbplus} = 1 and {parDcsEeMinus} = 1 and {parDcsEePlus} = 1 and {parDcsEsMinus} = 1 and {parDcsEsPlus} = 1 and {parDcsHbheA} = 1 and {parDcsHbheB} = 1 and {parDcsHbheC} = 1 and {parDcsH0} = 1 and {parDcsHf} = 1"
0096     while tries<maxAttempts:
0097         try:
0098             dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json'    , sel_dcstable)
0099             break
0100         except:
0101             tries += 1
0102             print("I was able to get the list of runs and now I am trying to access the detector status", tries, "/", maxAttempts)
0103             time.sleep(1)
0104             print("Exception type: ", sys.exc_info()[0])
0105
0106         if tries==maxAttempts:
0107             error = "Ok, now panic...run registry unaccessible...I'll get the runs from a json file!"
0108             print(error);
0109             return {};
0110
0111     #This is the original and shold work in the furture as soon as the server will be moved to a more powerfull PC
0112     #while tries<maxAttempts:
0113     #    try:
0114     #        run_data = server.DataExporter.export('RUN'           , 'GLOBAL', 'csv_runs', sel_runtable)
0115     #        dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json'    , sel_dcstable)
0116     #        #print run_data
0117     #        #print dcs_data
0118     #        break
0119     #    except:
0120     #        print "Something wrong in accessing runregistry, retrying in 5s...."
0121     #        tries += 1
0122     #        time.sleep(2)
0123     #        print "Exception type: ", sys.exc_info()[0]
0124     #
0125     #    if tries==maxAttempts:
0126     #        error = "Run registry unaccessible.....exiting now"
0127     #        sys.exit(error)
0128
0129
0130
0131     selected_dcs={}
0132     jsonList=json.loads(dcs_data)
0133
0134     #for element in jsonList:
0135     for element in listOfRuns:
0136         #if element in listOfRuns:
0137         if element in jsonList:
0138             selected_dcs[long(element)]=jsonList[element]
0139         else:
0140             #print "WARNING: Run " + element + " is a collision10 run with 0 lumis in Run Registry!"
0141             selected_dcs[long(element)]= [[]]
0142     return selected_dcs
0143
0144 #####################################################################################
0145 def main():
0146     usage = "USAGE: ./checkPayloads.py (optional tagNumber) (optional \"lumi\") (optional \"z\" (optional destDB)"
0147     printExtra = False
0148     tagNumber = "14"
0149     dbBase = ""
0150     sigmaZ = ""
0151
0152     if len(sys.argv) >= 2:
0153         if not sys.argv[1].isdigit():
0154             exit(usage)
0155         else:
0156             tagNumber = sys.argv[1]
0157     if len(sys.argv) >= 3:
0158         if not sys.argv[2] == "lumi":
0159             exit(usage)
0160         else:
0161             dbBase = "_LumiBased"
0162     if len(sys.argv) >= 4:
0163         if not sys.argv[3] == "z":
0164             exit(usage)
0165         else:
0166             sigmaZ = "_SigmaZ"
0167     destDB = ""
0168     if(len(sys.argv) > 4):
0169         destDB = sys.argv[4]
0170     #132573 Beam lost immediately
0171     #132958 Bad strips
0172     #133081 Bad pixels bad strips
0173     #133242 Bad strips
0174     #133472 Bad strips
0175     #133473 Only 20 lumisection, run duration 00:00:03:00
0176     #133509 Should be good!!!!!!!!!!
0177     #136290 Bad Pixels bad strips
0178     #138560 Bad pixels bad strips
0179     #138562 Bad HLT bad L1T, need to rescale the Jet Triggers
0180     #139363 NOT in the bad list but only 15 lumis and stopped for DAQ problems
0181     #139455 Bad Pixels and Strips and stopped because of HCAL trigger rate too high
0182     #140133 Beams dumped
0183     #140182 No pixel and Strips with few entries
0184     #141865 Pixel are bad but Strips work. Run is acceptable but need relaxed cuts since there are no pixels. BeamWidth measurement is bad 80um compared to 40um
0185     #142461 Run crashed immediately due to PIX, stable beams since LS1
0186     #142465 PostCollsions10, beams lost, HCAl DQM partly working
0187     #142503 Bad pixels bad strips
0188     #142653 Strips not in data taking
0189     #143977 No Beam Strips and Pixels bad
0190     #148859 Strips and Pixels HV off waiting for beam
0191
0192     knownMissingRunList = [132573,132958,133081,133242,133472,133473,136290,138560,138562,139455,140133,140182,142461,142465,142503,142653,143977,148859]
0193     tagName = "BeamSpotObjects_2009" + dbBase + sigmaZ + "_v" + tagNumber + "_offline"
0194     print("Checking payloads for tag " + tagName)
0195     listOfRunsAndLumi = {};
0196     #listOfRunsAndLumi = getListOfRunsAndLumiFromRR(-1);
0197     if(not listOfRunsAndLumi):
0198         listOfRunsAndLumi = getListOfRunsAndLumiFromFile(-1,"/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions10/7TeV/StreamExpress/Cert_132440-149442_7TeV_StreamExpress_Collisions10_JSON_v3.txt");
0199     tmpListOfIOVs = []
0200     if(destDB != ""):
0201         tmpListOfIOVs = getUploadedIOVs(tagName,destDB)
0202     else:
0203         tmpListOfIOVs = getUploadedIOVs(tagName)
0204
0205
0206     listOfIOVs = []
0207     if(dbBase == ''):
0208         listOfIOVs = tmpListOfIOVs
0209     else:
0210         for iov in tmpListOfIOVs:
0211             if((iov >> 32) not in listOfIOVs):
0212                 listOfIOVs.append(iov >>32)
0213     RRRuns = sorted(listOfRunsAndLumi.keys())
0214     for run in RRRuns:
0215         #print listOfRunsAndLumiFromRR[run]
0216         if run not in listOfIOVs:
0217             extraMsg = ""
0218             if listOfRunsAndLumi[run] == [[]]:
0219                 extraMsg = " but it is empty in the RR"
0220                 if not printExtra: continue
0221             if run in knownMissingRunList :
0222                 extraMsg = " but this run is know to be bad " #+ runErrors[run]
0223                 if not printExtra: continue
0224             print("Run: " + str(run) + " is missing for DB tag " + tagName + extraMsg)
0225
0226
0227 if __name__ == "__main__":
0228     main()
0229