BeamSpotProducer/scripts/checkPayloads.py

0001 #!/usr/bin/env python3
0002 import sys,os,subprocess,re
0003 import xmlrpclib
0004 from CommonMethods import *
0005
0006 try: # FUTURE: Python 2.6, prior to 2.6 requires simplejson
0007     import json
0008 except:
0009     try:
0010         import simplejson as json
0011     except:
0012         print("Please set a crab environment in order to get the proper JSON lib")
0013         sys.exit(1)
0014
0015 #####################################################################################
0016 def getUploadedIOVs(tagName,destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"):
0017     listIOVCommand = "cmscond_list_iov -c " + destDB + " -P /afs/cern.ch/cms/DB/conddb -t " + tagName
0018     dbError = subprocess.getstatusoutput( listIOVCommand )
0019     if dbError[0] != 0 :
0020         if dbError[1].find("metadata entry \"" + tagName + "\" does not exist") != -1:
0021             exit(dbError[1])
0022         else:
0023             exit("ERROR: Can\'t connect to db because:\n" + dbError[1])
0024
0025
0026     aCommand = listIOVCommand + " | grep DB= | awk \'{print $1}\'"
0027     #print aCommand
0028     output = subprocess.getstatusoutput( aCommand )
0029
0030     #WARNING when we pass to lumi IOV this should be long long
0031     if output[1] == '':
0032         exit("ERROR: The tag " + tagName + " exists but I can't get the value of the last IOV")
0033
0034     runs = []
0035     for run in output[1].split('\n'):
0036         runs.append(long(run))
0037
0038     return runs
0039
0040 #####################################################################################
0041 def getListOfRunsAndLumiFromFile(firstRun=-1,fileName=""):
0042     file = open(fileName);
0043     jsonFile = file.read();
0044     file.close()
0045     jsonList=json.loads(jsonFile);
0046
0047     selected_dcs = {};
0048     for element in jsonList:
0049         selected_dcs[long(element)]=jsonList[element]
0050     return selected_dcs
0051
0052 #####################################################################################
0053 def getListOfRunsAndLumiFromRR(firstRun=-1,error=""):
0054     RunReg  ="http://pccmsdqm04.cern.ch/runregistry"
0055     #RunReg  = "http://localhost:40010/runregistry"
0056     #Dataset=%Online%
0057     Group   = "Collisions10"
0058
0059     # get handler to RR XML-RPC server
0060     FULLADDRESS=RunReg + "/xmlrpc"
0061     #print "RunRegistry from: ",FULLADDRESS
0062     #firstRun = 153000
0063     server = xmlrpclib.ServerProxy(FULLADDRESS)
0064     #sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) + " and {datasetName} LIKE '" + Dataset + "'"
0065     sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun)
0066
0067     tries = 0;
0068     maxAttempts = 3
0069     while tries<maxAttempts:
0070         try:
0071             run_data = server.DataExporter.export('RUN', 'GLOBAL', 'csv_runs', sel_runtable)
0072             break
0073         except:
0074             tries += 1
0075             print("Trying to get run data. This fails only 2-3 times so don't panic yet...", tries, "/", maxAttempts)
0076             time.sleep(1)
0077             print("Exception type: ", sys.exc_info()[0])
0078         if tries==maxAttempts:
0079             error = "Ok, now panic...run registry unaccessible...I'll get the runs from a json file!"
0080             print(error);
0081             return {};
0082
0083     listOfRuns=[]
0084     runErrors = {}
0085     for line in run_data.split("\n"):
0086         run=line.split(',')[0]
0087         if run.isdigit():
0088             listOfRuns.append(run)
0089
0090     tries = 0
0091     maxAttempts = 3
0092     firstRun = listOfRuns[len(listOfRuns)-1];
0093     lastRun  = listOfRuns[0];
0094     sel_dcstable="{groupName} ='" + Group + "' and {runNumber} >= " + str(firstRun) + " and {runNumber} <= " + str(lastRun) + " and {parDcsBpix} = 1 and {parDcsFpix} = 1 and {parDcsTibtid} = 1 and {parDcsTecM} = 1 and {parDcsTecP} = 1 and {parDcsTob} = 1 and {parDcsEbminus} = 1 and {parDcsEbplus} = 1 and {parDcsEeMinus} = 1 and {parDcsEePlus} = 1 and {parDcsEsMinus} = 1 and {parDcsEsPlus} = 1 and {parDcsHbheA} = 1 and {parDcsHbheB} = 1 and {parDcsHbheC} = 1 and {parDcsH0} = 1 and {parDcsHf} = 1"
0095     while tries<maxAttempts:
0096         try:
0097             dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json'    , sel_dcstable)
0098             break
0099         except:
0100             tries += 1
0101             print("I was able to get the list of runs and now I am trying to access the detector status", tries, "/", maxAttempts)
0102             time.sleep(1)
0103             print("Exception type: ", sys.exc_info()[0])
0104
0105         if tries==maxAttempts:
0106             error = "Ok, now panic...run registry unaccessible...I'll get the runs from a json file!"
0107             print(error);
0108             return {};
0109
0110     #This is the original and shold work in the furture as soon as the server will be moved to a more powerfull PC
0111     #while tries<maxAttempts:
0112     #    try:
0113     #        run_data = server.DataExporter.export('RUN'           , 'GLOBAL', 'csv_runs', sel_runtable)
0114     #        dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json'    , sel_dcstable)
0115     #        #print run_data
0116     #        #print dcs_data
0117     #        break
0118     #    except:
0119     #        print "Something wrong in accessing runregistry, retrying in 5s...."
0120     #        tries += 1
0121     #        time.sleep(2)
0122     #        print "Exception type: ", sys.exc_info()[0]
0123     #
0124     #    if tries==maxAttempts:
0125     #        error = "Run registry unaccessible.....exiting now"
0126     #        sys.exit(error)
0127
0128
0129
0130     selected_dcs={}
0131     jsonList=json.loads(dcs_data)
0132
0133     #for element in jsonList:
0134     for element in listOfRuns:
0135         #if element in listOfRuns:
0136         if element in jsonList:
0137             selected_dcs[long(element)]=jsonList[element]
0138         else:
0139             #print "WARNING: Run " + element + " is a collision10 run with 0 lumis in Run Registry!"
0140             selected_dcs[long(element)]= [[]]
0141     return selected_dcs
0142
0143 #####################################################################################
0144 def main():
0145     usage = "USAGE: ./checkPayloads.py (optional tagNumber) (optional \"lumi\") (optional \"z\" (optional destDB)"
0146     printExtra = False
0147     tagNumber = "14"
0148     dbBase = ""
0149     sigmaZ = ""
0150
0151     if len(sys.argv) >= 2:
0152         if not sys.argv[1].isdigit():
0153             exit(usage)
0154         else:
0155             tagNumber = sys.argv[1]
0156     if len(sys.argv) >= 3:
0157         if not sys.argv[2] == "lumi":
0158             exit(usage)
0159         else:
0160             dbBase = "_LumiBased"
0161     if len(sys.argv) >= 4:
0162         if not sys.argv[3] == "z":
0163             exit(usage)
0164         else:
0165             sigmaZ = "_SigmaZ"
0166     destDB = ""
0167     if(len(sys.argv) > 4):
0168         destDB = sys.argv[4]
0169     #132573 Beam lost immediately
0170     #132958 Bad strips
0171     #133081 Bad pixels bad strips
0172     #133242 Bad strips
0173     #133472 Bad strips
0174     #133473 Only 20 lumisection, run duration 00:00:03:00
0175     #133509 Should be good!!!!!!!!!!
0176     #136290 Bad Pixels bad strips
0177     #138560 Bad pixels bad strips
0178     #138562 Bad HLT bad L1T, need to rescale the Jet Triggers
0179     #139363 NOT in the bad list but only 15 lumis and stopped for DAQ problems
0180     #139455 Bad Pixels and Strips and stopped because of HCAL trigger rate too high
0181     #140133 Beams dumped
0182     #140182 No pixel and Strips with few entries
0183     #141865 Pixel are bad but Strips work. Run is acceptable but need relaxed cuts since there are no pixels. BeamWidth measurement is bad 80um compared to 40um
0184     #142461 Run crashed immediately due to PIX, stable beams since LS1
0185     #142465 PostCollsions10, beams lost, HCAl DQM partly working
0186     #142503 Bad pixels bad strips
0187     #142653 Strips not in data taking
0188     #143977 No Beam Strips and Pixels bad
0189     #148859 Strips and Pixels HV off waiting for beam
0190
0191     knownMissingRunList = [132573,132958,133081,133242,133472,133473,136290,138560,138562,139455,140133,140182,142461,142465,142503,142653,143977,148859]
0192     tagName = "BeamSpotObjects_2009" + dbBase + sigmaZ + "_v" + tagNumber + "_offline"
0193     print("Checking payloads for tag " + tagName)
0194     listOfRunsAndLumi = {};
0195     #listOfRunsAndLumi = getListOfRunsAndLumiFromRR(-1);
0196     if(not listOfRunsAndLumi):
0197         listOfRunsAndLumi = getListOfRunsAndLumiFromFile(-1,"/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions10/7TeV/StreamExpress/Cert_132440-149442_7TeV_StreamExpress_Collisions10_JSON_v3.txt");
0198     tmpListOfIOVs = []
0199     if(destDB != ""):
0200         tmpListOfIOVs = getUploadedIOVs(tagName,destDB)
0201     else:
0202         tmpListOfIOVs = getUploadedIOVs(tagName)
0203
0204
0205     listOfIOVs = []
0206     if(dbBase == ''):
0207         listOfIOVs = tmpListOfIOVs
0208     else:
0209         for iov in tmpListOfIOVs:
0210             if((iov >> 32) not in listOfIOVs):
0211                 listOfIOVs.append(iov >>32)
0212     RRRuns = sorted(listOfRunsAndLumi.keys())
0213     for run in RRRuns:
0214         #print listOfRunsAndLumiFromRR[run]
0215         if run not in listOfIOVs:
0216             extraMsg = ""
0217             if listOfRunsAndLumi[run] == [[]]:
0218                 extraMsg = " but it is empty in the RR"
0219                 if not printExtra: continue
0220             if run in knownMissingRunList :
0221                 extraMsg = " but this run is know to be bad " #+ runErrors[run]
0222                 if not printExtra: continue
0223             print("Run: " + str(run) + " is missing for DB tag " + tagName + extraMsg)
0224
0225
0226 if __name__ == "__main__":
0227     main()
0228