CommonAlignmentProducer/scripts/AlCaHLTBitMon_QueryRunRegistry.py

0001 #!/usr/bin/env python3
0002
0003 #-----------------------------------------------------
0004 # original author: Andrea Lucaroni
0005 # Revision:        $Revision: 1.3 $
0006 # Last update:     $Date: 2012/05/22 08:22:04 $
0007 # by:              $Author: taroni $
0008 #-----------------------------------------------------
0009
0010 from xml.dom import minidom
0011 import re
0012 import json
0013 import os
0014 import stat
0015 import sys
0016
0017 #include DBS
0018 from DBSAPI.dbsApiException import DbsException
0019 import DBSAPI.dbsApi
0020 from DBSAPI.dbsApiException import *
0021
0022 # include XML-RPC client library
0023 # RR API uses XML-RPC webservices interface for data access
0024 import xmlrpclib
0025
0026 import array
0027 import pickle as pk
0028
0029 from optparse import OptionParser
0030 #####DEBUG
0031 DEBUG=0
0032 import inspect
0033
0034 def lineno():
0035     """Returns the current line number in our program."""
0036     return inspect.currentframe().f_back.f_lineno
0037
0038 #size file
0039 def filesize1(n):
0040     info = os.stat(n)
0041     sz = info[stat.ST_SIZE]
0042     return sz
0043
0044 ### lumiCalc
0045 def printLumi(file,namefile):
0046     if(filesize1(file) != 0):
0047         string= "lumiCalc2.py -c frontier://LumiCalc/CMS_LUMI_PROD -i "
0048         string1= " --nowarning overview >"
0049         string2= string + file + string1 + namefile
0050         data = os.system(string2)
0051     else:
0052         data = ""
0053         print("0 lumi are not avaible")
0054     return data
0055
0056 ###file  dbs
0057 def DBSquery(dataset,site,run):
0058
0059     url = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
0060     if DEBUG:
0061         print(lineno())
0062     args = {}
0063     args['url']     = url
0064     args['level']   = 'CRITICAL'
0065     if DEBUG:
0066         print(lineno())
0067     api = DBSAPI.dbsApi.DbsApi(args)
0068     if DEBUG:
0069         print(lineno())
0070     try:
0071         files = api.listFiles(path=dataset,tier_list =site,runNumber=run)
0072     except DbsApiException as ex:
0073         print("Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() ))
0074         files = ""
0075         if ex.getErrorCode() not in (None, ""):
0076             print("DBS Exception Error Code: ", ex.getErrorCode())
0077     if DEBUG:
0078         print(lineno())
0079     return files
0080
0081 ###file cff data
0082 def makecff(file_list,namefile):
0083     file1 = open(namefile ,'w')
0084     stringS  =           "process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) )\n"
0085     stringS  = stringS + "readFiles = cms.untracked.vstring()\n"
0086     stringS  = stringS + "secFiles = cms.untracked.vstring()\n"
0087     stringS  = stringS + "\n"
0088     file1.write(stringS)
0089
0090     filecount = 0
0091     extendOpen = 0
0092     for filename in file_list:
0093
0094         if extendOpen == 0:
0095             stringS  = "readFiles.extend([\n"
0096             file1.write(stringS)
0097             extendOpen = 1
0098
0099         stringS  =           "     '"
0100         stringS  = stringS + str(filename)
0101         stringS  = stringS + "',\n"
0102         file1.write(stringS)
0103         filecount = filecount + 1
0104         if filecount == 50:
0105             stringS  = "])\n\n"
0106             file1.write(stringS)
0107             filecount = 0
0108             extendOpen = 0
0109
0110     if extendOpen == 1:
0111         stringS  =           "])\n\n"
0112         file1.write(stringS)
0113
0114     stringS  =           "process.source = cms.Source(\"PoolSource\",\n"
0115     stringS  = stringS + "         fileNames = readFiles,\n"
0116     stringS  = stringS + "         secondaryFileNames = secFiles\n"
0117     stringS  = stringS + ")\n"
0118     file1.write(stringS)
0119     file1.close()
0120
0121
0122 def defineOptions():
0123     parser = OptionParser()
0124     parser.add_option("-w", "--workspace",
0125                       dest="workspaceName",
0126                       default="GLOBAL",
0127                       help="define workspace: GLOBAL TRACKER ")
0128
0129     parser.add_option("-r", "--regexp",
0130                       dest="regexp",
0131                       type="string",
0132                       default='groupName : LIKE %Collisions12% , runNumber : = 190000 ',
0133                       help=" \"{runNumber} >= 148127 and {runNumber} < 148128 \" ")
0134
0135     parser.add_option("-d", "--datasetPath",
0136                       dest="dataset", \
0137                       default="/MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO",
0138                       help="For example : --datasetPath /MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO")
0139
0140     parser.add_option("-s", "--site",
0141                       dest="site",
0142                       default="T2_CH_CAF",
0143                       help="For example : site T2_CH_CAF")
0144
0145     parser.add_option("-i", "--info",
0146                       action="store_true",
0147                       dest="info",
0148                       default=False,
0149                       help="printout the column names on which it's possible to cut")
0150
0151     (options, args) = parser.parse_args()
0152     if len(sys.argv) == 1:
0153         print("\nUsage: %s --help"%sys.argv[0])
0154         sys.exit(0)
0155
0156     return options
0157
0158
0159 def serverQuery(workspaceName,regexp):
0160
0161     # get handler to RR XML-RPC server
0162     server = xmlrpclib.ServerProxy('http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
0163     if DEBUG:
0164         print(lineno(), regexp)
0165     data = server.RunDatasetTable.export(workspaceName,'xml_all' ,regexp)
0166     return data
0167
0168 #----------------------------------------------------
0169
0170 def getText(nodelist):
0171     rc = ""
0172     for node in nodelist:
0173         if node.nodeType == node.TEXT_NODE:
0174             rc = rc + node.data
0175     return rc
0176
0177 def getElement(obj,name):
0178     return obj.getElementsByTagName(name)
0179
0180 def printObj(obj,name):
0181     return getText(getElement(obj,name)[0].childNodes)
0182
0183
0184 def getData(doc,options,dataset,site):
0185     if DEBUG:
0186         print(lineno(), 'getData')
0187     server = xmlrpclib.ServerProxy('http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
0188     runs = getElement(doc,'RUN')
0189     txtLongData=""
0190     txtkey=""
0191     lista=[]
0192
0193     sep="\t"
0194
0195     for run in runs:
0196         txtrun=printObj(run,'NUMBER') + sep + printObj(run,'HLTKEY')
0197         txtLongData+= txtrun + sep + "\n"
0198     for run in runs:
0199         test=printObj(run,'HLTKEY')
0200         if not (test in lista):
0201             lista.append(test)
0202
0203         file2=open("lista_key.txt",'w')
0204         for pkey in range(len(lista)):
0205             pwkey = lista[pkey] +"\n"
0206             file2.write(pwkey)
0207             if DEBUG:
0208                 print(lineno(),  lista[pkey])
0209
0210         file2.close()
0211
0212     for i in range(len(lista)):
0213         if DEBUG:
0214             print(lineno(), lista[i])
0215         nameDBS=""
0216         nameDBS=str(i)+".data"
0217         name=""
0218         name=str(i)+".json"
0219         nameLumi=""
0220         nameLumi=str(i)+".lumi"
0221         file1 = open( name ,'w')
0222         listaDBS = []
0223         if DEBUG:
0224             print(lineno(), nameDBS)
0225         for run in runs:
0226             key=printObj(run,'HLTKEY')
0227             if (key == lista[i]):
0228                 print("running......", key)
0229                 if DEBUG:
0230                     print(lineno(), printObj(run,'NUMBER'))
0231                 txtruns = "{runNumber} >= " + printObj(run,'NUMBER') +  " and {runNumber} <= " + str(int(printObj(run,'NUMBER')))
0232                 txtriv = txtruns + " and {cmpDcsBPix} = 1  and {cmpDcsFPix} = 1 and {cmpDcsTibTid} = 1 and {cmpDcsTob} = 1 and {cmpDcsTecM} = 1 and {cmpDcsTecP} = 1"
0233                 lumirun = server.RunLumiSectionRangeTable.export('GLOBAL', 'json',txtriv)
0234                 if DEBUG:
0235                     print(lineno(), lumirun)
0236                   ###dbs file
0237                 if lumirun:
0238                     file = DBSquery(dataset,site,str(printObj(run,'NUMBER')))
0239 ##                 if DEBUG:
0240 ##                     print lineno(), file
0241                 if (file != "") :
0242                     for uno in file:
0243                         stringDBS = {}
0244                         stringDBS = uno['LogicalFileName']
0245                         listaDBS    += [stringDBS]
0246                     if DEBUG:
0247                         print(lineno(), lumirun)
0248                     comp="{}"
0249                     if (lumirun == comp):
0250                         print("LUMI ZERO")
0251                     else:
0252                         file1.write(lumirun)
0253
0254         file1.close()
0255         string=""
0256         string="sed -i 's/}{/, /g'"
0257         string2=""
0258         string2= string + " " + name
0259         os.system(string2)
0260         printLumi(name,nameLumi)
0261         os.system("sed -i 's/\//_/g' lista_key.txt")
0262         listaDBS2 =[]
0263         for rootLSF in listaDBS:
0264             if not (rootLSF in listaDBS2):
0265                 listaDBS2.append(rootLSF)
0266         makecff(listaDBS2,nameDBS)
0267
0268     return txtLongData
0269
0270 #---------------------------------------------
0271
0272 def extractData(mode,reg,dataset,site,options):
0273     doc = minidom.parseString(serverQuery(mode,reg))
0274     return getData(doc,options,dataset,site)
0275
0276 def getRegExp(regexp):
0277     items = regexp.split(',')
0278     dd = {}
0279     for item in items:
0280         key,value = item.split(':')
0281         dd[key.replace(' ','')] = value
0282     return dd
0283
0284
0285 #---------------------------------------------MAIN
0286
0287 options = defineOptions()
0288 data=extractData(options.workspaceName,options.regexp,options.dataset,options.site,options)
0289