CommonAlignmentProducer/scripts/AlCaHLTBitMon_QueryRunRegistry.py

0001 #!/usr/bin/env python3
0002
0003 #-----------------------------------------------------
0004 # original author: Andrea Lucaroni
0005 # Revision:        $Revision: 1.3 $
0006 # Last update:     $Date: 2012/05/22 08:22:04 $
0007 # by:              $Author: taroni $
0008 #-----------------------------------------------------
0009
0010 from __future__ import print_function
0011 from xml.dom import minidom
0012 import re
0013 import json
0014 import os
0015 import stat
0016 import sys
0017
0018 #include DBS
0019 from DBSAPI.dbsApiException import DbsException
0020 import DBSAPI.dbsApi
0021 from DBSAPI.dbsApiException import *
0022
0023 # include XML-RPC client library
0024 # RR API uses XML-RPC webservices interface for data access
0025 import xmlrpclib
0026
0027 import array
0028 import pickle as pk
0029
0030 from optparse import OptionParser
0031 #####DEBUG
0032 DEBUG=0
0033 import inspect
0034
0035 def lineno():
0036     """Returns the current line number in our program."""
0037     return inspect.currentframe().f_back.f_lineno
0038
0039 #size file
0040 def filesize1(n):
0041     info = os.stat(n)
0042     sz = info[stat.ST_SIZE]
0043     return sz
0044
0045 ### lumiCalc
0046 def printLumi(file,namefile):
0047     if(filesize1(file) != 0):
0048         string= "lumiCalc2.py -c frontier://LumiCalc/CMS_LUMI_PROD -i "
0049         string1= " --nowarning overview >"
0050         string2= string + file + string1 + namefile
0051         data = os.system(string2)
0052     else:
0053         data = ""
0054         print("0 lumi are not avaible")
0055     return data
0056
0057 ###file  dbs
0058 def DBSquery(dataset,site,run):
0059
0060     url = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
0061     if DEBUG:
0062         print(lineno())
0063     args = {}
0064     args['url']     = url
0065     args['level']   = 'CRITICAL'
0066     if DEBUG:
0067         print(lineno())
0068     api = DBSAPI.dbsApi.DbsApi(args)
0069     if DEBUG:
0070         print(lineno())
0071     try:
0072         files = api.listFiles(path=dataset,tier_list =site,runNumber=run)
0073     except DbsApiException as ex:
0074         print("Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() ))
0075         files = ""
0076         if ex.getErrorCode() not in (None, ""):
0077             print("DBS Exception Error Code: ", ex.getErrorCode())
0078     if DEBUG:
0079         print(lineno())
0080     return files
0081
0082 ###file cff data
0083 def makecff(file_list,namefile):
0084     file1 = open(namefile ,'w')
0085     stringS  =           "process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) )\n"
0086     stringS  = stringS + "readFiles = cms.untracked.vstring()\n"
0087     stringS  = stringS + "secFiles = cms.untracked.vstring()\n"
0088     stringS  = stringS + "\n"
0089     file1.write(stringS)
0090
0091     filecount = 0
0092     extendOpen = 0
0093     for filename in file_list:
0094
0095         if extendOpen == 0:
0096             stringS  = "readFiles.extend([\n"
0097             file1.write(stringS)
0098             extendOpen = 1
0099
0100         stringS  =           "     '"
0101         stringS  = stringS + str(filename)
0102         stringS  = stringS + "',\n"
0103         file1.write(stringS)
0104         filecount = filecount + 1
0105         if filecount == 50:
0106             stringS  = "])\n\n"
0107             file1.write(stringS)
0108             filecount = 0
0109             extendOpen = 0
0110
0111     if extendOpen == 1:
0112         stringS  =           "])\n\n"
0113         file1.write(stringS)
0114
0115     stringS  =           "process.source = cms.Source(\"PoolSource\",\n"
0116     stringS  = stringS + "         fileNames = readFiles,\n"
0117     stringS  = stringS + "         secondaryFileNames = secFiles\n"
0118     stringS  = stringS + ")\n"
0119     file1.write(stringS)
0120     file1.close()
0121
0122
0123 def defineOptions():
0124     parser = OptionParser()
0125     parser.add_option("-w", "--workspace",
0126                       dest="workspaceName",
0127                       default="GLOBAL",
0128                       help="define workspace: GLOBAL TRACKER ")
0129
0130     parser.add_option("-r", "--regexp",
0131                       dest="regexp",
0132                       type="string",
0133                       default='groupName : LIKE %Collisions12% , runNumber : = 190000 ',
0134                       help=" \"{runNumber} >= 148127 and {runNumber} < 148128 \" ")
0135
0136     parser.add_option("-d", "--datasetPath",
0137                       dest="dataset", \
0138                       default="/MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO",
0139                       help="For example : --datasetPath /MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO")
0140
0141     parser.add_option("-s", "--site",
0142                       dest="site",
0143                       default="T2_CH_CAF",
0144                       help="For example : site T2_CH_CAF")
0145
0146     parser.add_option("-i", "--info",
0147                       action="store_true",
0148                       dest="info",
0149                       default=False,
0150                       help="printout the column names on which it's possible to cut")
0151
0152     (options, args) = parser.parse_args()
0153     if len(sys.argv) == 1:
0154         print("\nUsage: %s --help"%sys.argv[0])
0155         sys.exit(0)
0156
0157     return options
0158
0159
0160 def serverQuery(workspaceName,regexp):
0161
0162     # get handler to RR XML-RPC server
0163     server = xmlrpclib.ServerProxy('http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
0164     if DEBUG:
0165         print(lineno(), regexp)
0166     data = server.RunDatasetTable.export(workspaceName,'xml_all' ,regexp)
0167     return data
0168
0169 #----------------------------------------------------
0170
0171 def getText(nodelist):
0172     rc = ""
0173     for node in nodelist:
0174         if node.nodeType == node.TEXT_NODE:
0175             rc = rc + node.data
0176     return rc
0177
0178 def getElement(obj,name):
0179     return obj.getElementsByTagName(name)
0180
0181 def printObj(obj,name):
0182     return getText(getElement(obj,name)[0].childNodes)
0183
0184
0185 def getData(doc,options,dataset,site):
0186     if DEBUG:
0187         print(lineno(), 'getData')
0188     server = xmlrpclib.ServerProxy('http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
0189     runs = getElement(doc,'RUN')
0190     txtLongData=""
0191     txtkey=""
0192     lista=[]
0193
0194     sep="\t"
0195
0196     for run in runs:
0197         txtrun=printObj(run,'NUMBER') + sep + printObj(run,'HLTKEY')
0198         txtLongData+= txtrun + sep + "\n"
0199     for run in runs:
0200         test=printObj(run,'HLTKEY')
0201         if not (test in lista):
0202             lista.append(test)
0203
0204         file2=open("lista_key.txt",'w')
0205         for pkey in range(len(lista)):
0206             pwkey = lista[pkey] +"\n"
0207             file2.write(pwkey)
0208             if DEBUG:
0209                 print(lineno(),  lista[pkey])
0210
0211         file2.close()
0212
0213     for i in range(len(lista)):
0214         if DEBUG:
0215             print(lineno(), lista[i])
0216         nameDBS=""
0217         nameDBS=str(i)+".data"
0218         name=""
0219         name=str(i)+".json"
0220         nameLumi=""
0221         nameLumi=str(i)+".lumi"
0222         file1 = open( name ,'w')
0223         listaDBS = []
0224         if DEBUG:
0225             print(lineno(), nameDBS)
0226         for run in runs:
0227             key=printObj(run,'HLTKEY')
0228             if (key == lista[i]):
0229                 print("running......", key)
0230                 if DEBUG:
0231                     print(lineno(), printObj(run,'NUMBER'))
0232                 txtruns = "{runNumber} >= " + printObj(run,'NUMBER') +  " and {runNumber} <= " + str(int(printObj(run,'NUMBER')))
0233                 txtriv = txtruns + " and {cmpDcsBPix} = 1  and {cmpDcsFPix} = 1 and {cmpDcsTibTid} = 1 and {cmpDcsTob} = 1 and {cmpDcsTecM} = 1 and {cmpDcsTecP} = 1"
0234                 lumirun = server.RunLumiSectionRangeTable.export('GLOBAL', 'json',txtriv)
0235                 if DEBUG:
0236                     print(lineno(), lumirun)
0237                   ###dbs file
0238                 if lumirun:
0239                     file = DBSquery(dataset,site,str(printObj(run,'NUMBER')))
0240 ##                 if DEBUG:
0241 ##                     print lineno(), file
0242                 if (file != "") :
0243                     for uno in file:
0244                         stringDBS = {}
0245                         stringDBS = uno['LogicalFileName']
0246                         listaDBS    += [stringDBS]
0247                     if DEBUG:
0248                         print(lineno(), lumirun)
0249                     comp="{}"
0250                     if (lumirun == comp):
0251                         print("LUMI ZERO")
0252                     else:
0253                         file1.write(lumirun)
0254
0255         file1.close()
0256         string=""
0257         string="sed -i 's/}{/, /g'"
0258         string2=""
0259         string2= string + " " + name
0260         os.system(string2)
0261         printLumi(name,nameLumi)
0262         os.system("sed -i 's/\//_/g' lista_key.txt")
0263         listaDBS2 =[]
0264         for rootLSF in listaDBS:
0265             if not (rootLSF in listaDBS2):
0266                 listaDBS2.append(rootLSF)
0267         makecff(listaDBS2,nameDBS)
0268
0269     return txtLongData
0270
0271 #---------------------------------------------
0272
0273 def extractData(mode,reg,dataset,site,options):
0274     doc = minidom.parseString(serverQuery(mode,reg))
0275     return getData(doc,options,dataset,site)
0276
0277 def getRegExp(regexp):
0278     items = regexp.split(',')
0279     dd = {}
0280     for item in items:
0281         key,value = item.split(':')
0282         dd[key.replace(' ','')] = value
0283     return dd
0284
0285
0286 #---------------------------------------------MAIN
0287
0288 options = defineOptions()
0289 data=extractData(options.workspaceName,options.regexp,options.dataset,options.site,options)
0290