File indexing completed on 2023-03-17 10:39:02
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010 from __future__ import print_function
0011 from xml.dom import minidom
0012 import re
0013 import json
0014 import os
0015 import stat
0016 import sys
0017
0018
0019 from DBSAPI.dbsApiException import DbsException
0020 import DBSAPI.dbsApi
0021 from DBSAPI.dbsApiException import *
0022
0023
0024
0025 import xmlrpclib
0026
0027 import array
0028 import pickle as pk
0029
0030 from optparse import OptionParser
0031
0032 DEBUG=0
0033 import inspect
0034
0035 def lineno():
0036 """Returns the current line number in our program."""
0037 return inspect.currentframe().f_back.f_lineno
0038
0039
0040 def filesize1(n):
0041 info = os.stat(n)
0042 sz = info[stat.ST_SIZE]
0043 return sz
0044
0045
0046 def printLumi(file,namefile):
0047 if(filesize1(file) != 0):
0048 string= "lumiCalc2.py -c frontier://LumiCalc/CMS_LUMI_PROD -i "
0049 string1= " --nowarning overview >"
0050 string2= string + file + string1 + namefile
0051 data = os.system(string2)
0052 else:
0053 data = ""
0054 print("0 lumi are not avaible")
0055 return data
0056
0057
0058 def DBSquery(dataset,site,run):
0059
0060 url = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
0061 if DEBUG:
0062 print(lineno())
0063 args = {}
0064 args['url'] = url
0065 args['level'] = 'CRITICAL'
0066 if DEBUG:
0067 print(lineno())
0068 api = DBSAPI.dbsApi.DbsApi(args)
0069 if DEBUG:
0070 print(lineno())
0071 try:
0072 files = api.listFiles(path=dataset,tier_list =site,runNumber=run)
0073 except DbsApiException as ex:
0074 print("Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() ))
0075 files = ""
0076 if ex.getErrorCode() not in (None, ""):
0077 print("DBS Exception Error Code: ", ex.getErrorCode())
0078 if DEBUG:
0079 print(lineno())
0080 return files
0081
0082
0083 def makecff(file_list,namefile):
0084 file1 = open(namefile ,'w')
0085 stringS = "process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) )\n"
0086 stringS = stringS + "readFiles = cms.untracked.vstring()\n"
0087 stringS = stringS + "secFiles = cms.untracked.vstring()\n"
0088 stringS = stringS + "\n"
0089 file1.write(stringS)
0090
0091 filecount = 0
0092 extendOpen = 0
0093 for filename in file_list:
0094
0095 if extendOpen == 0:
0096 stringS = "readFiles.extend([\n"
0097 file1.write(stringS)
0098 extendOpen = 1
0099
0100 stringS = " '"
0101 stringS = stringS + str(filename)
0102 stringS = stringS + "',\n"
0103 file1.write(stringS)
0104 filecount = filecount + 1
0105 if filecount == 50:
0106 stringS = "])\n\n"
0107 file1.write(stringS)
0108 filecount = 0
0109 extendOpen = 0
0110
0111 if extendOpen == 1:
0112 stringS = "])\n\n"
0113 file1.write(stringS)
0114
0115 stringS = "process.source = cms.Source(\"PoolSource\",\n"
0116 stringS = stringS + " fileNames = readFiles,\n"
0117 stringS = stringS + " secondaryFileNames = secFiles\n"
0118 stringS = stringS + ")\n"
0119 file1.write(stringS)
0120 file1.close()
0121
0122
0123 def defineOptions():
0124 parser = OptionParser()
0125 parser.add_option("-w", "--workspace",
0126 dest="workspaceName",
0127 default="GLOBAL",
0128 help="define workspace: GLOBAL TRACKER ")
0129
0130 parser.add_option("-r", "--regexp",
0131 dest="regexp",
0132 type="string",
0133 default='groupName : LIKE %Collisions12% , runNumber : = 190000 ',
0134 help=" \"{runNumber} >= 148127 and {runNumber} < 148128 \" ")
0135
0136 parser.add_option("-d", "--datasetPath",
0137 dest="dataset", \
0138 default="/MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO",
0139 help="For example : --datasetPath /MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO")
0140
0141 parser.add_option("-s", "--site",
0142 dest="site",
0143 default="T2_CH_CAF",
0144 help="For example : site T2_CH_CAF")
0145
0146 parser.add_option("-i", "--info",
0147 action="store_true",
0148 dest="info",
0149 default=False,
0150 help="printout the column names on which it's possible to cut")
0151
0152 (options, args) = parser.parse_args()
0153 if len(sys.argv) == 1:
0154 print("\nUsage: %s --help"%sys.argv[0])
0155 sys.exit(0)
0156
0157 return options
0158
0159
0160 def serverQuery(workspaceName,regexp):
0161
0162
0163 server = xmlrpclib.ServerProxy('http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
0164 if DEBUG:
0165 print(lineno(), regexp)
0166 data = server.RunDatasetTable.export(workspaceName,'xml_all' ,regexp)
0167 return data
0168
0169
0170
0171 def getText(nodelist):
0172 rc = ""
0173 for node in nodelist:
0174 if node.nodeType == node.TEXT_NODE:
0175 rc = rc + node.data
0176 return rc
0177
0178 def getElement(obj,name):
0179 return obj.getElementsByTagName(name)
0180
0181 def printObj(obj,name):
0182 return getText(getElement(obj,name)[0].childNodes)
0183
0184
0185 def getData(doc,options,dataset,site):
0186 if DEBUG:
0187 print(lineno(), 'getData')
0188 server = xmlrpclib.ServerProxy('http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
0189 runs = getElement(doc,'RUN')
0190 txtLongData=""
0191 txtkey=""
0192 lista=[]
0193
0194 sep="\t"
0195
0196 for run in runs:
0197 txtrun=printObj(run,'NUMBER') + sep + printObj(run,'HLTKEY')
0198 txtLongData+= txtrun + sep + "\n"
0199 for run in runs:
0200 test=printObj(run,'HLTKEY')
0201 if not (test in lista):
0202 lista.append(test)
0203
0204 file2=open("lista_key.txt",'w')
0205 for pkey in range(len(lista)):
0206 pwkey = lista[pkey] +"\n"
0207 file2.write(pwkey)
0208 if DEBUG:
0209 print(lineno(), lista[pkey])
0210
0211 file2.close()
0212
0213 for i in range(len(lista)):
0214 if DEBUG:
0215 print(lineno(), lista[i])
0216 nameDBS=""
0217 nameDBS=str(i)+".data"
0218 name=""
0219 name=str(i)+".json"
0220 nameLumi=""
0221 nameLumi=str(i)+".lumi"
0222 file1 = open( name ,'w')
0223 listaDBS = []
0224 if DEBUG:
0225 print(lineno(), nameDBS)
0226 for run in runs:
0227 key=printObj(run,'HLTKEY')
0228 if (key == lista[i]):
0229 print("running......", key)
0230 if DEBUG:
0231 print(lineno(), printObj(run,'NUMBER'))
0232 txtruns = "{runNumber} >= " + printObj(run,'NUMBER') + " and {runNumber} <= " + str(int(printObj(run,'NUMBER')))
0233 txtriv = txtruns + " and {cmpDcsBPix} = 1 and {cmpDcsFPix} = 1 and {cmpDcsTibTid} = 1 and {cmpDcsTob} = 1 and {cmpDcsTecM} = 1 and {cmpDcsTecP} = 1"
0234 lumirun = server.RunLumiSectionRangeTable.export('GLOBAL', 'json',txtriv)
0235 if DEBUG:
0236 print(lineno(), lumirun)
0237
0238 if lumirun:
0239 file = DBSquery(dataset,site,str(printObj(run,'NUMBER')))
0240
0241
0242 if (file != "") :
0243 for uno in file:
0244 stringDBS = {}
0245 stringDBS = uno['LogicalFileName']
0246 listaDBS += [stringDBS]
0247 if DEBUG:
0248 print(lineno(), lumirun)
0249 comp="{}"
0250 if (lumirun == comp):
0251 print("LUMI ZERO")
0252 else:
0253 file1.write(lumirun)
0254
0255 file1.close()
0256 string=""
0257 string="sed -i 's/}{/, /g'"
0258 string2=""
0259 string2= string + " " + name
0260 os.system(string2)
0261 printLumi(name,nameLumi)
0262 os.system("sed -i 's/\//_/g' lista_key.txt")
0263 listaDBS2 =[]
0264 for rootLSF in listaDBS:
0265 if not (rootLSF in listaDBS2):
0266 listaDBS2.append(rootLSF)
0267 makecff(listaDBS2,nameDBS)
0268
0269 return txtLongData
0270
0271
0272
0273 def extractData(mode,reg,dataset,site,options):
0274 doc = minidom.parseString(serverQuery(mode,reg))
0275 return getData(doc,options,dataset,site)
0276
0277 def getRegExp(regexp):
0278 items = regexp.split(',')
0279 dd = {}
0280 for item in items:
0281 key,value = item.split(':')
0282 dd[key.replace(' ','')] = value
0283 return dd
0284
0285
0286
0287
0288 options = defineOptions()
0289 data=extractData(options.workspaceName,options.regexp,options.dataset,options.site,options)
0290