File indexing completed on 2024-11-25 02:29:02
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010 from xml.dom import minidom
0011 import re
0012 import json
0013 import os
0014 import stat
0015 import sys
0016
0017
0018 from DBSAPI.dbsApiException import DbsException
0019 import DBSAPI.dbsApi
0020 from DBSAPI.dbsApiException import *
0021
0022
0023
0024 import xmlrpclib
0025
0026 import array
0027 import pickle as pk
0028
0029 from optparse import OptionParser
0030
0031 DEBUG=0
0032 import inspect
0033
0034 def lineno():
0035 """Returns the current line number in our program."""
0036 return inspect.currentframe().f_back.f_lineno
0037
0038
0039 def filesize1(n):
0040 info = os.stat(n)
0041 sz = info[stat.ST_SIZE]
0042 return sz
0043
0044
0045 def printLumi(file,namefile):
0046 if(filesize1(file) != 0):
0047 string= "lumiCalc2.py -c frontier://LumiCalc/CMS_LUMI_PROD -i "
0048 string1= " --nowarning overview >"
0049 string2= string + file + string1 + namefile
0050 data = os.system(string2)
0051 else:
0052 data = ""
0053 print("0 lumi are not avaible")
0054 return data
0055
0056
0057 def DBSquery(dataset,site,run):
0058
0059 url = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
0060 if DEBUG:
0061 print(lineno())
0062 args = {}
0063 args['url'] = url
0064 args['level'] = 'CRITICAL'
0065 if DEBUG:
0066 print(lineno())
0067 api = DBSAPI.dbsApi.DbsApi(args)
0068 if DEBUG:
0069 print(lineno())
0070 try:
0071 files = api.listFiles(path=dataset,tier_list =site,runNumber=run)
0072 except DbsApiException as ex:
0073 print("Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() ))
0074 files = ""
0075 if ex.getErrorCode() not in (None, ""):
0076 print("DBS Exception Error Code: ", ex.getErrorCode())
0077 if DEBUG:
0078 print(lineno())
0079 return files
0080
0081
0082 def makecff(file_list,namefile):
0083 file1 = open(namefile ,'w')
0084 stringS = "process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) )\n"
0085 stringS = stringS + "readFiles = cms.untracked.vstring()\n"
0086 stringS = stringS + "secFiles = cms.untracked.vstring()\n"
0087 stringS = stringS + "\n"
0088 file1.write(stringS)
0089
0090 filecount = 0
0091 extendOpen = 0
0092 for filename in file_list:
0093
0094 if extendOpen == 0:
0095 stringS = "readFiles.extend([\n"
0096 file1.write(stringS)
0097 extendOpen = 1
0098
0099 stringS = " '"
0100 stringS = stringS + str(filename)
0101 stringS = stringS + "',\n"
0102 file1.write(stringS)
0103 filecount = filecount + 1
0104 if filecount == 50:
0105 stringS = "])\n\n"
0106 file1.write(stringS)
0107 filecount = 0
0108 extendOpen = 0
0109
0110 if extendOpen == 1:
0111 stringS = "])\n\n"
0112 file1.write(stringS)
0113
0114 stringS = "process.source = cms.Source(\"PoolSource\",\n"
0115 stringS = stringS + " fileNames = readFiles,\n"
0116 stringS = stringS + " secondaryFileNames = secFiles\n"
0117 stringS = stringS + ")\n"
0118 file1.write(stringS)
0119 file1.close()
0120
0121
0122 def defineOptions():
0123 parser = OptionParser()
0124 parser.add_option("-w", "--workspace",
0125 dest="workspaceName",
0126 default="GLOBAL",
0127 help="define workspace: GLOBAL TRACKER ")
0128
0129 parser.add_option("-r", "--regexp",
0130 dest="regexp",
0131 type="string",
0132 default='groupName : LIKE %Collisions12% , runNumber : = 190000 ',
0133 help=" \"{runNumber} >= 148127 and {runNumber} < 148128 \" ")
0134
0135 parser.add_option("-d", "--datasetPath",
0136 dest="dataset", \
0137 default="/MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO",
0138 help="For example : --datasetPath /MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO")
0139
0140 parser.add_option("-s", "--site",
0141 dest="site",
0142 default="T2_CH_CAF",
0143 help="For example : site T2_CH_CAF")
0144
0145 parser.add_option("-i", "--info",
0146 action="store_true",
0147 dest="info",
0148 default=False,
0149 help="printout the column names on which it's possible to cut")
0150
0151 (options, args) = parser.parse_args()
0152 if len(sys.argv) == 1:
0153 print("\nUsage: %s --help"%sys.argv[0])
0154 sys.exit(0)
0155
0156 return options
0157
0158
0159 def serverQuery(workspaceName,regexp):
0160
0161
0162 server = xmlrpclib.ServerProxy('http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
0163 if DEBUG:
0164 print(lineno(), regexp)
0165 data = server.RunDatasetTable.export(workspaceName,'xml_all' ,regexp)
0166 return data
0167
0168
0169
0170 def getText(nodelist):
0171 rc = ""
0172 for node in nodelist:
0173 if node.nodeType == node.TEXT_NODE:
0174 rc = rc + node.data
0175 return rc
0176
0177 def getElement(obj,name):
0178 return obj.getElementsByTagName(name)
0179
0180 def printObj(obj,name):
0181 return getText(getElement(obj,name)[0].childNodes)
0182
0183
0184 def getData(doc,options,dataset,site):
0185 if DEBUG:
0186 print(lineno(), 'getData')
0187 server = xmlrpclib.ServerProxy('http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
0188 runs = getElement(doc,'RUN')
0189 txtLongData=""
0190 txtkey=""
0191 lista=[]
0192
0193 sep="\t"
0194
0195 for run in runs:
0196 txtrun=printObj(run,'NUMBER') + sep + printObj(run,'HLTKEY')
0197 txtLongData+= txtrun + sep + "\n"
0198 for run in runs:
0199 test=printObj(run,'HLTKEY')
0200 if not (test in lista):
0201 lista.append(test)
0202
0203 file2=open("lista_key.txt",'w')
0204 for pkey in range(len(lista)):
0205 pwkey = lista[pkey] +"\n"
0206 file2.write(pwkey)
0207 if DEBUG:
0208 print(lineno(), lista[pkey])
0209
0210 file2.close()
0211
0212 for i in range(len(lista)):
0213 if DEBUG:
0214 print(lineno(), lista[i])
0215 nameDBS=""
0216 nameDBS=str(i)+".data"
0217 name=""
0218 name=str(i)+".json"
0219 nameLumi=""
0220 nameLumi=str(i)+".lumi"
0221 file1 = open( name ,'w')
0222 listaDBS = []
0223 if DEBUG:
0224 print(lineno(), nameDBS)
0225 for run in runs:
0226 key=printObj(run,'HLTKEY')
0227 if (key == lista[i]):
0228 print("running......", key)
0229 if DEBUG:
0230 print(lineno(), printObj(run,'NUMBER'))
0231 txtruns = "{runNumber} >= " + printObj(run,'NUMBER') + " and {runNumber} <= " + str(int(printObj(run,'NUMBER')))
0232 txtriv = txtruns + " and {cmpDcsBPix} = 1 and {cmpDcsFPix} = 1 and {cmpDcsTibTid} = 1 and {cmpDcsTob} = 1 and {cmpDcsTecM} = 1 and {cmpDcsTecP} = 1"
0233 lumirun = server.RunLumiSectionRangeTable.export('GLOBAL', 'json',txtriv)
0234 if DEBUG:
0235 print(lineno(), lumirun)
0236
0237 if lumirun:
0238 file = DBSquery(dataset,site,str(printObj(run,'NUMBER')))
0239
0240
0241 if (file != "") :
0242 for uno in file:
0243 stringDBS = {}
0244 stringDBS = uno['LogicalFileName']
0245 listaDBS += [stringDBS]
0246 if DEBUG:
0247 print(lineno(), lumirun)
0248 comp="{}"
0249 if (lumirun == comp):
0250 print("LUMI ZERO")
0251 else:
0252 file1.write(lumirun)
0253
0254 file1.close()
0255 string=""
0256 string="sed -i 's/}{/, /g'"
0257 string2=""
0258 string2= string + " " + name
0259 os.system(string2)
0260 printLumi(name,nameLumi)
0261 os.system("sed -i 's/\//_/g' lista_key.txt")
0262 listaDBS2 =[]
0263 for rootLSF in listaDBS:
0264 if not (rootLSF in listaDBS2):
0265 listaDBS2.append(rootLSF)
0266 makecff(listaDBS2,nameDBS)
0267
0268 return txtLongData
0269
0270
0271
0272 def extractData(mode,reg,dataset,site,options):
0273 doc = minidom.parseString(serverQuery(mode,reg))
0274 return getData(doc,options,dataset,site)
0275
0276 def getRegExp(regexp):
0277 items = regexp.split(',')
0278 dd = {}
0279 for item in items:
0280 key,value = item.split(':')
0281 dd[key.replace(' ','')] = value
0282 return dd
0283
0284
0285
0286
0287 options = defineOptions()
0288 data=extractData(options.workspaceName,options.regexp,options.dataset,options.site,options)
0289