PyReleaseValidation/python/MatrixUtil.py

0001 import os
0002 class Matrix(dict):
0003     def __setitem__(self,key,value):
0004         if key in self:
0005             print("ERROR in Matrix")
0006             print("overwriting",key,"not allowed")
0007         else:
0008             self.update({float(key):WF(float(key),value)})
0009
0010     def addOverride(self,key,override):
0011         self[key].addOverride(override)
0012
0013 #the class to collect all possible steps
0014 class Steps(dict):
0015     def __setitem__(self,key,value):
0016         if key in self:
0017             print("ERROR in Step")
0018             print("overwriting",key,"not allowed")
0019             import sys
0020             sys.exit(-9)
0021         else:
0022             self.update({key:value})
0023             # make the python file named <step>.py
0024             #if not '--python' in value:                self[key].update({'--python':'%s.py'%(key,)})
0025
0026     def overwrite(self,keypair):
0027         value=self[keypair[1]]
0028         self.update({keypair[0]:value})
0029
0030 class WF(list):
0031     def __init__(self,n,l):
0032         self.extend(l)
0033         self.num=n
0034         #the actual steps of this WF
0035         self.steps=[]
0036         self.overrides={}
0037     def addOverride(self,overrides):
0038         self.overrides=overrides
0039
0040     def interpret(self,stepsDict):
0041         for s in self:
0042             print('steps',s,stepsDict[s])
0043             steps.append(stepsDict[s])
0044
0045
0046
0047 def expandLsInterval(lumis):
0048     return range(lumis[0],(lumis[1]+1))
0049
0050 from DPGAnalysis.Skims.golden_json_2015 import *
0051 jsonFile2015 = findFileInPath("DPGAnalysis/Skims/data/Cert_13TeV_16Dec2015ReReco_Collisions15_25ns_50ns_JSON.txt")
0052 jsonFile2016 = findFileInPath("DPGAnalysis/Skims/data/Cert_271036-274240_13TeV_PromptReco_Collisions16_JSON.txt")
0053
0054 import json
0055 with open(jsonFile2015) as data_file:
0056     data_json2015 = json.load(data_file)
0057
0058 with open(jsonFile2016) as data_file:
0059     data_json2016 = json.load(data_file)
0060
0061 # return a portion of the 2015 golden json
0062 # LS for a full run by default; otherwise a subset of which you determined the size
0063 def selectedLS(list_runs=[],maxNum=-1,l_json=data_json2015):
0064     # print "maxNum is %s"%(maxNum)
0065     if not isinstance(list_runs[0], int):
0066         print("ERROR: list_runs must be a list of integers")
0067         return None
0068     local_dict = {}
0069     ls_count = 0
0070
0071     for run in list_runs:
0072         if str(run) in l_json.keys():
0073             # print "run %s is there"%(run)
0074             runNumber = run
0075             # print "Doing lumi-section selection for run %s: "%(run)
0076             for LSsegment in l_json[str(run)] :
0077                 # print LSsegment
0078                 ls_count += (LSsegment[-1] - LSsegment[0] + 1)
0079                 if (ls_count > maxNum) & (maxNum != -1):
0080                     break
0081                     # return local_dict
0082                 if runNumber in local_dict.keys():
0083                     local_dict[runNumber].append(LSsegment)
0084                 else:
0085                     local_dict[runNumber] = [LSsegment]
0086                 # print "total LS so far  %s    -   grow %s"%(ls_count,local_dict)
0087             #local_dict[runNumber] = [1,2,3]
0088         else:
0089             print("run %s is NOT present in json %s\n\n"%(run, l_json))
0090         # print "++    %s"%(local_dict)
0091
0092     if ( len(local_dict) > 0 ) :
0093         return local_dict
0094     else :
0095         print("No luminosity section interval passed the json and your selection; returning None")
0096         return None
0097
0098 # print "\n\n\n THIS IS WHAT I RETURN: %s \n\n"%( selectedLS([251244,251251]) )
0099
0100
0101
0102
0103 InputInfoNDefault=2000000
0104 class InputInfo(object):
0105     def __init__(self,dataSet,dataSetParent='',label='',run=[],ls={},files=1000,events=InputInfoNDefault,split=10,location='CAF',ib_blacklist=None,ib_block=None,skimEvents=False) :
0106         self.run = run
0107         self.ls = ls
0108         self.files = files
0109         self.events = events
0110         self.location = location
0111         self.label = label
0112         self.dataSet = dataSet
0113         self.split = split
0114         self.ib_blacklist = ib_blacklist
0115         self.ib_block = ib_block
0116         self.dataSetParent = dataSetParent
0117         self.skimEvents = skimEvents
0118
0119     def das(self, das_options, dataset):
0120         if not self.skimEvents and (len(self.run) != 0 or self.ls):
0121             queries = self.queries(dataset)
0122             if len(self.run) != 0:
0123                 command = ";".join(["dasgoclient %s --query '%s'" % (das_options, query) for query in queries])
0124             else:
0125               lumis = self.lumis()
0126               commands = []
0127               while queries:
0128                     commands.append("dasgoclient %s --query 'lumi,%s' --format json | das-selected-lumis.py %s " % (das_options, queries.pop(), lumis.pop()))
0129               command = ";".join(commands)
0130             command = "({0})".format(command)
0131         elif not self.skimEvents:
0132             command = "dasgoclient %s --query '%s'" % (das_options, self.queries(dataset)[0])
0133         elif self.skimEvents:
0134             from os import getenv
0135             if getenv("JENKINS_PREFIX") is not None:
0136                 # to be sure that whatever happens the files are only those at CERN
0137                 command = "das-up-to-nevents.py -d %s -e %d -pc -l lumi_ranges.txt"%(dataset,self.events)
0138             else:
0139                 command = "das-up-to-nevents.py -d %s -e %d -l lumi_ranges.txt"%(dataset,self.events)
0140         # Run filter on DAS output
0141         if self.ib_blacklist:
0142             command += " | grep -E -v "
0143             command += " ".join(["-e '{0}'".format(pattern) for pattern in self.ib_blacklist])
0144         if not self.skimEvents: ## keep run-lumi sorting
0145             from os import getenv
0146             if getenv("CMSSW_USE_IBEOS","false")=="true":
0147                 return "export CMSSW_USE_IBEOS=true; " + command + " | ibeos-lfn-sort"
0148             return command + " | sort -u"
0149         else:
0150             return command
0151
0152     def lumiRanges(self):
0153         if len(self.run) != 0:
0154             return "echo '{\n"+",".join(('"%d":[[1,268435455]]\n'%(x,) for x in self.run))+"}'"
0155         if self.ls :
0156             return "echo '{\n"+",".join(('"%d" : %s\n'%( int(x),self.ls[x]) for x in self.ls.keys()))+"}'"
0157         return None
0158
0159     def lumis(self):
0160       query_lumis = []
0161       if self.ls:
0162         for run in sorted(self.ls.keys()):
0163           run_lumis = []
0164           for rng in self.ls[run]:
0165               if isinstance(rng, int):
0166                   run_lumis.append(str(rng))
0167               else:
0168                   run_lumis.append(str(rng[0])+","+str(rng[1]))
0169           query_lumis.append(":".join(run_lumis))
0170       return query_lumis
0171
0172     def queries(self, dataset):
0173         query_by = "block" if self.ib_block else "dataset"
0174         query_source = "{0}#{1}".format(dataset, self.ib_block) if self.ib_block else dataset
0175
0176         if self.ls :
0177             the_queries = []
0178             #for query_run in self.ls.keys():
0179             # print "run is %s"%(query_run)
0180             # if you have a LS list specified, still query das for the full run (multiple ls queries take forever)
0181             # and use step1_lumiRanges.log to run only on LS which respect your selection
0182
0183             # DO WE WANT T2_CERN ?
0184             return ["file {0}={1} run={2}".format(query_by, query_source, query_run) for query_run in sorted(self.ls.keys())]
0185             #return ["file {0}={1} run={2} site=T2_CH_CERN".format(query_by, query_source, query_run) for query_run in self.ls.keys()]
0186
0187
0188                 #
0189                 #for a_range in self.ls[query_run]:
0190                 #    # print "a_range is %s"%(a_range)
0191                 #    the_queries +=  ["file {0}={1} run={2} lumi={3} ".format(query_by, query_source, query_run, query_ls) for query_ls in expandLsInterval(a_range) ]
0192             #print the_queries
0193             return the_queries
0194
0195         site = " site=T2_CH_CERN"
0196         if "CMSSW_DAS_QUERY_SITES" in os.environ:
0197             if os.environ["CMSSW_DAS_QUERY_SITES"]:
0198                 site = " site=%s" % os.environ["CMSSW_DAS_QUERY_SITES"]
0199             else:
0200                 site = ""
0201         if len(self.run) != 0:
0202             return ["file {0}={1} run={2}{3}".format(query_by, query_source, query_run, site) for query_run in self.run]
0203             #return ["file {0}={1} run={2} ".format(query_by, query_source, query_run) for query_run in self.run]
0204         else:
0205             return ["file {0}={1}{2}".format(query_by, query_source, site)]
0206             #return ["file {0}={1} ".format(query_by, query_source)]
0207
0208     def __str__(self):
0209         if self.ib_block:
0210             return "input from: {0} with run {1}#{2}".format(self.dataSet, self.ib_block, self.run)
0211         return "input from: {0} with run {1}".format(self.dataSet, self.run)
0212
0213
0214 # merge dictionaries, with prioty on the [0] index
0215 def merge(dictlist,TELL=False):
0216     import copy
0217     last=len(dictlist)-1
0218     if TELL: print(last,dictlist)
0219     if last==0:
0220         # ONLY ONE ITEM LEFT
0221         return copy.copy(dictlist[0])
0222     else:
0223         reducedlist=dictlist[0:max(0,last-1)]
0224         if TELL: print(reducedlist)
0225         # make a copy of the last item
0226         d=copy.copy(dictlist[last])
0227         # update with the last but one item
0228         d.update(dictlist[last-1])
0229         # and recursively do the rest
0230         reducedlist.append(d)
0231         return merge(reducedlist,TELL)
0232
0233 def remove(d,key,TELL=False):
0234     import copy
0235     e = copy.deepcopy(d)
0236     if TELL: print("original dict, BEF: %s"%d)
0237     del e[key]
0238     if TELL: print("copy-removed dict, AFT: %s"%e)
0239     return e
0240
0241
0242 #### Standard release validation samples ####
0243
0244 stCond={'--conditions':'auto:run1_mc'}
0245 def Kby(N,s):
0246     return {'--relval':'%s000,%s'%(N,s)}
0247 def Mby(N,s):
0248     return {'--relval':'%s000000,%s'%(N,s)}
0249
0250 def changeRefRelease(steps,listOfPairs):
0251     for s in steps:
0252         if ('INPUT' in steps[s]):
0253             oldD=steps[s]['INPUT'].dataSet
0254             for (ref,newRef) in listOfPairs:
0255                 if  ref in oldD:
0256                     steps[s]['INPUT'].dataSet=oldD.replace(ref,newRef)
0257         if '--pileup_input' in steps[s]:
0258             for (ref,newRef) in listOfPairs:
0259                 if ref in steps[s]['--pileup_input']:
0260                     steps[s]['--pileup_input']=steps[s]['--pileup_input'].replace(ref,newRef)
0261
0262 def addForAll(steps,d):
0263     for s in steps:
0264         steps[s].update(d)
0265
0266
0267 def genvalid(fragment,d,suffix='all',fi='',dataSet=''):
0268     import copy
0269     c=copy.copy(d)
0270     if suffix:
0271         c['-s']=c['-s'].replace('genvalid','genvalid_'+suffix)
0272     if fi:
0273         c['--filein']='lhe:%d'%(fi,)
0274     if dataSet:
0275         c['--filein']='das:%s'%(dataSet,)
0276     c['cfg']=fragment
0277     return c
0278
0279 def check_dups(input):
0280     seen = set()
0281     dups = set(x for x in input if x in seen or seen.add(x))
0282
0283     return dups