DataProcessing/python/Reco.py

0001 #!/usr/bin/env python3
0002 """
0003 _pp_
0004
0005 Scenario supporting proton collisions
0006
0007 """
0008
0009 import os
0010 import sys
0011
0012 from Configuration.DataProcessing.Scenario import *
0013 from Configuration.DataProcessing.Utils import stepALCAPRODUCER,stepSKIMPRODUCER,addMonitoring,dictIO,dqmIOSource,harvestingMode,dqmSeq,nanoFlavours,gtNameAndConnect
0014 import FWCore.ParameterSet.Config as cms
0015 from Configuration.DataProcessing.RecoTLR import customisePrompt,customiseExpress
0016
0017 class Reco(Scenario):
0018     def __init__(self):
0019         Scenario.__init__(self)
0020         self.recoSeq=''
0021         self.cbSc=self.__class__.__name__
0022         self.promptModifiers = cms.ModifierChain()
0023         self.expressModifiers = cms.ModifierChain()
0024         self.visModifiers = cms.ModifierChain()
0025     """
0026     _pp_
0027
0028     Implement configuration building for data processing for proton
0029     collision data taking
0030
0031     """
0032
0033
0034     def _checkRepackedFlag(self, options, **args):
0035         if 'repacked' in args:
0036             if args['repacked'] == True:
0037                 options.isRepacked = True
0038             else:
0039                 options.isRepacked = False
0040
0041
0042
0043     def promptReco(self, globalTag, **args):
0044         """
0045         _promptReco_
0046
0047         Proton collision data taking prompt reco
0048
0049         """
0050         step = stepALCAPRODUCER(args['skims'])
0051         PhysicsSkimStep = ''
0052         if ("PhysicsSkims" in args) :
0053             PhysicsSkimStep = stepSKIMPRODUCER(args['PhysicsSkims'])
0054         dqmStep = dqmSeq(args,'')
0055         options = Options()
0056         options.__dict__.update(defaultOptions.__dict__)
0057         options.scenario = self.cbSc
0058         if ('nThreads' in args) :
0059             options.nThreads=args['nThreads']
0060
0061         miniAODStep = ''
0062         nanoAODStep = ''
0063         if not 'customs' in args:
0064             args['customs']= []
0065
0066         if 'outputs' in args:
0067             print(args['outputs'])
0068             for a in args['outputs']:
0069                 if a['dataTier'] == 'MINIAOD':
0070                     miniAODStep = ',PAT'
0071                 if a['dataTier'] in ['NANOAOD', 'NANOEDMAOD']:
0072                     if "nanoFlavours" in args:
0073                         nanoAODStep = ',NANO'+nanoFlavours(args['nanoFlavours'])
0074                     else:
0075                         nanoAODStep = ',NANO:@PHYS+@L1'
0076
0077         self._checkRepackedFlag(options, **args)
0078
0079         if 'customs' in args:
0080             options.customisation_file=args['customs']
0081
0082         eiStep=''
0083
0084         options.step = 'RAW2DIGI,L1Reco,RECO'
0085         options.step += self.recoSeq + eiStep + step + PhysicsSkimStep
0086         options.step += miniAODStep + nanoAODStep
0087         options.step += ',DQM' + dqmStep + ',ENDJOB'
0088
0089         dictIO(options,args)
0090         options.conditions = gtNameAndConnect(globalTag, args)
0091
0092         process = cms.Process('RECO', cms.ModifierChain(self.eras, self.promptModifiers) )
0093         cb = ConfigBuilder(options, process = process, with_output = True)
0094
0095         # Input source
0096         process.source = cms.Source("PoolSource",
0097             fileNames = cms.untracked.vstring()
0098         )
0099         cb.prepare()
0100
0101         addMonitoring(process)
0102
0103         return process
0104
0105
0106     def expressProcessing(self, globalTag, **args):
0107         """
0108         _expressProcessing_
0109
0110         Proton collision data taking express processing
0111
0112         """
0113         skims = args['skims']
0114         # the AlCaReco skims for PCL should only run during AlCaSkimming step which uses the same configuration on the Tier0 side, for this reason we drop them here
0115         pclWkflws = [x for x in skims if "PromptCalibProd" in x]
0116         for wfl in pclWkflws:
0117             skims.remove(wfl)
0118
0119         step = stepALCAPRODUCER(skims)
0120         dqmStep= dqmSeq(args,'')
0121         options = Options()
0122         options.__dict__.update(defaultOptions.__dict__)
0123         options.scenario = self.cbSc
0124         if ('nThreads' in args) :
0125             options.nThreads=args['nThreads']
0126
0127         eiStep=''
0128
0129         options.step = 'RAW2DIGI,L1Reco,RECO'+self.recoSeq+eiStep+step+',DQM'+dqmStep+',ENDJOB'
0130
0131         dictIO(options,args)
0132         options.conditions = gtNameAndConnect(globalTag, args)
0133
0134
0135         options.filein = 'tobeoverwritten.xyz'
0136         if 'inputSource' in args:
0137             options.filetype = args['inputSource']
0138         process = cms.Process('RECO', cms.ModifierChain(self.eras, self.expressModifiers) )
0139
0140         if 'customs' in args:
0141             options.customisation_file=args['customs']
0142
0143         self._checkRepackedFlag(options,**args)
0144
0145         cb = ConfigBuilder(options, process = process, with_output = True, with_input = True)
0146
0147         cb.prepare()
0148
0149         addMonitoring(process)
0150
0151         return process
0152
0153
0154     def visualizationProcessing(self, globalTag, **args):
0155         """
0156         _visualizationProcessing_
0157
0158         """
0159
0160         options = Options()
0161         options.__dict__.update(defaultOptions.__dict__)
0162         options.scenario = self.cbSc
0163         # FIXME: do we need L1Reco here?
0164         options.step =''
0165         if 'preFilter' in args:
0166             options.step +='FILTER:'+args['preFilter']+','
0167
0168         eiStep=''
0169
0170         if 'beamSplashRun' in args:
0171             options.step += 'RAW2DIGI,L1Reco,RECO'+args['beamSplashRun']+',ENDJOB'
0172             print("Using RECO%s step in visualizationProcessing" % args['beamSplashRun'])
0173         else :
0174             options.step += 'RAW2DIGI,L1Reco,RECO'+eiStep+',ENDJOB'
0175
0176
0177
0178         dictIO(options,args)
0179         options.conditions = gtNameAndConnect(globalTag, args)
0180         options.timeoutOutput = True
0181         # FIXME: maybe can go...maybe not
0182         options.filein = 'tobeoverwritten.xyz'
0183
0184         if 'inputSource' in args:
0185             options.filetype = args['inputSource']
0186         else:
0187             # this is the default as this is what is needed on the OnlineCluster
0188             options.filetype = 'DQMDAQ'
0189
0190         print("Using %s source"%options.filetype)
0191
0192         process = cms.Process('RECO', cms.ModifierChain(self.eras, self.visModifiers) )
0193
0194         if 'customs' in args:
0195             options.customisation_file=args['customs']
0196
0197         self._checkRepackedFlag(options, **args)
0198
0199         cb = ConfigBuilder(options, process = process, with_output = True, with_input = True)
0200
0201         cb.prepare()
0202
0203
0204
0205
0206         # FIXME: not sure abou this one...drop for the moment
0207         # addMonitoring(process)
0208
0209         return process
0210
0211
0212
0213
0214     def alcaSkim(self, skims, **args):
0215         """
0216         _alcaSkim_
0217
0218         AlcaReco processing & skims for proton collisions
0219
0220         """
0221
0222         step = ""
0223         pclWflws = [x for x in skims if "PromptCalibProd" in x]
0224         skims = [x for x in skims if x not in pclWflws]
0225
0226         if len(pclWflws):
0227             step += 'ALCA:'+('+'.join(pclWflws))
0228
0229         if len( skims ) > 0:
0230             if step != "":
0231                 step += ","
0232             step += "ALCAOUTPUT:"+('+'.join(skims))
0233
0234         options = Options()
0235         options.__dict__.update(defaultOptions.__dict__)
0236         options.scenario = self.cbSc
0237         options.step = step
0238         options.conditions = args['globaltag'] if 'globaltag' in args else 'None'
0239         if 'globalTagConnect' in args and args['globalTagConnect'] != '':
0240             options.conditions += ','+args['globalTagConnect']
0241
0242         options.triggerResultsProcess = 'RECO'
0243
0244         if 'customs' in args:
0245             options.customisation_file=args['customs']
0246
0247         process = cms.Process('ALCA', self.eras)
0248         cb = ConfigBuilder(options, process = process)
0249
0250         # Input source
0251         process.source = cms.Source(
0252            "PoolSource",
0253            fileNames = cms.untracked.vstring()
0254         )
0255
0256         cb.prepare()
0257
0258         # FIXME: dirty hack..any way around this?
0259         # Tier0 needs the dataset used for ALCAHARVEST step to be a different data-tier
0260         for wfl in pclWflws:
0261             methodToCall = getattr(process, 'ALCARECOStream'+wfl)
0262             methodToCall.dataset.dataTier = cms.untracked.string('ALCAPROMPT')
0263
0264         return process
0265
0266
0267     def dqmHarvesting(self, datasetName, runNumber, globalTag, **args):
0268         """
0269         _dqmHarvesting_
0270
0271         Proton collisions data taking DQM Harvesting
0272
0273         """
0274         options = defaultOptions
0275         options.scenario = self.cbSc
0276         options.step = "HARVESTING"+dqmSeq(args,':dqmHarvesting')
0277         options.name = "EDMtoMEConvert"
0278         options.conditions = gtNameAndConnect(globalTag, args)
0279
0280         process = cms.Process("HARVESTING", self.eras)
0281         process.source = dqmIOSource(args)
0282
0283         if 'customs' in args:
0284             options.customisation_file=args['customs']
0285
0286         configBuilder = ConfigBuilder(options, process = process)
0287         configBuilder.prepare()
0288
0289         harvestingMode(process,datasetName,args,rANDl=False)
0290         return process
0291
0292
0293     def alcaHarvesting(self, globalTag, datasetName, **args):
0294         """
0295         _alcaHarvesting_
0296
0297         Proton collisions data taking AlCa Harvesting
0298
0299         """
0300         skims = []
0301         if 'skims' in args:
0302             skims = args['skims']
0303
0304
0305         if 'alcapromptdataset' in args:
0306             skims.append('@'+args['alcapromptdataset'])
0307
0308         if len(skims) == 0: return None
0309         options = defaultOptions
0310         options.scenario = self.cbSc if hasattr(self,'cbSc') else self.__class__.__name__
0311         options.step = "ALCAHARVEST:"+('+'.join(skims))
0312         options.name = "ALCAHARVEST"
0313         options.conditions = gtNameAndConnect(globalTag, args)
0314
0315         process = cms.Process("ALCAHARVEST", self.eras)
0316         process.source = cms.Source("PoolSource")
0317
0318         if 'customs' in args:
0319             options.customisation_file=args['customs']
0320
0321         configBuilder = ConfigBuilder(options, process = process)
0322         configBuilder.prepare()
0323
0324         #
0325         # customise process for particular job
0326         #
0327         process.source.processingMode = cms.untracked.string('RunsAndLumis')
0328         process.source.fileNames = cms.untracked(cms.vstring())
0329         process.maxEvents.input = -1
0330         process.dqmSaver.workflow = datasetName
0331
0332         return process
0333
0334     def skimming(self, skims, globalTag,**options):
0335         """
0336         _skimming_
0337
0338         skimming method overload for the prompt skiming
0339
0340         """
0341         options = defaultOptions
0342         options.scenario = self.cbSc if hasattr(self,'cbSc') else self.__class__.__name__
0343         options.step = "SKIM:"+('+'.join(skims))
0344         options.name = "SKIM"
0345         options.conditions = gtNameAndConnect(globalTag, args)
0346         process = cms.Process("SKIM", self.eras)
0347         process.source = cms.Source("PoolSource")
0348
0349         if 'customs' in args:
0350             options.customisation_file=args['customs']
0351
0352         configBuilder = ConfigBuilder(options, process = process)
0353         configBuilder.prepare()
0354
0355         return process
0356
0357     """
0358     def repack(self, **args):
0359         options = defaultOptions
0360         dictIO(options,args)
0361         options.filein='file.dat'
0362         options.filetype='DAT'
0363         options.scenario = self.cbSc if hasattr(self,'cbSc') else self.__class__.__name__
0364         process = cms.Process('REPACK', self.eras)
0365         cb = ConfigBuilder(options, process = process, with_output = True,with_input=True)
0366         cb.prepare()
0367         print cb.pythonCfgCode
0368         return process
0369     """