DataProcessing/python/Reco.py

0001 #!/usr/bin/env python3
0002 """
0003 _pp_
0004
0005 Scenario supporting proton collisions
0006
0007 """
0008 from __future__ import print_function
0009
0010 import os
0011 import sys
0012
0013 from Configuration.DataProcessing.Scenario import *
0014 from Configuration.DataProcessing.Utils import stepALCAPRODUCER,stepSKIMPRODUCER,addMonitoring,dictIO,dqmIOSource,harvestingMode,dqmSeq,nanoFlavours,gtNameAndConnect
0015 import FWCore.ParameterSet.Config as cms
0016 from Configuration.DataProcessing.RecoTLR import customisePrompt,customiseExpress
0017
0018 class Reco(Scenario):
0019     def __init__(self):
0020         Scenario.__init__(self)
0021         self.recoSeq=''
0022         self.cbSc=self.__class__.__name__
0023         self.promptModifiers = cms.ModifierChain()
0024         self.expressModifiers = cms.ModifierChain()
0025         self.visModifiers = cms.ModifierChain()
0026     """
0027     _pp_
0028
0029     Implement configuration building for data processing for proton
0030     collision data taking
0031
0032     """
0033
0034
0035     def _checkRepackedFlag(self, options, **args):
0036         if 'repacked' in args:
0037             if args['repacked'] == True:
0038                 options.isRepacked = True
0039             else:
0040                 options.isRepacked = False
0041
0042
0043
0044     def promptReco(self, globalTag, **args):
0045         """
0046         _promptReco_
0047
0048         Proton collision data taking prompt reco
0049
0050         """
0051         step = stepALCAPRODUCER(args['skims'])
0052         PhysicsSkimStep = ''
0053         if ("PhysicsSkims" in args) :
0054             PhysicsSkimStep = stepSKIMPRODUCER(args['PhysicsSkims'])
0055         dqmStep = dqmSeq(args,'')
0056         options = Options()
0057         options.__dict__.update(defaultOptions.__dict__)
0058         options.scenario = self.cbSc
0059         if ('nThreads' in args) :
0060             options.nThreads=args['nThreads']
0061
0062         miniAODStep = ''
0063         nanoAODStep = ''
0064         if not 'customs' in args:
0065             args['customs']= []
0066
0067         if 'outputs' in args:
0068             print(args['outputs'])
0069             for a in args['outputs']:
0070                 if a['dataTier'] == 'MINIAOD':
0071                     miniAODStep = ',PAT'
0072                 if a['dataTier'] in ['NANOAOD', 'NANOEDMAOD']:
0073                     if "nanoFlavours" in args:
0074                         nanoAODStep = ',NANO'+nanoFlavours(args['nanoFlavours'])
0075                     else:
0076                         nanoAODStep = ',NANO:@PHYS+@L1'
0077
0078         self._checkRepackedFlag(options, **args)
0079
0080         if 'customs' in args:
0081             options.customisation_file=args['customs']
0082
0083         eiStep=''
0084
0085         options.step = 'RAW2DIGI,L1Reco,RECO'
0086         options.step += self.recoSeq + eiStep + step + PhysicsSkimStep
0087         options.step += miniAODStep + nanoAODStep
0088         options.step += ',DQM' + dqmStep + ',ENDJOB'
0089
0090         dictIO(options,args)
0091         options.conditions = gtNameAndConnect(globalTag, args)
0092
0093         process = cms.Process('RECO', cms.ModifierChain(self.eras, self.promptModifiers) )
0094         cb = ConfigBuilder(options, process = process, with_output = True)
0095
0096         # Input source
0097         process.source = cms.Source("PoolSource",
0098             fileNames = cms.untracked.vstring()
0099         )
0100         cb.prepare()
0101
0102         addMonitoring(process)
0103
0104         return process
0105
0106
0107     def expressProcessing(self, globalTag, **args):
0108         """
0109         _expressProcessing_
0110
0111         Proton collision data taking express processing
0112
0113         """
0114         skims = args['skims']
0115         # the AlCaReco skims for PCL should only run during AlCaSkimming step which uses the same configuration on the Tier0 side, for this reason we drop them here
0116         pclWkflws = [x for x in skims if "PromptCalibProd" in x]
0117         for wfl in pclWkflws:
0118             skims.remove(wfl)
0119
0120         step = stepALCAPRODUCER(skims)
0121         dqmStep= dqmSeq(args,'')
0122         options = Options()
0123         options.__dict__.update(defaultOptions.__dict__)
0124         options.scenario = self.cbSc
0125         if ('nThreads' in args) :
0126             options.nThreads=args['nThreads']
0127
0128         eiStep=''
0129
0130         options.step = 'RAW2DIGI,L1Reco,RECO'+self.recoSeq+eiStep+step+',DQM'+dqmStep+',ENDJOB'
0131
0132         dictIO(options,args)
0133         options.conditions = gtNameAndConnect(globalTag, args)
0134
0135
0136         options.filein = 'tobeoverwritten.xyz'
0137         if 'inputSource' in args:
0138             options.filetype = args['inputSource']
0139         process = cms.Process('RECO', cms.ModifierChain(self.eras, self.expressModifiers) )
0140
0141         if 'customs' in args:
0142             options.customisation_file=args['customs']
0143
0144         self._checkRepackedFlag(options,**args)
0145
0146         cb = ConfigBuilder(options, process = process, with_output = True, with_input = True)
0147
0148         cb.prepare()
0149
0150         addMonitoring(process)
0151
0152         return process
0153
0154
0155     def visualizationProcessing(self, globalTag, **args):
0156         """
0157         _visualizationProcessing_
0158
0159         """
0160
0161         options = Options()
0162         options.__dict__.update(defaultOptions.__dict__)
0163         options.scenario = self.cbSc
0164         # FIXME: do we need L1Reco here?
0165         options.step =''
0166         if 'preFilter' in args:
0167             options.step +='FILTER:'+args['preFilter']+','
0168
0169         eiStep=''
0170
0171         if 'beamSplashRun' in args:
0172             options.step += 'RAW2DIGI,L1Reco,RECO'+args['beamSplashRun']+',ENDJOB'
0173             print("Using RECO%s step in visualizationProcessing" % args['beamSplashRun'])
0174         else :
0175             options.step += 'RAW2DIGI,L1Reco,RECO'+eiStep+',ENDJOB'
0176
0177
0178
0179         dictIO(options,args)
0180         options.conditions = gtNameAndConnect(globalTag, args)
0181         options.timeoutOutput = True
0182         # FIXME: maybe can go...maybe not
0183         options.filein = 'tobeoverwritten.xyz'
0184
0185         if 'inputSource' in args:
0186             options.filetype = args['inputSource']
0187         else:
0188             # this is the default as this is what is needed on the OnlineCluster
0189             options.filetype = 'DQMDAQ'
0190
0191         print("Using %s source"%options.filetype)
0192
0193         process = cms.Process('RECO', cms.ModifierChain(self.eras, self.visModifiers) )
0194
0195         if 'customs' in args:
0196             options.customisation_file=args['customs']
0197
0198         self._checkRepackedFlag(options, **args)
0199
0200         cb = ConfigBuilder(options, process = process, with_output = True, with_input = True)
0201
0202         cb.prepare()
0203
0204
0205
0206
0207         # FIXME: not sure abou this one...drop for the moment
0208         # addMonitoring(process)
0209
0210         return process
0211
0212
0213
0214
0215     def alcaSkim(self, skims, **args):
0216         """
0217         _alcaSkim_
0218
0219         AlcaReco processing & skims for proton collisions
0220
0221         """
0222
0223         step = ""
0224         pclWflws = [x for x in skims if "PromptCalibProd" in x]
0225         skims = [x for x in skims if x not in pclWflws]
0226
0227         if len(pclWflws):
0228             step += 'ALCA:'+('+'.join(pclWflws))
0229
0230         if len( skims ) > 0:
0231             if step != "":
0232                 step += ","
0233             step += "ALCAOUTPUT:"+('+'.join(skims))
0234
0235         options = Options()
0236         options.__dict__.update(defaultOptions.__dict__)
0237         options.scenario = self.cbSc
0238         options.step = step
0239         options.conditions = args['globaltag'] if 'globaltag' in args else 'None'
0240         if 'globalTagConnect' in args and args['globalTagConnect'] != '':
0241             options.conditions += ','+args['globalTagConnect']
0242
0243         options.triggerResultsProcess = 'RECO'
0244
0245         if 'customs' in args:
0246             options.customisation_file=args['customs']
0247
0248         process = cms.Process('ALCA', self.eras)
0249         cb = ConfigBuilder(options, process = process)
0250
0251         # Input source
0252         process.source = cms.Source(
0253            "PoolSource",
0254            fileNames = cms.untracked.vstring()
0255         )
0256
0257         cb.prepare()
0258
0259         # FIXME: dirty hack..any way around this?
0260         # Tier0 needs the dataset used for ALCAHARVEST step to be a different data-tier
0261         for wfl in pclWflws:
0262             methodToCall = getattr(process, 'ALCARECOStream'+wfl)
0263             methodToCall.dataset.dataTier = cms.untracked.string('ALCAPROMPT')
0264
0265         return process
0266
0267
0268     def dqmHarvesting(self, datasetName, runNumber, globalTag, **args):
0269         """
0270         _dqmHarvesting_
0271
0272         Proton collisions data taking DQM Harvesting
0273
0274         """
0275         options = defaultOptions
0276         options.scenario = self.cbSc
0277         options.step = "HARVESTING"+dqmSeq(args,':dqmHarvesting')
0278         options.name = "EDMtoMEConvert"
0279         options.conditions = gtNameAndConnect(globalTag, args)
0280
0281         process = cms.Process("HARVESTING", self.eras)
0282         process.source = dqmIOSource(args)
0283
0284         if 'customs' in args:
0285             options.customisation_file=args['customs']
0286
0287         configBuilder = ConfigBuilder(options, process = process)
0288         configBuilder.prepare()
0289
0290         harvestingMode(process,datasetName,args,rANDl=False)
0291         return process
0292
0293
0294     def alcaHarvesting(self, globalTag, datasetName, **args):
0295         """
0296         _alcaHarvesting_
0297
0298         Proton collisions data taking AlCa Harvesting
0299
0300         """
0301         skims = []
0302         if 'skims' in args:
0303             skims = args['skims']
0304
0305
0306         if 'alcapromptdataset' in args:
0307             skims.append('@'+args['alcapromptdataset'])
0308
0309         if len(skims) == 0: return None
0310         options = defaultOptions
0311         options.scenario = self.cbSc if hasattr(self,'cbSc') else self.__class__.__name__
0312         options.step = "ALCAHARVEST:"+('+'.join(skims))
0313         options.name = "ALCAHARVEST"
0314         options.conditions = gtNameAndConnect(globalTag, args)
0315
0316         process = cms.Process("ALCAHARVEST", self.eras)
0317         process.source = cms.Source("PoolSource")
0318
0319         if 'customs' in args:
0320             options.customisation_file=args['customs']
0321
0322         configBuilder = ConfigBuilder(options, process = process)
0323         configBuilder.prepare()
0324
0325         #
0326         # customise process for particular job
0327         #
0328         process.source.processingMode = cms.untracked.string('RunsAndLumis')
0329         process.source.fileNames = cms.untracked(cms.vstring())
0330         process.maxEvents.input = -1
0331         process.dqmSaver.workflow = datasetName
0332
0333         return process
0334
0335     def skimming(self, skims, globalTag,**options):
0336         """
0337         _skimming_
0338
0339         skimming method overload for the prompt skiming
0340
0341         """
0342         options = defaultOptions
0343         options.scenario = self.cbSc if hasattr(self,'cbSc') else self.__class__.__name__
0344         options.step = "SKIM:"+('+'.join(skims))
0345         options.name = "SKIM"
0346         options.conditions = gtNameAndConnect(globalTag, args)
0347         process = cms.Process("SKIM", self.eras)
0348         process.source = cms.Source("PoolSource")
0349
0350         if 'customs' in args:
0351             options.customisation_file=args['customs']
0352
0353         configBuilder = ConfigBuilder(options, process = process)
0354         configBuilder.prepare()
0355
0356         return process
0357
0358     """
0359     def repack(self, **args):
0360         options = defaultOptions
0361         dictIO(options,args)
0362         options.filein='file.dat'
0363         options.filetype='DAT'
0364         options.scenario = self.cbSc if hasattr(self,'cbSc') else self.__class__.__name__
0365         process = cms.Process('REPACK', self.eras)
0366         cb = ConfigBuilder(options, process = process, with_output = True,with_input=True)
0367         cb.prepare()
0368         print cb.pythonCfgCode
0369         return process
0370     """