DataProcessing/python/Reco.py

0001 #!/usr/bin/env python3
0002 """
0003 _pp_
0004
0005 Scenario supporting proton collisions
0006
0007 """
0008 from __future__ import print_function
0009
0010 import os
0011 import sys
0012
0013 from Configuration.DataProcessing.Scenario import *
0014 from Configuration.DataProcessing.Utils import stepALCAPRODUCER,stepSKIMPRODUCER,addMonitoring,dictIO,dqmIOSource,harvestingMode,dqmSeq,gtNameAndConnect
0015 import FWCore.ParameterSet.Config as cms
0016 from Configuration.DataProcessing.RecoTLR import customisePrompt,customiseExpress
0017
0018 class Reco(Scenario):
0019     def __init__(self):
0020         Scenario.__init__(self)
0021         self.recoSeq=''
0022         self.cbSc=self.__class__.__name__
0023         self.promptModifiers = cms.ModifierChain()
0024         self.expressModifiers = cms.ModifierChain()
0025         self.visModifiers = cms.ModifierChain()
0026     """
0027     _pp_
0028
0029     Implement configuration building for data processing for proton
0030     collision data taking
0031
0032     """
0033
0034
0035     def _checkRepackedFlag(self, options, **args):
0036         if 'repacked' in args:
0037             if args['repacked'] == True:
0038                 options.isRepacked = True
0039             else:
0040                 options.isRepacked = False
0041
0042
0043
0044     def promptReco(self, globalTag, **args):
0045         """
0046         _promptReco_
0047
0048         Proton collision data taking prompt reco
0049
0050         """
0051         step = stepALCAPRODUCER(args['skims'])
0052         PhysicsSkimStep = ''
0053         if ("PhysicsSkims" in args) :
0054             PhysicsSkimStep = stepSKIMPRODUCER(args['PhysicsSkims'])
0055         dqmStep = dqmSeq(args,'')
0056         options = Options()
0057         options.__dict__.update(defaultOptions.__dict__)
0058         options.scenario = self.cbSc
0059         if ('nThreads' in args) :
0060             options.nThreads=args['nThreads']
0061
0062         miniAODStep = ''
0063         nanoAODStep = ''
0064
0065         if 'outputs' in args:
0066             print(args['outputs'])
0067             for a in args['outputs']:
0068                 if a['dataTier'] == 'MINIAOD':
0069                     miniAODStep = ',PAT'
0070                 if a['dataTier'] in ['NANOAOD', 'NANOEDMAOD']:
0071                     nanoAODStep = ',NANO'
0072
0073         self._checkRepackedFlag(options, **args)
0074
0075         if 'customs' in args:
0076             options.customisation_file=args['customs']
0077
0078         eiStep=''
0079
0080         options.step = 'RAW2DIGI,L1Reco,RECO'
0081         options.step += self.recoSeq + eiStep + step + PhysicsSkimStep
0082         options.step += miniAODStep + nanoAODStep
0083         options.step += ',DQM' + dqmStep + ',ENDJOB'
0084
0085         dictIO(options,args)
0086         options.conditions = gtNameAndConnect(globalTag, args)
0087
0088         process = cms.Process('RECO', cms.ModifierChain(self.eras, self.promptModifiers) )
0089         cb = ConfigBuilder(options, process = process, with_output = True)
0090
0091         # Input source
0092         process.source = cms.Source("PoolSource",
0093             fileNames = cms.untracked.vstring()
0094         )
0095         cb.prepare()
0096
0097         addMonitoring(process)
0098
0099         return process
0100
0101
0102     def expressProcessing(self, globalTag, **args):
0103         """
0104         _expressProcessing_
0105
0106         Proton collision data taking express processing
0107
0108         """
0109         skims = args['skims']
0110         # the AlCaReco skims for PCL should only run during AlCaSkimming step which uses the same configuration on the Tier0 side, for this reason we drop them here
0111         pclWkflws = [x for x in skims if "PromptCalibProd" in x]
0112         for wfl in pclWkflws:
0113             skims.remove(wfl)
0114
0115         step = stepALCAPRODUCER(skims)
0116         dqmStep= dqmSeq(args,'')
0117         options = Options()
0118         options.__dict__.update(defaultOptions.__dict__)
0119         options.scenario = self.cbSc
0120         if ('nThreads' in args) :
0121             options.nThreads=args['nThreads']
0122
0123         eiStep=''
0124
0125         options.step = 'RAW2DIGI,L1Reco,RECO'+self.recoSeq+eiStep+step+',DQM'+dqmStep+',ENDJOB'
0126
0127         dictIO(options,args)
0128         options.conditions = gtNameAndConnect(globalTag, args)
0129
0130
0131         options.filein = 'tobeoverwritten.xyz'
0132         if 'inputSource' in args:
0133             options.filetype = args['inputSource']
0134         process = cms.Process('RECO', cms.ModifierChain(self.eras, self.expressModifiers) )
0135
0136         if 'customs' in args:
0137             options.customisation_file=args['customs']
0138
0139         self._checkRepackedFlag(options,**args)
0140
0141         cb = ConfigBuilder(options, process = process, with_output = True, with_input = True)
0142
0143         cb.prepare()
0144
0145         addMonitoring(process)
0146
0147         return process
0148
0149
0150     def visualizationProcessing(self, globalTag, **args):
0151         """
0152         _visualizationProcessing_
0153
0154         """
0155
0156         options = Options()
0157         options.__dict__.update(defaultOptions.__dict__)
0158         options.scenario = self.cbSc
0159         # FIXME: do we need L1Reco here?
0160         options.step =''
0161         if 'preFilter' in args:
0162             options.step +='FILTER:'+args['preFilter']+','
0163
0164         eiStep=''
0165
0166         if 'beamSplashRun' in args:
0167             options.step += 'RAW2DIGI,L1Reco,RECO'+args['beamSplashRun']+',ENDJOB'
0168             print("Using RECO%s step in visualizationProcessing" % args['beamSplashRun'])
0169         else :
0170             options.step += 'RAW2DIGI,L1Reco,RECO'+eiStep+',ENDJOB'
0171
0172
0173
0174         dictIO(options,args)
0175         options.conditions = gtNameAndConnect(globalTag, args)
0176         options.timeoutOutput = True
0177         # FIXME: maybe can go...maybe not
0178         options.filein = 'tobeoverwritten.xyz'
0179
0180         if 'inputSource' in args:
0181             options.filetype = args['inputSource']
0182         else:
0183             # this is the default as this is what is needed on the OnlineCluster
0184             options.filetype = 'DQMDAQ'
0185
0186         print("Using %s source"%options.filetype)
0187
0188         process = cms.Process('RECO', cms.ModifierChain(self.eras, self.visModifiers) )
0189
0190         if 'customs' in args:
0191             options.customisation_file=args['customs']
0192
0193         self._checkRepackedFlag(options, **args)
0194
0195         cb = ConfigBuilder(options, process = process, with_output = True, with_input = True)
0196
0197         cb.prepare()
0198
0199
0200
0201
0202         # FIXME: not sure abou this one...drop for the moment
0203         # addMonitoring(process)
0204
0205         return process
0206
0207
0208
0209
0210     def alcaSkim(self, skims, **args):
0211         """
0212         _alcaSkim_
0213
0214         AlcaReco processing & skims for proton collisions
0215
0216         """
0217
0218         step = ""
0219         pclWflws = [x for x in skims if "PromptCalibProd" in x]
0220         skims = [x for x in skims if x not in pclWflws]
0221
0222         if len(pclWflws):
0223             step += 'ALCA:'+('+'.join(pclWflws))
0224
0225         if len( skims ) > 0:
0226             if step != "":
0227                 step += ","
0228             step += "ALCAOUTPUT:"+('+'.join(skims))
0229
0230         options = Options()
0231         options.__dict__.update(defaultOptions.__dict__)
0232         options.scenario = self.cbSc
0233         options.step = step
0234         options.conditions = args['globaltag'] if 'globaltag' in args else 'None'
0235         if 'globalTagConnect' in args and args['globalTagConnect'] != '':
0236             options.conditions += ','+args['globalTagConnect']
0237
0238         options.triggerResultsProcess = 'RECO'
0239
0240         if 'customs' in args:
0241             options.customisation_file=args['customs']
0242
0243         process = cms.Process('ALCA', self.eras)
0244         cb = ConfigBuilder(options, process = process)
0245
0246         # Input source
0247         process.source = cms.Source(
0248            "PoolSource",
0249            fileNames = cms.untracked.vstring()
0250         )
0251
0252         cb.prepare()
0253
0254         # FIXME: dirty hack..any way around this?
0255         # Tier0 needs the dataset used for ALCAHARVEST step to be a different data-tier
0256         for wfl in pclWflws:
0257             methodToCall = getattr(process, 'ALCARECOStream'+wfl)
0258             methodToCall.dataset.dataTier = cms.untracked.string('ALCAPROMPT')
0259
0260         return process
0261
0262
0263     def dqmHarvesting(self, datasetName, runNumber, globalTag, **args):
0264         """
0265         _dqmHarvesting_
0266
0267         Proton collisions data taking DQM Harvesting
0268
0269         """
0270         options = defaultOptions
0271         options.scenario = self.cbSc
0272         options.step = "HARVESTING"+dqmSeq(args,':dqmHarvesting')
0273         options.name = "EDMtoMEConvert"
0274         options.conditions = gtNameAndConnect(globalTag, args)
0275
0276         process = cms.Process("HARVESTING", self.eras)
0277         process.source = dqmIOSource(args)
0278
0279         if 'customs' in args:
0280             options.customisation_file=args['customs']
0281
0282         configBuilder = ConfigBuilder(options, process = process)
0283         configBuilder.prepare()
0284
0285         harvestingMode(process,datasetName,args,rANDl=False)
0286         return process
0287
0288
0289     def alcaHarvesting(self, globalTag, datasetName, **args):
0290         """
0291         _alcaHarvesting_
0292
0293         Proton collisions data taking AlCa Harvesting
0294
0295         """
0296         skims = []
0297         if 'skims' in args:
0298             skims = args['skims']
0299
0300
0301         if 'alcapromptdataset' in args:
0302             skims.append('@'+args['alcapromptdataset'])
0303
0304         if len(skims) == 0: return None
0305         options = defaultOptions
0306         options.scenario = self.cbSc if hasattr(self,'cbSc') else self.__class__.__name__
0307         options.step = "ALCAHARVEST:"+('+'.join(skims))
0308         options.name = "ALCAHARVEST"
0309         options.conditions = gtNameAndConnect(globalTag, args)
0310
0311         process = cms.Process("ALCAHARVEST", self.eras)
0312         process.source = cms.Source("PoolSource")
0313
0314         if 'customs' in args:
0315             options.customisation_file=args['customs']
0316
0317         configBuilder = ConfigBuilder(options, process = process)
0318         configBuilder.prepare()
0319
0320         #
0321         # customise process for particular job
0322         #
0323         process.source.processingMode = cms.untracked.string('RunsAndLumis')
0324         process.source.fileNames = cms.untracked(cms.vstring())
0325         process.maxEvents.input = -1
0326         process.dqmSaver.workflow = datasetName
0327
0328         return process
0329
0330     def skimming(self, skims, globalTag,**options):
0331         """
0332         _skimming_
0333
0334         skimming method overload for the prompt skiming
0335
0336         """
0337         options = defaultOptions
0338         options.scenario = self.cbSc if hasattr(self,'cbSc') else self.__class__.__name__
0339         options.step = "SKIM:"+('+'.join(skims))
0340         options.name = "SKIM"
0341         options.conditions = gtNameAndConnect(globalTag, args)
0342         process = cms.Process("SKIM", self.eras)
0343         process.source = cms.Source("PoolSource")
0344
0345         if 'customs' in args:
0346             options.customisation_file=args['customs']
0347
0348         configBuilder = ConfigBuilder(options, process = process)
0349         configBuilder.prepare()
0350
0351         return process
0352
0353     """
0354     def repack(self, **args):
0355         options = defaultOptions
0356         dictIO(options,args)
0357         options.filein='file.dat'
0358         options.filetype='DAT'
0359         options.scenario = self.cbSc if hasattr(self,'cbSc') else self.__class__.__name__
0360         process = cms.Process('REPACK', self.eras)
0361         cb = ConfigBuilder(options, process = process, with_output = True,with_input=True)
0362         cb.prepare()
0363         print cb.pythonCfgCode
0364         return process
0365     """