Applications/python/ConfigBuilder.py

0001 #! /usr/bin/env python3
0002
0003 from __future__ import print_function
0004 __version__ = "$Revision: 1.19 $"
0005 __source__ = "$Source: /local/reps/CMSSW/CMSSW/Configuration/Applications/python/ConfigBuilder.py,v $"
0006
0007 import FWCore.ParameterSet.Config as cms
0008 from FWCore.ParameterSet.Modules import _Module
0009 # The following import is provided for backward compatibility reasons.
0010 # The function used to be defined in this file.
0011 from FWCore.ParameterSet.MassReplace import massReplaceInputTag as MassReplaceInputTag
0012
0013 import hashlib
0014 import sys
0015 import re
0016 import collections
0017 from subprocess import Popen,PIPE
0018 import FWCore.ParameterSet.DictTypes as DictTypes
0019 class Options:
0020     pass
0021
0022 # the canonical defaults
0023 defaultOptions = Options()
0024 defaultOptions.datamix = 'DataOnSim'
0025 defaultOptions.isMC=False
0026 defaultOptions.isData=True
0027 defaultOptions.step=''
0028 defaultOptions.pileup='NoPileUp'
0029 defaultOptions.pileup_input = None
0030 defaultOptions.pileup_dasoption = ''
0031 defaultOptions.geometry = 'SimDB'
0032 defaultOptions.geometryExtendedOptions = ['ExtendedGFlash','Extended','NoCastor']
0033 defaultOptions.magField = ''
0034 defaultOptions.conditions = None
0035 defaultOptions.scenarioOptions=['pp','cosmics','nocoll','HeavyIons']
0036 defaultOptions.harvesting= 'AtRunEnd'
0037 defaultOptions.gflash = False
0038 defaultOptions.number = -1
0039 defaultOptions.number_out = None
0040 defaultOptions.arguments = ""
0041 defaultOptions.name = "NO NAME GIVEN"
0042 defaultOptions.evt_type = ""
0043 defaultOptions.filein = ""
0044 defaultOptions.dasquery=""
0045 defaultOptions.dasoption=""
0046 defaultOptions.secondfilein = ""
0047 defaultOptions.customisation_file = []
0048 defaultOptions.customisation_file_unsch = []
0049 defaultOptions.customise_commands = ""
0050 defaultOptions.inline_custom=False
0051 defaultOptions.particleTable = 'pythiapdt'
0052 defaultOptions.particleTableList = ['pythiapdt','pdt']
0053 defaultOptions.dirin = ''
0054 defaultOptions.dirout = ''
0055 defaultOptions.filetype = 'EDM'
0056 defaultOptions.fileout = 'output.root'
0057 defaultOptions.filtername = ''
0058 defaultOptions.lazy_download = False
0059 defaultOptions.custom_conditions = ''
0060 defaultOptions.hltProcess = ''
0061 defaultOptions.eventcontent = None
0062 defaultOptions.datatier = None
0063 defaultOptions.inlineEventContent = True
0064 defaultOptions.inlineObjets =''
0065 defaultOptions.hideGen=False
0066 from Configuration.StandardSequences.VtxSmeared import VtxSmearedDefaultKey,VtxSmearedHIDefaultKey
0067 defaultOptions.beamspot=None
0068 defaultOptions.outputDefinition =''
0069 defaultOptions.inputCommands = None
0070 defaultOptions.outputCommands = None
0071 defaultOptions.inputEventContent = ''
0072 defaultOptions.dropDescendant = False
0073 defaultOptions.relval = None
0074 defaultOptions.profile = None
0075 defaultOptions.isRepacked = False
0076 defaultOptions.restoreRNDSeeds = False
0077 defaultOptions.donotDropOnInput = ''
0078 defaultOptions.python_filename =''
0079 defaultOptions.io=None
0080 defaultOptions.lumiToProcess=None
0081 defaultOptions.fast=False
0082 defaultOptions.runsAndWeightsForMC = None
0083 defaultOptions.runsScenarioForMC = None
0084 defaultOptions.runsAndWeightsForMCIntegerWeights = None
0085 defaultOptions.runsScenarioForMCIntegerWeights = None
0086 defaultOptions.runUnscheduled = False
0087 defaultOptions.timeoutOutput = False
0088 defaultOptions.nThreads = '1'
0089 defaultOptions.nStreams = '0'
0090 defaultOptions.nConcurrentLumis = '0'
0091 defaultOptions.nConcurrentIOVs = '1'
0092
0093 # some helper routines
0094 def dumpPython(process,name):
0095     theObject = getattr(process,name)
0096     if isinstance(theObject,cms.Path) or isinstance(theObject,cms.EndPath) or isinstance(theObject,cms.Sequence):
0097         return "process."+name+" = " + theObject.dumpPython()
0098     elif isinstance(theObject,_Module) or isinstance(theObject,cms.ESProducer):
0099         return "process."+name+" = " + theObject.dumpPython()+"\n"
0100     else:
0101         return "process."+name+" = " + theObject.dumpPython()+"\n"
0102 def filesFromList(fileName,s=None):
0103     import os
0104     import FWCore.ParameterSet.Config as cms
0105     prim=[]
0106     sec=[]
0107     for line in open(fileName,'r'):
0108         if line.count(".root")>=2:
0109             #two files solution...
0110             entries=line.replace("\n","").split()
0111             prim.append(entries[0])
0112             sec.append(entries[1])
0113         elif (line.find(".root")!=-1):
0114             entry=line.replace("\n","")
0115             prim.append(entry)
0116     # remove any duplicates but keep the order
0117     file_seen = set()
0118     prim = [f for f in prim if not (f in file_seen or file_seen.add(f))]
0119     file_seen = set()
0120     sec = [f for f in sec if not (f in file_seen or file_seen.add(f))]
0121     if s:
0122         if not hasattr(s,"fileNames"):
0123             s.fileNames=cms.untracked.vstring(prim)
0124         else:
0125             s.fileNames.extend(prim)
0126         if len(sec)!=0:
0127             if not hasattr(s,"secondaryFileNames"):
0128                 s.secondaryFileNames=cms.untracked.vstring(sec)
0129             else:
0130                 s.secondaryFileNames.extend(sec)
0131     print("found files: ",prim)
0132     if len(prim)==0:
0133         raise Exception("There are not files in input from the file list")
0134     if len(sec)!=0:
0135         print("found parent files:",sec)
0136     return (prim,sec)
0137
0138 def filesFromDASQuery(query,option="",s=None):
0139     import os,time
0140     import FWCore.ParameterSet.Config as cms
0141     prim=[]
0142     sec=[]
0143     print("the query is",query)
0144     eC=5
0145     count=0
0146     while eC!=0 and count<3:
0147         if count!=0:
0148             print('Sleeping, then retrying DAS')
0149             time.sleep(100)
0150         p = Popen('dasgoclient %s --query "%s"'%(option,query), stdout=PIPE,shell=True, universal_newlines=True)
0151         pipe=p.stdout.read()
0152         tupleP = os.waitpid(p.pid, 0)
0153         eC=tupleP[1]
0154         count=count+1
0155     if eC==0:
0156         print("DAS succeeded after",count,"attempts",eC)
0157     else:
0158         print("DAS failed 3 times- I give up")
0159     for line in pipe.split('\n'):
0160         if line.count(".root")>=2:
0161             #two files solution...
0162             entries=line.replace("\n","").split()
0163             prim.append(entries[0])
0164             sec.append(entries[1])
0165         elif (line.find(".root")!=-1):
0166             entry=line.replace("\n","")
0167             prim.append(entry)
0168     # remove any duplicates
0169     prim = sorted(list(set(prim)))
0170     sec = sorted(list(set(sec)))
0171     if s:
0172         if not hasattr(s,"fileNames"):
0173             s.fileNames=cms.untracked.vstring(prim)
0174         else:
0175             s.fileNames.extend(prim)
0176         if len(sec)!=0:
0177             if not hasattr(s,"secondaryFileNames"):
0178                 s.secondaryFileNames=cms.untracked.vstring(sec)
0179             else:
0180                 s.secondaryFileNames.extend(sec)
0181     print("found files: ",prim)
0182     if len(sec)!=0:
0183         print("found parent files:",sec)
0184     return (prim,sec)
0185
0186 def anyOf(listOfKeys,dict,opt=None):
0187     for k in listOfKeys:
0188         if k in dict:
0189             toReturn=dict[k]
0190             dict.pop(k)
0191             return toReturn
0192     if opt!=None:
0193         return opt
0194     else:
0195         raise Exception("any of "+','.join(listOfKeys)+" are mandatory entries of --output options")
0196
0197 class ConfigBuilder(object):
0198     """The main building routines """
0199
0200     def __init__(self, options, process = None, with_output = False, with_input = False ):
0201         """options taken from old cmsDriver and optparse """
0202
0203         options.outfile_name = options.dirout+options.fileout
0204
0205         self._options = options
0206
0207         if self._options.isData and options.isMC:
0208             raise Exception("ERROR: You may specify only --data or --mc, not both")
0209         #if not self._options.conditions:
0210         #        raise Exception("ERROR: No conditions given!\nPlease specify conditions. E.g. via --conditions=IDEAL_30X::All")
0211
0212         # check that MEtoEDMConverter (running in ENDJOB) and DQMIO don't run in the same job
0213         if 'ENDJOB' in self._options.step:
0214             if  (hasattr(self._options,"outputDefinition") and \
0215                 self._options.outputDefinition != '' and \
0216                 any(anyOf(['t','tier','dataTier'],outdic) == 'DQMIO' for outdic in eval(self._options.outputDefinition))) or \
0217                 (hasattr(self._options,"datatier") and \
0218                 self._options.datatier and \
0219                 'DQMIO' in self._options.datatier):
0220                 print("removing ENDJOB from steps since not compatible with DQMIO dataTier")
0221                 self._options.step=self._options.step.replace(',ENDJOB','')
0222
0223
0224
0225         # what steps are provided by this class?
0226         stepList = [re.sub(r'^prepare_', '', methodName) for methodName in ConfigBuilder.__dict__ if methodName.startswith('prepare_')]
0227         self.stepMap={}
0228         self.stepKeys=[]
0229         for step in self._options.step.split(","):
0230             if step=='': continue
0231             stepParts = step.split(":")
0232             stepName = stepParts[0]
0233             if stepName not in stepList and not stepName.startswith('re'):
0234                 raise ValueError("Step "+stepName+" unknown")
0235             if len(stepParts)==1:
0236                 self.stepMap[stepName]=""
0237             elif len(stepParts)==2:
0238                 self.stepMap[stepName]=stepParts[1].split('+')
0239             elif len(stepParts)==3:
0240                 self.stepMap[stepName]=(stepParts[2].split('+'),stepParts[1])
0241             else:
0242                 raise ValueError("Step definition "+step+" invalid")
0243             self.stepKeys.append(stepName)
0244
0245         #print "map of steps is:",self.stepMap
0246
0247         self.with_output = with_output
0248         self.process=process
0249
0250         if hasattr(self._options,"no_output_flag") and self._options.no_output_flag:
0251             self.with_output = False
0252         self.with_input = with_input
0253         self.imports = []
0254         self.create_process()
0255         self.define_Configs()
0256         self.schedule = list()
0257         self.scheduleIndexOfFirstHLTPath = None
0258
0259         # we are doing three things here:
0260         # creating a process to catch errors
0261         # building the code to re-create the process
0262
0263         self.additionalCommands = []
0264         # TODO: maybe a list of to be dumped objects would help as well
0265         self.blacklist_paths = []
0266         self.addedObjects = []
0267         self.additionalOutputs = {}
0268
0269         self.productionFilterSequence = None
0270         self.labelsToAssociate=[]
0271         self.nextScheduleIsConditional=False
0272         self.conditionalPaths=[]
0273         self.excludedPaths=[]
0274
0275     def profileOptions(self):
0276         """
0277         addIgProfService
0278         Function to add the igprof profile service so that you can dump in the middle
0279         of the run.
0280         """
0281         profileOpts = self._options.profile.split(':')
0282         profilerStart = 1
0283         profilerInterval = 100
0284         profilerFormat = None
0285         profilerJobFormat = None
0286
0287         if len(profileOpts):
0288             #type, given as first argument is unused here
0289             profileOpts.pop(0)
0290         if len(profileOpts):
0291             startEvent = profileOpts.pop(0)
0292             if not startEvent.isdigit():
0293                 raise Exception("%s is not a number" % startEvent)
0294             profilerStart = int(startEvent)
0295         if len(profileOpts):
0296             eventInterval = profileOpts.pop(0)
0297             if not eventInterval.isdigit():
0298                 raise Exception("%s is not a number" % eventInterval)
0299             profilerInterval = int(eventInterval)
0300         if len(profileOpts):
0301             profilerFormat = profileOpts.pop(0)
0302
0303
0304         if not profilerFormat:
0305             profilerFormat = "%s___%s___%%I.gz" % (
0306                 self._options.evt_type.replace("_cfi", ""),
0307                 hashlib.md5(
0308                     (str(self._options.step) + str(self._options.pileup) + str(self._options.conditions) +
0309                     str(self._options.datatier) + str(self._options.profileTypeLabel)).encode('utf-8')
0310                 ).hexdigest()
0311             )
0312         if not profilerJobFormat and profilerFormat.endswith(".gz"):
0313             profilerJobFormat = profilerFormat.replace(".gz", "_EndOfJob.gz")
0314         elif not profilerJobFormat:
0315             profilerJobFormat = profilerFormat + "_EndOfJob.gz"
0316
0317         return (profilerStart,profilerInterval,profilerFormat,profilerJobFormat)
0318
0319     def load(self,includeFile):
0320         includeFile = includeFile.replace('/','.')
0321         self.process.load(includeFile)
0322         return sys.modules[includeFile]
0323
0324     def loadAndRemember(self, includeFile):
0325         """helper routine to load am memorize imports"""
0326         # we could make the imports a on-the-fly data method of the process instance itself
0327         # not sure if the latter is a good idea
0328         includeFile = includeFile.replace('/','.')
0329         self.imports.append(includeFile)
0330         self.process.load(includeFile)
0331         return sys.modules[includeFile]
0332
0333     def executeAndRemember(self, command):
0334         """helper routine to remember replace statements"""
0335         self.additionalCommands.append(command)
0336         if not command.strip().startswith("#"):
0337         # substitute: process.foo = process.bar -> self.process.foo = self.process.bar
0338             import re
0339             exec(re.sub(r"([^a-zA-Z_0-9]|^)(process)([^a-zA-Z_0-9])",r"\1self.process\3",command))
0340             #exec(command.replace("process.","self.process."))
0341
0342     def addCommon(self):
0343         if 'HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys():
0344             self.process.options.Rethrow = ['ProductNotFound']
0345             self.process.options.fileMode = 'FULLMERGE'
0346
0347         self.addedObjects.append(("","options"))
0348
0349         if self._options.lazy_download:
0350             self.process.AdaptorConfig = cms.Service("AdaptorConfig",
0351                                                      stats = cms.untracked.bool(True),
0352                                                      enable = cms.untracked.bool(True),
0353                                                      cacheHint = cms.untracked.string("lazy-download"),
0354                                                      readHint = cms.untracked.string("read-ahead-buffered")
0355                                                      )
0356             self.addedObjects.append(("Setup lazy download","AdaptorConfig"))
0357
0358         #self.process.cmsDriverCommand = cms.untracked.PSet( command=cms.untracked.string('cmsDriver.py '+self._options.arguments) )
0359         #self.addedObjects.append(("what cmsDriver command was used","cmsDriverCommand"))
0360
0361         if self._options.profile:
0362             (start, interval, eventFormat, jobFormat)=self.profileOptions()
0363             self.process.IgProfService = cms.Service("IgProfService",
0364                                                      reportFirstEvent            = cms.untracked.int32(start),
0365                                                      reportEventInterval         = cms.untracked.int32(interval),
0366                                                      reportToFileAtPostEvent     = cms.untracked.string("| gzip -c > %s"%(eventFormat)),
0367                                                      reportToFileAtPostEndJob    = cms.untracked.string("| gzip -c > %s"%(jobFormat)))
0368             self.addedObjects.append(("Setup IGProf Service for profiling","IgProfService"))
0369
0370     def addMaxEvents(self):
0371         """Here we decide how many evts will be processed"""
0372         self.process.maxEvents.input = int(self._options.number)
0373         if self._options.number_out:
0374             self.process.maxEvents.output = int(self._options.number_out)
0375         self.addedObjects.append(("","maxEvents"))
0376
0377     def addSource(self):
0378         """Here the source is built. Priority: file, generator"""
0379         self.addedObjects.append(("Input source","source"))
0380
0381         def filesFromOption(self):
0382             for entry in self._options.filein.split(','):
0383                 print("entry",entry)
0384                 if entry.startswith("filelist:"):
0385                     filesFromList(entry[9:],self.process.source)
0386                 elif entry.startswith("dbs:") or entry.startswith("das:"):
0387                     filesFromDASQuery('file dataset = %s'%(entry[4:]),self._options.dasoption,self.process.source)
0388                 else:
0389                     self.process.source.fileNames.append(self._options.dirin+entry)
0390             if self._options.secondfilein:
0391                 if not hasattr(self.process.source,"secondaryFileNames"):
0392                     raise Exception("--secondfilein not compatible with "+self._options.filetype+"input type")
0393                 for entry in self._options.secondfilein.split(','):
0394                     print("entry",entry)
0395                     if entry.startswith("filelist:"):
0396                         self.process.source.secondaryFileNames.extend((filesFromList(entry[9:]))[0])
0397                     elif entry.startswith("dbs:") or entry.startswith("das:"):
0398                         self.process.source.secondaryFileNames.extend((filesFromDASQuery('file dataset = %s'%(entry[4:]),self._options.dasoption))[0])
0399                     else:
0400                         self.process.source.secondaryFileNames.append(self._options.dirin+entry)
0401
0402         if self._options.filein or self._options.dasquery:
0403             if self._options.filetype == "EDM":
0404                 self.process.source=cms.Source("PoolSource",
0405                                                fileNames = cms.untracked.vstring(),
0406                                                secondaryFileNames= cms.untracked.vstring())
0407                 filesFromOption(self)
0408             elif self._options.filetype == "DAT":
0409                 self.process.source=cms.Source("NewEventStreamFileReader",fileNames = cms.untracked.vstring())
0410                 filesFromOption(self)
0411             elif self._options.filetype == "LHE":
0412                 self.process.source=cms.Source("LHESource", fileNames = cms.untracked.vstring())
0413                 if self._options.filein.startswith("lhe:"):
0414                     #list the article directory automatically
0415                     args=self._options.filein.split(':')
0416                     article=args[1]
0417                     print('LHE input from article ',article)
0418                     location='/store/lhe/'
0419                     import os
0420                     textOfFiles=os.popen('cmsLHEtoEOSManager.py -l '+article)
0421                     for line in textOfFiles:
0422                         for fileName in [x for x in line.split() if '.lhe' in x]:
0423                             self.process.source.fileNames.append(location+article+'/'+fileName)
0424                     #check first if list of LHE files is loaded (not empty)
0425                     if len(line)<2:
0426                         print('Issue to load LHE files, please check and try again.')
0427                         sys.exit(-1)
0428                     #Additional check to protect empty fileNames in process.source
0429                     if len(self.process.source.fileNames)==0:
0430                         print('Issue with empty filename, but can pass line check')
0431                         sys.exit(-1)
0432                     if len(args)>2:
0433                         self.process.source.skipEvents = cms.untracked.uint32(int(args[2]))
0434                 else:
0435                     filesFromOption(self)
0436
0437             elif self._options.filetype == "DQM":
0438                 self.process.source=cms.Source("DQMRootSource",
0439                                                fileNames = cms.untracked.vstring())
0440                 filesFromOption(self)
0441
0442             elif self._options.filetype == "DQMDAQ":
0443                 # FIXME: how to configure it if there are no input files specified?
0444                 self.process.source=cms.Source("DQMStreamerReader")
0445
0446
0447             if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"):
0448                 self.process.source.processingMode = cms.untracked.string("RunsAndLumis")
0449
0450         if self._options.dasquery!='':
0451             self.process.source=cms.Source("PoolSource", fileNames = cms.untracked.vstring(),secondaryFileNames = cms.untracked.vstring())
0452             filesFromDASQuery(self._options.dasquery,self._options.dasoption,self.process.source)
0453
0454             if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"):
0455                 self.process.source.processingMode = cms.untracked.string("RunsAndLumis")
0456
0457         ##drop LHEXMLStringProduct on input to save memory if appropriate
0458         if 'GEN' in self.stepMap.keys() and not self._options.filetype == "LHE":
0459             if self._options.inputCommands:
0460                 self._options.inputCommands+=',drop LHEXMLStringProduct_*_*_*,'
0461             else:
0462                 self._options.inputCommands='keep *, drop LHEXMLStringProduct_*_*_*,'
0463
0464         if self.process.source and self._options.inputCommands and not self._options.filetype == "LHE":
0465             if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring()
0466             for command in self._options.inputCommands.split(','):
0467                 # remove whitespace around the keep/drop statements
0468                 command = command.strip()
0469                 if command=='': continue
0470                 self.process.source.inputCommands.append(command)
0471             if not self._options.dropDescendant:
0472                 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False)
0473
0474         if self._options.lumiToProcess:
0475             import FWCore.PythonUtilities.LumiList as LumiList
0476             self.process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange( LumiList.LumiList(self._options.lumiToProcess).getCMSSWString().split(',') )
0477
0478         if 'GEN' in self.stepMap.keys() or 'LHE' in self.stepMap or (not self._options.filein and hasattr(self._options, "evt_type")):
0479             if self.process.source is None:
0480                 self.process.source=cms.Source("EmptySource")
0481
0482         # modify source in case of run-dependent MC
0483         self.runsAndWeights=None
0484         if self._options.runsAndWeightsForMC or self._options.runsScenarioForMC :
0485             if not self._options.isMC :
0486                 raise Exception("options --runsAndWeightsForMC and --runsScenarioForMC are only valid for MC")
0487             if self._options.runsAndWeightsForMC:
0488                 self.runsAndWeights = eval(self._options.runsAndWeightsForMC)
0489             else:
0490                 from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights
0491                 if isinstance(RunsAndWeights[self._options.runsScenarioForMC], str):
0492                     __import__(RunsAndWeights[self._options.runsScenarioForMC])
0493                     self.runsAndWeights = sys.modules[RunsAndWeights[self._options.runsScenarioForMC]].runProbabilityDistribution
0494                 else:
0495                     self.runsAndWeights = RunsAndWeights[self._options.runsScenarioForMC]
0496
0497         if self.runsAndWeights:
0498             import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun
0499             ThrowAndSetRandomRun.throwAndSetRandomRun(self.process.source,self.runsAndWeights)
0500             self.additionalCommands.append('import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun')
0501             self.additionalCommands.append('ThrowAndSetRandomRun.throwAndSetRandomRun(process.source,%s)'%(self.runsAndWeights))
0502
0503         # modify source in case of run-dependent MC (Run-3 method)
0504         self.runsAndWeightsInt=None
0505         if self._options.runsAndWeightsForMCIntegerWeights or self._options.runsScenarioForMCIntegerWeights:
0506             if not self._options.isMC :
0507                 raise Exception("options --runsAndWeightsForMCIntegerWeights and --runsScenarioForMCIntegerWeights are only valid for MC")
0508             if self._options.runsAndWeightsForMCIntegerWeights:
0509                 self.runsAndWeightsInt = eval(self._options.runsAndWeightsForMCIntegerWeights)
0510             else:
0511                 from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights
0512                 if isinstance(RunsAndWeights[self._options.runsScenarioForMCIntegerWeights], str):
0513                     __import__(RunsAndWeights[self._options.runsScenarioForMCIntegerWeights])
0514                     self.runsAndWeightsInt = sys.modules[RunsAndWeights[self._options.runsScenarioForMCIntegerWeights]].runProbabilityDistribution
0515                 else:
0516                     self.runsAndWeightsInt = RunsAndWeights[self._options.runsScenarioForMCIntegerWeights]
0517
0518         if self.runsAndWeightsInt:
0519             if not self._options.relval:
0520                 raise Exception("--relval option required when using --runsAndWeightsInt")
0521             if 'DATAMIX' in self._options.step:
0522                 from SimGeneral.Configuration.LumiToRun import lumi_to_run
0523                 total_events, events_per_job  = self._options.relval.split(',')
0524                 lumi_to_run_mapping = lumi_to_run(self.runsAndWeightsInt, int(total_events), int(events_per_job))
0525                 self.additionalCommands.append("process.source.firstLuminosityBlockForEachRun = cms.untracked.VLuminosityBlockID(*[cms.LuminosityBlockID(x,y) for x,y in " + str(lumi_to_run_mapping) + "])")
0526
0527         return
0528
0529     def addOutput(self):
0530         """ Add output module to the process """
0531         result=""
0532         if self._options.outputDefinition:
0533             if self._options.datatier:
0534                 print("--datatier & --eventcontent options ignored")
0535
0536             #new output convention with a list of dict
0537             outList = eval(self._options.outputDefinition)
0538             for (id,outDefDict) in enumerate(outList):
0539                 outDefDictStr=outDefDict.__str__()
0540                 if not isinstance(outDefDict,dict):
0541                     raise Exception("--output needs to be passed a list of dict"+self._options.outputDefinition+" is invalid")
0542                 #requires option: tier
0543                 theTier=anyOf(['t','tier','dataTier'],outDefDict)
0544                 #optional option: eventcontent, filtername, selectEvents, moduleLabel, filename
0545                 ## event content
0546                 theStreamType=anyOf(['e','ec','eventContent','streamType'],outDefDict,theTier)
0547                 theFilterName=anyOf(['f','ftN','filterName'],outDefDict,'')
0548                 theSelectEvent=anyOf(['s','sE','selectEvents'],outDefDict,'')
0549                 theModuleLabel=anyOf(['l','mL','moduleLabel'],outDefDict,'')
0550                 theExtraOutputCommands=anyOf(['o','oC','outputCommands'],outDefDict,'')
0551                 # module label has a particular role
0552                 if not theModuleLabel:
0553                     tryNames=[theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+'output',
0554                               theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+'output',
0555                               theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+theSelectEvent.split(',')[0].replace(':','for').replace(' ','')+'output'
0556                               ]
0557                     for name in tryNames:
0558                         if not hasattr(self.process,name):
0559                             theModuleLabel=name
0560                             break
0561                 if not theModuleLabel:
0562                     raise Exception("cannot find a module label for specification: "+outDefDictStr)
0563                 if id==0:
0564                     defaultFileName=self._options.outfile_name
0565                 else:
0566                     defaultFileName=self._options.outfile_name.replace('.root','_in'+theTier+'.root')
0567
0568                 theFileName=self._options.dirout+anyOf(['fn','fileName'],outDefDict,defaultFileName)
0569                 if not theFileName.endswith('.root'):
0570                     theFileName+='.root'
0571
0572                 if len(outDefDict):
0573                     raise Exception("unused keys from --output options: "+','.join(outDefDict.keys()))
0574                 if theStreamType=='DQMIO': theStreamType='DQM'
0575                 if theStreamType=='ALL':
0576                     theEventContent = cms.PSet(outputCommands = cms.untracked.vstring('keep *'))
0577                 else:
0578                     theEventContent = getattr(self.process, theStreamType+"EventContent")
0579
0580
0581                 addAlCaSelects=False
0582                 if theStreamType=='ALCARECO' and not theFilterName:
0583                     theFilterName='StreamALCACombined'
0584                     addAlCaSelects=True
0585
0586                 CppType='PoolOutputModule'
0587                 if self._options.timeoutOutput:
0588                     CppType='TimeoutPoolOutputModule'
0589                 if theStreamType=='DQM' and theTier=='DQMIO': CppType='DQMRootOutputModule'
0590                 output = cms.OutputModule(CppType,
0591                                           theEventContent.clone(),
0592                                           fileName = cms.untracked.string(theFileName),
0593                                           dataset = cms.untracked.PSet(
0594                                              dataTier = cms.untracked.string(theTier),
0595                                              filterName = cms.untracked.string(theFilterName))
0596                                           )
0597                 if not theSelectEvent and hasattr(self.process,'generation_step') and theStreamType!='LHE':
0598                     output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step'))
0599                 if not theSelectEvent and hasattr(self.process,'filtering_step'):
0600                     output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step'))
0601                 if theSelectEvent:
0602                     output.SelectEvents =cms.untracked.PSet(SelectEvents = cms.vstring(theSelectEvent))
0603
0604                 if addAlCaSelects:
0605                     if not hasattr(output,'SelectEvents'):
0606                         output.SelectEvents=cms.untracked.PSet(SelectEvents=cms.vstring())
0607                     for alca in self.AlCaPaths:
0608                         output.SelectEvents.SelectEvents.extend(getattr(self.process,'OutALCARECO'+alca).SelectEvents.SelectEvents)
0609
0610
0611                 if hasattr(self.process,theModuleLabel):
0612                     raise Exception("the current process already has a module "+theModuleLabel+" defined")
0613                 #print "creating output module ",theModuleLabel
0614                 setattr(self.process,theModuleLabel,output)
0615                 outputModule=getattr(self.process,theModuleLabel)
0616                 setattr(self.process,theModuleLabel+'_step',cms.EndPath(outputModule))
0617                 path=getattr(self.process,theModuleLabel+'_step')
0618                 self.schedule.append(path)
0619
0620                 if not self._options.inlineEventContent and hasattr(self.process,theStreamType+"EventContent"):
0621                     def doNotInlineEventContent(instance,label = "cms.untracked.vstring(process."+theStreamType+"EventContent.outputCommands)"):
0622                         return label
0623                     outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
0624                 if theExtraOutputCommands:
0625                     if not isinstance(theExtraOutputCommands,list):
0626                         raise Exception("extra ouput command in --option must be a list of strings")
0627                     if hasattr(self.process,theStreamType+"EventContent"):
0628                         self.executeAndRemember('process.%s.outputCommands.extend(%s)'%(theModuleLabel,theExtraOutputCommands))
0629                     else:
0630                         outputModule.outputCommands.extend(theExtraOutputCommands)
0631
0632                 result+="\nprocess."+theModuleLabel+" = "+outputModule.dumpPython()
0633
0634             ##ends the --output options model
0635             return result
0636
0637         streamTypes=self._options.eventcontent.split(',')
0638         tiers=self._options.datatier.split(',')
0639         if not self._options.outputDefinition and len(streamTypes)!=len(tiers):
0640             raise Exception("number of event content arguments does not match number of datatier arguments")
0641
0642         # if the only step is alca we don't need to put in an output
0643         if self._options.step.split(',')[0].split(':')[0] == 'ALCA':
0644             return "\n"
0645
0646         for i,(streamType,tier) in enumerate(zip(streamTypes,tiers)):
0647             if streamType=='': continue
0648             if streamType == 'ALCARECO' and not 'ALCAPRODUCER' in self._options.step: continue
0649             if streamType=='DQMIO': streamType='DQM'
0650             eventContent=streamType
0651             ## override streamType to eventContent in case NANOEDM
0652             if streamType == "NANOEDMAOD" :
0653                 eventContent = "NANOAOD"
0654             elif streamType == "NANOEDMAODSIM" :
0655                 eventContent = "NANOAODSIM"
0656             theEventContent = getattr(self.process, eventContent+"EventContent")
0657             if i==0:
0658                 theFileName=self._options.outfile_name
0659                 theFilterName=self._options.filtername
0660             else:
0661                 theFileName=self._options.outfile_name.replace('.root','_in'+streamType+'.root')
0662                 theFilterName=self._options.filtername
0663             CppType='PoolOutputModule'
0664             if self._options.timeoutOutput:
0665                 CppType='TimeoutPoolOutputModule'
0666             if streamType=='DQM' and tier=='DQMIO': CppType='DQMRootOutputModule'
0667             if "NANOAOD" in streamType : CppType='NanoAODOutputModule'
0668             output = cms.OutputModule(CppType,
0669                                       theEventContent,
0670                                       fileName = cms.untracked.string(theFileName),
0671                                       dataset = cms.untracked.PSet(dataTier = cms.untracked.string(tier),
0672                                                                    filterName = cms.untracked.string(theFilterName)
0673                                                                    )
0674                                       )
0675             if hasattr(self.process,"generation_step") and streamType!='LHE':
0676                 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step'))
0677             if hasattr(self.process,"filtering_step"):
0678                 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step'))
0679
0680             if streamType=='ALCARECO':
0681                 output.dataset.filterName = cms.untracked.string('StreamALCACombined')
0682
0683             if "MINIAOD" in streamType:
0684                 from PhysicsTools.PatAlgos.slimming.miniAOD_tools import miniAOD_customizeOutput
0685                 miniAOD_customizeOutput(output)
0686
0687             outputModuleName=streamType+'output'
0688             setattr(self.process,outputModuleName,output)
0689             outputModule=getattr(self.process,outputModuleName)
0690             setattr(self.process,outputModuleName+'_step',cms.EndPath(outputModule))
0691             path=getattr(self.process,outputModuleName+'_step')
0692             self.schedule.append(path)
0693
0694             if self._options.outputCommands and streamType!='DQM':
0695                 for evct in self._options.outputCommands.split(','):
0696                     if not evct: continue
0697                     self.executeAndRemember("process.%s.outputCommands.append('%s')"%(outputModuleName,evct.strip()))
0698
0699             if not self._options.inlineEventContent:
0700                 tmpstreamType=streamType
0701                 if "NANOEDM" in tmpstreamType :
0702                     tmpstreamType=tmpstreamType.replace("NANOEDM","NANO")
0703                 def doNotInlineEventContent(instance,label = "process."+tmpstreamType+"EventContent.outputCommands"):
0704                     return label
0705                 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
0706
0707             result+="\nprocess."+outputModuleName+" = "+outputModule.dumpPython()
0708
0709         return result
0710
0711     def addStandardSequences(self):
0712         """
0713         Add selected standard sequences to the process
0714         """
0715         # load the pile up file
0716         if self._options.pileup:
0717             pileupSpec=self._options.pileup.split(',')[0]
0718
0719             # Does the requested pile-up scenario exist?
0720             from Configuration.StandardSequences.Mixing import Mixing,defineMixing
0721             if not pileupSpec in Mixing and '.' not in pileupSpec and 'file:' not in pileupSpec:
0722                 message = pileupSpec+' is not a know mixing scenario:\n available are: '+'\n'.join(Mixing.keys())
0723                 raise Exception(message)
0724
0725             # Put mixing parameters in a dictionary
0726             if '.' in pileupSpec:
0727                 mixingDict={'file':pileupSpec}
0728             elif pileupSpec.startswith('file:'):
0729                 mixingDict={'file':pileupSpec[5:]}
0730             else:
0731                 import copy
0732                 mixingDict=copy.copy(Mixing[pileupSpec])
0733             if len(self._options.pileup.split(','))>1:
0734                 mixingDict.update(eval(self._options.pileup[self._options.pileup.find(',')+1:]))
0735
0736             # Load the pu cfg file corresponding to the requested pu scenario
0737             if 'file:' in pileupSpec:
0738                 #the file is local
0739                 self.process.load(mixingDict['file'])
0740                 print("inlining mixing module configuration")
0741                 self._options.inlineObjets+=',mix'
0742             else:
0743                 self.loadAndRemember(mixingDict['file'])
0744
0745             mixingDict.pop('file')
0746             if not "DATAMIX" in self.stepMap.keys(): # when DATAMIX is present, pileup_input refers to pre-mixed GEN-RAW
0747                 if self._options.pileup_input:
0748                     if self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:'):
0749                         mixingDict['F']=filesFromDASQuery('file dataset = %s'%(self._options.pileup_input[4:],),self._options.pileup_dasoption)[0]
0750                     elif self._options.pileup_input.startswith("filelist:"):
0751                         mixingDict['F']=(filesFromList(self._options.pileup_input[9:]))[0]
0752                     else:
0753                         mixingDict['F']=self._options.pileup_input.split(',')
0754                 specialization=defineMixing(mixingDict)
0755                 for command in specialization:
0756                     self.executeAndRemember(command)
0757                 if len(mixingDict)!=0:
0758                     raise Exception('unused mixing specification: '+mixingDict.keys().__str__())
0759
0760
0761         # load the geometry file
0762         try:
0763             if len(self.stepMap):
0764                 self.loadAndRemember(self.GeometryCFF)
0765                 if ('SIM' in self.stepMap or 'reSIM' in self.stepMap) and not self._options.fast:
0766                     self.loadAndRemember(self.SimGeometryCFF)
0767                     if self.geometryDBLabel:
0768                         self.executeAndRemember('if hasattr(process, "XMLFromDBSource"): process.XMLFromDBSource.label="%s"'%(self.geometryDBLabel))
0769                         self.executeAndRemember('if hasattr(process, "DDDetectorESProducerFromDB"): process.DDDetectorESProducerFromDB.label="%s"'%(self.geometryDBLabel))
0770
0771         except ImportError:
0772             print("Geometry option",self._options.geometry,"unknown.")
0773             raise
0774
0775         if len(self.stepMap):
0776             self.loadAndRemember(self.magFieldCFF)
0777
0778         for stepName in self.stepKeys:
0779             stepSpec = self.stepMap[stepName]
0780             print("Step:", stepName,"Spec:",stepSpec)
0781             if stepName.startswith('re'):
0782                 ##add the corresponding input content
0783                 if stepName[2:] not in self._options.donotDropOnInput:
0784                     self._options.inputEventContent='%s,%s'%(stepName.upper(),self._options.inputEventContent)
0785                 stepName=stepName[2:]
0786             if stepSpec=="":
0787                 getattr(self,"prepare_"+stepName)(sequence = getattr(self,stepName+"DefaultSeq"))
0788             elif isinstance(stepSpec, list):
0789                 getattr(self,"prepare_"+stepName)(sequence = '+'.join(stepSpec))
0790             elif isinstance(stepSpec, tuple):
0791                 getattr(self,"prepare_"+stepName)(sequence = ','.join([stepSpec[1],'+'.join(stepSpec[0])]))
0792             else:
0793                 raise ValueError("Invalid step definition")
0794
0795         if self._options.restoreRNDSeeds!=False:
0796             #it is either True, or a process name
0797             if self._options.restoreRNDSeeds==True:
0798                 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateLabel=cms.untracked.string("randomEngineStateProducer")')
0799             else:
0800                 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateTag=cms.untracked.InputTag("randomEngineStateProducer","","%s")'%(self._options.restoreRNDSeeds))
0801             if self._options.inputEventContent or self._options.inputCommands:
0802                 if self._options.inputCommands:
0803                     self._options.inputCommands+='keep *_randomEngineStateProducer_*_*,'
0804                 else:
0805                     self._options.inputCommands='keep *_randomEngineStateProducer_*_*,'
0806
0807
0808     def completeInputCommand(self):
0809         if self._options.inputEventContent:
0810             import copy
0811             def dropSecondDropStar(iec):
0812                 #drop occurence of 'drop *' in the list
0813                 count=0
0814                 for item in iec:
0815                     if item=='drop *':
0816                         if count!=0:
0817                             iec.remove(item)
0818                         count+=1
0819
0820             ## allow comma separated input eventcontent
0821             if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring()
0822             for evct in self._options.inputEventContent.split(','):
0823                 if evct=='': continue
0824                 theEventContent = getattr(self.process, evct+"EventContent")
0825                 if hasattr(theEventContent,'outputCommands'):
0826                     self.process.source.inputCommands.extend(copy.copy(theEventContent.outputCommands))
0827                 if hasattr(theEventContent,'inputCommands'):
0828                     self.process.source.inputCommands.extend(copy.copy(theEventContent.inputCommands))
0829
0830             dropSecondDropStar(self.process.source.inputCommands)
0831
0832             if not self._options.dropDescendant:
0833                 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False)
0834
0835
0836         return
0837
0838     def addConditions(self):
0839         """Add conditions to the process"""
0840         if not self._options.conditions: return
0841
0842         if 'FrontierConditions_GlobalTag' in self._options.conditions:
0843             print('using FrontierConditions_GlobalTag in --conditions is not necessary anymore and will be deprecated soon. please update your command line')
0844             self._options.conditions = self._options.conditions.replace("FrontierConditions_GlobalTag,",'')
0845
0846         self.loadAndRemember(self.ConditionsDefaultCFF)
0847         from Configuration.AlCa.GlobalTag import GlobalTag
0848         self.process.GlobalTag = GlobalTag(self.process.GlobalTag, self._options.conditions, self._options.custom_conditions)
0849         self.additionalCommands.append('from Configuration.AlCa.GlobalTag import GlobalTag')
0850         self.additionalCommands.append('process.GlobalTag = GlobalTag(process.GlobalTag, %s, %s)' % (repr(self._options.conditions), repr(self._options.custom_conditions)))
0851
0852
0853     def addCustomise(self,unsch=0):
0854         """Include the customise code """
0855
0856         custOpt=[]
0857         if unsch==0:
0858             for c in self._options.customisation_file:
0859                 custOpt.extend(c.split(","))
0860         else:
0861             for c in self._options.customisation_file_unsch:
0862                 custOpt.extend(c.split(","))
0863
0864         custMap=DictTypes.SortedKeysDict()
0865         for opt in custOpt:
0866             if opt=='': continue
0867             if opt.count('.')>1:
0868                 raise Exception("more than . in the specification:"+opt)
0869             fileName=opt.split('.')[0]
0870             if opt.count('.')==0:   rest='customise'
0871             else:
0872                 rest=opt.split('.')[1]
0873                 if rest=='py': rest='customise' #catch the case of --customise file.py
0874
0875             if fileName in custMap:
0876                 custMap[fileName].extend(rest.split('+'))
0877             else:
0878                 custMap[fileName]=rest.split('+')
0879
0880         if len(custMap)==0:
0881             final_snippet='\n'
0882         else:
0883             final_snippet='\n# customisation of the process.\n'
0884
0885         allFcn=[]
0886         for opt in custMap:
0887             allFcn.extend(custMap[opt])
0888         for fcn in allFcn:
0889             if allFcn.count(fcn)!=1:
0890                 raise Exception("cannot specify twice "+fcn+" as a customisation method")
0891
0892         for f in custMap:
0893             # let python search for that package and do syntax checking at the same time
0894             packageName = f.replace(".py","").replace("/",".")
0895             __import__(packageName)
0896             package = sys.modules[packageName]
0897
0898             # now ask the package for its definition and pick .py instead of .pyc
0899             customiseFile = re.sub(r'\.pyc$', '.py', package.__file__)
0900
0901             final_snippet+='\n# Automatic addition of the customisation function from '+packageName+'\n'
0902             if self._options.inline_custom:
0903                 for line in file(customiseFile,'r'):
0904                     if "import FWCore.ParameterSet.Config" in line:
0905                         continue
0906                     final_snippet += line
0907             else:
0908                 final_snippet += 'from %s import %s \n'%(packageName,','.join(custMap[f]))
0909             for fcn in custMap[f]:
0910                 print("customising the process with",fcn,"from",f)
0911                 if not hasattr(package,fcn):
0912                     #bound to fail at run time
0913                     raise Exception("config "+f+" has no function "+fcn)
0914                 #execute the command
0915                 self.process=getattr(package,fcn)(self.process)
0916                 #and print it in the configuration
0917                 final_snippet += "\n#call to customisation function "+fcn+" imported from "+packageName
0918                 final_snippet += "\nprocess = %s(process)\n"%(fcn,)
0919
0920         if len(custMap)!=0:
0921             final_snippet += '\n# End of customisation functions\n'
0922
0923         ### now for a useful command
0924         return final_snippet
0925
0926     def addCustomiseCmdLine(self):
0927         final_snippet='\n# Customisation from command line\n'
0928         if self._options.customise_commands:
0929             import string
0930             for com in self._options.customise_commands.split('\\n'):
0931                 com=com.lstrip()
0932                 self.executeAndRemember(com)
0933                 final_snippet +='\n'+com
0934
0935         return final_snippet
0936
0937     #----------------------------------------------------------------------------
0938     # here the methods to define the python includes for each step or
0939     # conditions
0940     #----------------------------------------------------------------------------
0941     def define_Configs(self):
0942         if len(self.stepMap):
0943             self.loadAndRemember('Configuration/StandardSequences/Services_cff')
0944         if self._options.particleTable not in defaultOptions.particleTableList:
0945             print('Invalid particle table provided. Options are:')
0946             print(defaultOptions.particleTable)
0947             sys.exit(-1)
0948         else:
0949             if len(self.stepMap):
0950                 self.loadAndRemember('SimGeneral.HepPDTESSource.'+self._options.particleTable+'_cfi')
0951
0952         self.loadAndRemember('FWCore/MessageService/MessageLogger_cfi')
0953
0954         self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreams_cff"
0955         self.GENDefaultCFF="Configuration/StandardSequences/Generator_cff"
0956         self.SIMDefaultCFF="Configuration/StandardSequences/Sim_cff"
0957         self.DIGIDefaultCFF="Configuration/StandardSequences/Digi_cff"
0958         self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRaw_cff"
0959         self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1Emulator_cff'
0960         self.L1MENUDefaultCFF="Configuration/StandardSequences/L1TriggerDefaultMenu_cff"
0961         self.HLTDefaultCFF="Configuration/StandardSequences/HLTtable_cff"
0962         self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_Data_cff"
0963         if self._options.isRepacked: self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_DataMapper_cff"
0964         self.L1RecoDefaultCFF="Configuration/StandardSequences/L1Reco_cff"
0965         self.L1TrackTriggerDefaultCFF="Configuration/StandardSequences/L1TrackTrigger_cff"
0966         self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_Data_cff"
0967         self.RECOSIMDefaultCFF="Configuration/StandardSequences/RecoSim_cff"
0968         self.PATDefaultCFF="Configuration/StandardSequences/PAT_cff"
0969         self.NANODefaultCFF="PhysicsTools/NanoAOD/nano_cff"
0970         self.NANOGENDefaultCFF="PhysicsTools/NanoAOD/nanogen_cff"
0971         self.SKIMDefaultCFF="Configuration/StandardSequences/Skims_cff"
0972         self.POSTRECODefaultCFF="Configuration/StandardSequences/PostRecoGenerator_cff"
0973         self.VALIDATIONDefaultCFF="Configuration/StandardSequences/Validation_cff"
0974         self.L1HwValDefaultCFF = "Configuration/StandardSequences/L1HwVal_cff"
0975         self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOffline_cff"
0976         self.HARVESTINGDefaultCFF="Configuration/StandardSequences/Harvesting_cff"
0977         self.ALCAHARVESTDefaultCFF="Configuration/StandardSequences/AlCaHarvesting_cff"
0978         self.ENDJOBDefaultCFF="Configuration/StandardSequences/EndOfProcess_cff"
0979         self.ConditionsDefaultCFF = "Configuration/StandardSequences/FrontierConditions_GlobalTag_cff"
0980         self.CFWRITERDefaultCFF = "Configuration/StandardSequences/CrossingFrameWriter_cff"
0981         self.REPACKDefaultCFF="Configuration/StandardSequences/DigiToRaw_Repack_cff"
0982
0983         if "DATAMIX" in self.stepMap.keys():
0984             self.DATAMIXDefaultCFF="Configuration/StandardSequences/DataMixer"+self._options.datamix+"_cff"
0985             self.DIGIDefaultCFF="Configuration/StandardSequences/DigiDM_cff"
0986             self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRawDM_cff"
0987             self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1EmulatorDM_cff'
0988
0989         self.ALCADefaultSeq=None
0990         self.LHEDefaultSeq='externalLHEProducer'
0991         self.GENDefaultSeq='pgen'
0992         self.SIMDefaultSeq='psim'
0993         self.DIGIDefaultSeq='pdigi'
0994         self.DATAMIXDefaultSeq=None
0995         self.DIGI2RAWDefaultSeq='DigiToRaw'
0996         self.HLTDefaultSeq='GRun'
0997         self.L1DefaultSeq=None
0998         self.L1REPACKDefaultSeq='GT'
0999         self.HARVESTINGDefaultSeq=None
1000         self.ALCAHARVESTDefaultSeq=None
1001         self.CFWRITERDefaultSeq=None
1002         self.RAW2DIGIDefaultSeq='RawToDigi'
1003         self.L1RecoDefaultSeq='L1Reco'
1004         self.L1TrackTriggerDefaultSeq='L1TrackTrigger'
1005         if self._options.fast or ('RAW2DIGI' in self.stepMap and 'RECO' in self.stepMap):
1006             self.RECODefaultSeq='reconstruction'
1007         else:
1008             self.RECODefaultSeq='reconstruction_fromRECO'
1009         self.RECOSIMDefaultSeq='recosim'
1010         self.POSTRECODefaultSeq=None
1011         self.L1HwValDefaultSeq='L1HwVal'
1012         self.DQMDefaultSeq='DQMOffline'
1013         self.VALIDATIONDefaultSeq=''
1014         self.ENDJOBDefaultSeq='endOfProcess'
1015         self.REPACKDefaultSeq='DigiToRawRepack'
1016         self.PATDefaultSeq='miniAOD'
1017         self.PATGENDefaultSeq='miniGEN'
1018         #TODO: Check based of file input
1019         self.NANOGENDefaultSeq='nanogenSequence'
1020         self.NANODefaultSeq='nanoSequence'
1021
1022         self.EVTCONTDefaultCFF="Configuration/EventContent/EventContent_cff"
1023
1024         if not self._options.beamspot:
1025             self._options.beamspot=VtxSmearedDefaultKey
1026
1027         # if its MC then change the raw2digi
1028         if self._options.isMC==True:
1029             self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_cff"
1030             self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_cff"
1031             self.PATDefaultCFF="Configuration/StandardSequences/PATMC_cff"
1032             self.PATGENDefaultCFF="Configuration/StandardSequences/PATGEN_cff"
1033             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineMC_cff"
1034             self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreamsMC_cff"
1035             self.NANODefaultSeq='nanoSequenceMC'
1036         else:
1037             self._options.beamspot = None
1038
1039         #patch for gen, due to backward incompatibility
1040         if 'reGEN' in self.stepMap:
1041             self.GENDefaultSeq='fixGenInfo'
1042
1043         if self._options.scenario=='cosmics':
1044             self._options.pileup='Cosmics'
1045             self.DIGIDefaultCFF="Configuration/StandardSequences/DigiCosmics_cff"
1046             self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionCosmics_cff"
1047             self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsCosmics_cff"
1048             self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentCosmics_cff"
1049             self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationCosmics_cff"
1050             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmics_cff"
1051             if self._options.isMC==True:
1052                 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmicsMC_cff"
1053             self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingCosmics_cff"
1054             self.RECODefaultSeq='reconstructionCosmics'
1055             self.DQMDefaultSeq='DQMOfflineCosmics'
1056
1057         if self._options.scenario=='HeavyIons':
1058             if not self._options.beamspot:
1059                 self._options.beamspot=VtxSmearedHIDefaultKey
1060             self.HLTDefaultSeq = 'HIon'
1061             self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationHeavyIons_cff"
1062             self.VALIDATIONDefaultSeq=''
1063             self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentHeavyIons_cff"
1064             self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionHeavyIons_cff"
1065             self.RECODefaultSeq='reconstructionHeavyIons'
1066             self.ALCADefaultCFF = "Configuration/StandardSequences/AlCaRecoStreamsHeavyIons_cff"
1067             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIons_cff"
1068             self.DQMDefaultSeq='DQMOfflineHeavyIons'
1069             self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsHeavyIons_cff"
1070             self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingHeavyIons_cff"
1071             if self._options.isMC==True:
1072                 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIonsMC_cff"
1073
1074
1075         self.RAW2RECODefaultSeq=','.join([self.RAW2DIGIDefaultSeq,self.RECODefaultSeq])
1076
1077         self.USERDefaultSeq='user'
1078         self.USERDefaultCFF=None
1079
1080         # the magnetic field
1081         self.magFieldCFF = 'Configuration/StandardSequences/MagneticField_'+self._options.magField.replace('.','')+'_cff'
1082         self.magFieldCFF = self.magFieldCFF.replace("__",'_')
1083
1084         # the geometry
1085         self.GeometryCFF='Configuration/StandardSequences/GeometryRecoDB_cff'
1086         self.geometryDBLabel=None
1087         simGeometry=''
1088         if self._options.fast:
1089             if 'start' in self._options.conditions.lower():
1090                 self.GeometryCFF='FastSimulation/Configuration/Geometries_START_cff'
1091             else:
1092                 self.GeometryCFF='FastSimulation/Configuration/Geometries_MC_cff'
1093         else:
1094             def inGeometryKeys(opt):
1095                 from Configuration.StandardSequences.GeometryConf import GeometryConf
1096                 if opt in GeometryConf:
1097                     return GeometryConf[opt]
1098                 else:
1099                     return opt
1100
1101             geoms=self._options.geometry.split(',')
1102             if len(geoms)==1: geoms=inGeometryKeys(geoms[0]).split(',')
1103             if len(geoms)==2:
1104                 #may specify the reco geometry
1105                 if '/' in geoms[1] or '_cff' in geoms[1]:
1106                     self.GeometryCFF=geoms[1]
1107                 else:
1108                     self.GeometryCFF='Configuration/Geometry/Geometry'+geoms[1]+'_cff'
1109
1110             if (geoms[0].startswith('DB:')):
1111                 self.SimGeometryCFF='Configuration/StandardSequences/GeometrySimDB_cff'
1112                 self.geometryDBLabel=geoms[0][3:]
1113                 print("with DB:")
1114             else:
1115                 if '/' in geoms[0] or '_cff' in geoms[0]:
1116                     self.SimGeometryCFF=geoms[0]
1117                 else:
1118                     simGeometry=geoms[0]
1119                     if self._options.gflash==True:
1120                         self.SimGeometryCFF='Configuration/Geometry/Geometry'+geoms[0]+'GFlash_cff'
1121                     else:
1122                         self.SimGeometryCFF='Configuration/Geometry/Geometry'+geoms[0]+'_cff'
1123
1124         # synchronize the geometry configuration and the FullSimulation sequence to be used
1125         if simGeometry not in defaultOptions.geometryExtendedOptions:
1126             self.SIMDefaultCFF="Configuration/StandardSequences/SimIdeal_cff"
1127
1128         if self._options.scenario=='nocoll' or self._options.scenario=='cosmics':
1129             self.SIMDefaultCFF="Configuration/StandardSequences/SimNOBEAM_cff"
1130             self._options.beamspot='NoSmear'
1131
1132         # fastsim requires some changes to the default cff files and sequences
1133         if self._options.fast:
1134             self.SIMDefaultCFF = 'FastSimulation.Configuration.SimIdeal_cff'
1135             self.RECODefaultCFF= 'FastSimulation.Configuration.Reconstruction_AftMix_cff'
1136             self.RECOBEFMIXDefaultCFF = 'FastSimulation.Configuration.Reconstruction_BefMix_cff'
1137             self.RECOBEFMIXDefaultSeq = 'reconstruction_befmix'
1138             self.NANODefaultSeq = 'nanoSequenceFS'
1139             self.DQMOFFLINEDefaultCFF="DQMOffline.Configuration.DQMOfflineFS_cff"
1140
1141         # Mixing
1142         if self._options.pileup=='default':
1143             from Configuration.StandardSequences.Mixing import MixingDefaultKey
1144             self._options.pileup=MixingDefaultKey
1145
1146
1147         #not driven by a default cff anymore
1148         if self._options.isData:
1149             self._options.pileup=None
1150
1151
1152         self.REDIGIDefaultSeq=self.DIGIDefaultSeq
1153
1154     # for alca, skims, etc
1155     def addExtraStream(self, name, stream, workflow='full'):
1156             # define output module and go from there
1157         output = cms.OutputModule("PoolOutputModule")
1158         if stream.selectEvents.parameters_().__len__()!=0:
1159             output.SelectEvents = stream.selectEvents
1160         else:
1161             output.SelectEvents = cms.untracked.PSet()
1162             output.SelectEvents.SelectEvents=cms.vstring()
1163             if isinstance(stream.paths,tuple):
1164                 for path in stream.paths:
1165                     output.SelectEvents.SelectEvents.append(path.label())
1166             else:
1167                 output.SelectEvents.SelectEvents.append(stream.paths.label())
1168
1169
1170
1171         if isinstance(stream.content,str):
1172             evtPset=getattr(self.process,stream.content)
1173             for p in evtPset.parameters_():
1174                 setattr(output,p,getattr(evtPset,p))
1175             if not self._options.inlineEventContent:
1176                 def doNotInlineEventContent(instance,label = "process."+stream.content+".outputCommands"):
1177                     return label
1178                 output.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
1179         else:
1180             output.outputCommands = stream.content
1181
1182
1183         output.fileName = cms.untracked.string(self._options.dirout+stream.name+'.root')
1184
1185         output.dataset  = cms.untracked.PSet( dataTier = stream.dataTier,
1186                                               filterName = cms.untracked.string(stream.name))
1187
1188         if self._options.filtername:
1189             output.dataset.filterName= cms.untracked.string(self._options.filtername+"_"+stream.name)
1190
1191         #add an automatic flushing to limit memory consumption
1192         output.eventAutoFlushCompressedSize=cms.untracked.int32(5*1024*1024)
1193
1194         if workflow in ("producers,full"):
1195             if isinstance(stream.paths,tuple):
1196                 for path in stream.paths:
1197                     self.schedule.append(path)
1198             else:
1199                 self.schedule.append(stream.paths)
1200
1201
1202         # in case of relvals we don't want to have additional outputs
1203         if (not self._options.relval) and workflow in ("full","output"):
1204             self.additionalOutputs[name] = output
1205             setattr(self.process,name,output)
1206
1207         if workflow == 'output':
1208             # adjust the select events to the proper trigger results from previous process
1209             filterList = output.SelectEvents.SelectEvents
1210             for i, filter in enumerate(filterList):
1211                 filterList[i] = filter+":"+self._options.triggerResultsProcess
1212
1213         return output
1214
1215     #----------------------------------------------------------------------------
1216     # here the methods to create the steps. Of course we are doing magic here ;)
1217     # prepare_STEPNAME modifies self.process and what else's needed.
1218     #----------------------------------------------------------------------------
1219
1220     def loadDefaultOrSpecifiedCFF(self, sequence,defaultCFF):
1221         if ( len(sequence.split('.'))==1 ):
1222             l=self.loadAndRemember(defaultCFF)
1223         elif ( len(sequence.split('.'))==2 ):
1224             l=self.loadAndRemember(sequence.split('.')[0])
1225             sequence=sequence.split('.')[1]
1226         else:
1227             print("sub sequence configuration must be of the form dir/subdir/cff.a+b+c or cff.a")
1228             print(sequence,"not recognized")
1229             raise
1230         return l
1231
1232     def scheduleSequence(self,seq,prefix,what='Path'):
1233         if '*' in seq:
1234             #create only one path with all sequences in it
1235             for i,s in enumerate(seq.split('*')):
1236                 if i==0:
1237                     setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, s) ))
1238                 else:
1239                     p=getattr(self.process,prefix)
1240                     tmp = getattr(self.process, s)
1241                     if isinstance(tmp, cms.Task):
1242                         p.associate(tmp)
1243                     else:
1244                         p+=tmp
1245             self.schedule.append(getattr(self.process,prefix))
1246             return
1247         else:
1248             #create as many path as many sequences
1249             if not '+' in seq:
1250                 if self.nextScheduleIsConditional:
1251                     self.conditionalPaths.append(prefix)
1252                 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, seq) ))
1253                 self.schedule.append(getattr(self.process,prefix))
1254             else:
1255                 for i,s in enumerate(seq.split('+')):
1256                     sn=prefix+'%d'%(i)
1257                     setattr(self.process,sn,getattr(cms,what)( getattr(self.process, s) ))
1258                     self.schedule.append(getattr(self.process,sn))
1259             return
1260
1261     def scheduleSequenceAtEnd(self,seq,prefix):
1262         self.scheduleSequence(seq,prefix,what='EndPath')
1263         return
1264
1265     def prepare_ALCAPRODUCER(self, sequence = None):
1266         self.prepare_ALCA(sequence, workflow = "producers")
1267
1268     def prepare_ALCAOUTPUT(self, sequence = None):
1269         self.prepare_ALCA(sequence, workflow = "output")
1270
1271     def prepare_ALCA(self, sequence = None, workflow = 'full'):
1272         """ Enrich the process with alca streams """
1273         alcaConfig=self.loadDefaultOrSpecifiedCFF(sequence,self.ALCADefaultCFF)
1274         sequence = sequence.split('.')[-1]
1275
1276         # decide which ALCA paths to use
1277         alcaList = sequence.split("+")
1278         maxLevel=0
1279         from Configuration.AlCa.autoAlca import autoAlca, AlCaNoConcurrentLumis
1280         # support @X from autoAlca.py, and recursion support: i.e T0:@Mu+@EG+...
1281         self.expandMapping(alcaList,autoAlca)
1282         self.AlCaPaths=[]
1283         for name in alcaConfig.__dict__:
1284             alcastream = getattr(alcaConfig,name)
1285             shortName = name.replace('ALCARECOStream','')
1286             if shortName in alcaList and isinstance(alcastream,cms.FilteredStream):
1287                 if shortName in AlCaNoConcurrentLumis:
1288                     print("Setting numberOfConcurrentLuminosityBlocks=1 because of AlCa sequence {}".format(shortName))
1289                     self._options.nConcurrentLumis = "1"
1290                     self._options.nConcurrentIOVs = "1"
1291                 output = self.addExtraStream(name,alcastream, workflow = workflow)
1292                 self.executeAndRemember('process.ALCARECOEventContent.outputCommands.extend(process.OutALCARECO'+shortName+'_noDrop.outputCommands)')
1293                 self.AlCaPaths.append(shortName)
1294                 if 'DQM' in alcaList:
1295                     if not self._options.inlineEventContent and hasattr(self.process,name):
1296                         self.executeAndRemember('process.' + name + '.outputCommands.append("keep *_MEtoEDMConverter_*_*")')
1297                     else:
1298                         output.outputCommands.append("keep *_MEtoEDMConverter_*_*")
1299
1300                 #rename the HLT process name in the alca modules
1301                 if self._options.hltProcess or 'HLT' in self.stepMap:
1302                     if isinstance(alcastream.paths,tuple):
1303                         for path in alcastream.paths:
1304                             self.renameHLTprocessInSequence(path.label())
1305                     else:
1306                         self.renameHLTprocessInSequence(alcastream.paths.label())
1307
1308                 for i in range(alcaList.count(shortName)):
1309                     alcaList.remove(shortName)
1310
1311             # DQM needs a special handling
1312             elif name == 'pathALCARECODQM' and 'DQM' in alcaList:
1313                 path = getattr(alcaConfig,name)
1314                 self.schedule.append(path)
1315                 alcaList.remove('DQM')
1316
1317             if isinstance(alcastream,cms.Path):
1318                 #black list the alca path so that they do not appear in the cfg
1319                 self.blacklist_paths.append(alcastream)
1320
1321
1322         if len(alcaList) != 0:
1323             available=[]
1324             for name in alcaConfig.__dict__:
1325                 alcastream = getattr(alcaConfig,name)
1326                 if isinstance(alcastream,cms.FilteredStream):
1327                     available.append(name.replace('ALCARECOStream',''))
1328             print("The following alcas could not be found "+str(alcaList))
1329             print("available ",available)
1330             #print "verify your configuration, ignoring for now"
1331             raise Exception("The following alcas could not be found "+str(alcaList))
1332
1333     def prepare_LHE(self, sequence = None):
1334             #load the fragment
1335             ##make it loadable
1336         loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','').replace('/','.')
1337         print("Loading lhe fragment from",loadFragment)
1338         __import__(loadFragment)
1339         self.process.load(loadFragment)
1340         ##inline the modules
1341         self._options.inlineObjets+=','+sequence
1342
1343         getattr(self.process,sequence).nEvents = int(self._options.number)
1344
1345         #schedule it
1346         self.process.lhe_step = cms.Path( getattr( self.process,sequence)  )
1347         self.excludedPaths.append("lhe_step")
1348         self.schedule.append( self.process.lhe_step )
1349
1350     def prepare_GEN(self, sequence = None):
1351         """ load the fragment of generator configuration """
1352         loadFailure=False
1353         #remove trailing .py
1354         #support old style .cfi by changing into something.cfi into something_cfi
1355         #remove python/ from the name
1356         loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','')
1357         #standard location of fragments
1358         if not '/' in loadFragment:
1359             loadFragment='Configuration.Generator.'+loadFragment
1360         else:
1361             loadFragment=loadFragment.replace('/','.')
1362         try:
1363             print("Loading generator fragment from",loadFragment)
1364             __import__(loadFragment)
1365         except:
1366             loadFailure=True
1367             #if self.process.source and self.process.source.type_()=='EmptySource':
1368             if not (self._options.filein or self._options.dasquery):
1369                 raise Exception("Neither gen fragment of input files provided: this is an inconsistent GEN step configuration")
1370
1371         if not loadFailure:
1372             from Configuration.Generator.concurrentLumisDisable import noConcurrentLumiGenerators
1373
1374             generatorModule=sys.modules[loadFragment]
1375             genModules=generatorModule.__dict__
1376             #remove lhe producer module since this should have been
1377             #imported instead in the LHE step
1378             if self.LHEDefaultSeq in genModules:
1379                 del genModules[self.LHEDefaultSeq]
1380
1381             if self._options.hideGen:
1382                 self.loadAndRemember(loadFragment)
1383             else:
1384                 self.process.load(loadFragment)
1385                 # expose the objects from that fragment to the configuration
1386                 import FWCore.ParameterSet.Modules as cmstypes
1387                 for name in genModules:
1388                     theObject = getattr(generatorModule,name)
1389                     if isinstance(theObject, cmstypes._Module):
1390                         self._options.inlineObjets=name+','+self._options.inlineObjets
1391                         if theObject.type_() in noConcurrentLumiGenerators:
1392                             print("Setting numberOfConcurrentLuminosityBlocks=1 because of generator {}".format(theObject.type_()))
1393                             self._options.nConcurrentLumis = "1"
1394                             self._options.nConcurrentIOVs = "1"
1395                     elif isinstance(theObject, cms.Sequence) or isinstance(theObject, cmstypes.ESProducer):
1396                         self._options.inlineObjets+=','+name
1397
1398             if sequence == self.GENDefaultSeq or sequence == 'pgen_genonly':
1399                 if 'ProductionFilterSequence' in genModules and ('generator' in genModules):
1400                     self.productionFilterSequence = 'ProductionFilterSequence'
1401                 elif 'generator' in genModules:
1402                     self.productionFilterSequence = 'generator'
1403
1404         """ Enrich the schedule with the rest of the generation step """
1405         self.loadDefaultOrSpecifiedCFF(sequence,self.GENDefaultCFF)
1406         genSeqName=sequence.split('.')[-1]
1407
1408         if True:
1409             try:
1410                 from Configuration.StandardSequences.VtxSmeared import VtxSmeared
1411                 cffToBeLoaded=VtxSmeared[self._options.beamspot]
1412                 self.loadAndRemember(cffToBeLoaded)
1413             except ImportError:
1414                 raise Exception("VertexSmearing type or beamspot "+self._options.beamspot+" unknown.")
1415
1416             if self._options.scenario == 'HeavyIons':
1417                 if self._options.pileup=='HiMixGEN':
1418                     self.loadAndRemember("Configuration/StandardSequences/GeneratorMix_cff")
1419                 elif self._options.pileup=='HiMixEmbGEN':
1420                     self.loadAndRemember("Configuration/StandardSequences/GeneratorEmbMix_cff")
1421                 else:
1422                     self.loadAndRemember("Configuration/StandardSequences/GeneratorHI_cff")
1423
1424         self.process.generation_step = cms.Path( getattr(self.process,genSeqName) )
1425         self.schedule.append(self.process.generation_step)
1426
1427         #register to the genstepfilter the name of the path (static right now, but might evolve)
1428         self.executeAndRemember('process.genstepfilter.triggerConditions=cms.vstring("generation_step")')
1429
1430         if 'reGEN' in self.stepMap:
1431             #stop here
1432             return
1433
1434         """ Enrich the schedule with the summary of the filter step """
1435         #the gen filter in the endpath
1436         self.loadAndRemember("GeneratorInterface/Core/genFilterSummary_cff")
1437         self.scheduleSequenceAtEnd('genFilterSummary','genfiltersummary_step')
1438         return
1439
1440     def prepare_SIM(self, sequence = None):
1441         """ Enrich the schedule with the simulation step"""
1442         self.loadDefaultOrSpecifiedCFF(sequence,self.SIMDefaultCFF)
1443         if not self._options.fast:
1444             if self._options.gflash==True:
1445                 self.loadAndRemember("Configuration/StandardSequences/GFlashSIM_cff")
1446
1447             if self._options.magField=='0T':
1448                 self.executeAndRemember("process.g4SimHits.UseMagneticField = cms.bool(False)")
1449         else:
1450             if self._options.magField=='0T':
1451                 self.executeAndRemember("process.fastSimProducer.detectorDefinition.magneticFieldZ = cms.untracked.double(0.)")
1452
1453         self.scheduleSequence(sequence.split('.')[-1],'simulation_step')
1454         return
1455
1456     def prepare_DIGI(self, sequence = None):
1457         """ Enrich the schedule with the digitisation step"""
1458         self.loadDefaultOrSpecifiedCFF(sequence,self.DIGIDefaultCFF)
1459
1460         if self._options.gflash==True:
1461             self.loadAndRemember("Configuration/StandardSequences/GFlashDIGI_cff")
1462
1463         if sequence == 'pdigi_valid' or sequence == 'pdigi_hi':
1464             self.executeAndRemember("process.mix.digitizers = cms.PSet(process.theDigitizersValid)")
1465
1466         if sequence != 'pdigi_nogen' and sequence != 'pdigi_valid_nogen' and sequence != 'pdigi_hi_nogen' and not self.process.source.type_()=='EmptySource' and not self._options.filetype == "LHE":
1467             if self._options.inputEventContent=='':
1468                 self._options.inputEventContent='REGEN'
1469             else:
1470                 self._options.inputEventContent=self._options.inputEventContent+',REGEN'
1471
1472
1473         self.scheduleSequence(sequence.split('.')[-1],'digitisation_step')
1474         return
1475
1476     def prepare_CFWRITER(self, sequence = None):
1477         """ Enrich the schedule with the crossing frame writer step"""
1478         self.loadAndRemember(self.CFWRITERDefaultCFF)
1479         self.scheduleSequence('pcfw','cfwriter_step')
1480         return
1481
1482     def prepare_DATAMIX(self, sequence = None):
1483         """ Enrich the schedule with the digitisation step"""
1484         self.loadAndRemember(self.DATAMIXDefaultCFF)
1485         self.scheduleSequence('pdatamix','datamixing_step')
1486
1487         if self._options.pileup_input:
1488             theFiles=''
1489             if self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:'):
1490                 theFiles=filesFromDASQuery('file dataset = %s'%(self._options.pileup_input[4:],),self._options.pileup_dasoption)[0]
1491             elif self._options.pileup_input.startswith("filelist:"):
1492                 theFiles= (filesFromList(self._options.pileup_input[9:]))[0]
1493             else:
1494                 theFiles=self._options.pileup_input.split(',')
1495             #print theFiles
1496             self.executeAndRemember( "process.mixData.input.fileNames = cms.untracked.vstring(%s)"%(  theFiles ) )
1497
1498         return
1499
1500     def prepare_DIGI2RAW(self, sequence = None):
1501         self.loadDefaultOrSpecifiedCFF(sequence,self.DIGI2RAWDefaultCFF)
1502         self.scheduleSequence(sequence.split('.')[-1],'digi2raw_step')
1503         return
1504
1505     def prepare_REPACK(self, sequence = None):
1506         self.loadDefaultOrSpecifiedCFF(sequence,self.REPACKDefaultCFF)
1507         self.scheduleSequence(sequence.split('.')[-1],'digi2repack_step')
1508         return
1509
1510     def prepare_L1(self, sequence = None):
1511         """ Enrich the schedule with the L1 simulation step"""
1512         assert(sequence == None)
1513         self.loadAndRemember(self.L1EMDefaultCFF)
1514         self.scheduleSequence('SimL1Emulator','L1simulation_step')
1515         return
1516
1517     def prepare_L1REPACK(self, sequence = None):
1518         """ Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection"""
1519         supported = ['GT','GT1','GT2','GCTGT','Full','FullSimTP','FullMC','Full2015Data','uGT','CalouGT']
1520         if sequence in supported:
1521             self.loadAndRemember('Configuration/StandardSequences/SimL1EmulatorRepack_%s_cff'%sequence)
1522             if self._options.scenario == 'HeavyIons':
1523                 self.renameInputTagsInSequence("SimL1Emulator","rawDataCollector","rawDataRepacker")
1524             self.scheduleSequence('SimL1Emulator','L1RePack_step')
1525         else:
1526             print("L1REPACK with '",sequence,"' is not supported! Supported choices are: ",supported)
1527             raise Exception('unsupported feature')
1528
1529     def prepare_HLT(self, sequence = None):
1530         """ Enrich the schedule with the HLT simulation step"""
1531         if not sequence:
1532             print("no specification of the hlt menu has been given, should never happen")
1533             raise  Exception('no HLT sequence provided')
1534
1535         if '@' in sequence:
1536             # case where HLT:@something was provided
1537             from Configuration.HLT.autoHLT import autoHLT
1538             key = sequence[1:]
1539             if key in autoHLT:
1540                 sequence = autoHLT[key]
1541             else:
1542                 raise ValueError('no HLT mapping key "%s" found in autoHLT' % key)
1543
1544         if ',' in sequence:
1545             #case where HLT:something:something was provided
1546             self.executeAndRemember('import HLTrigger.Configuration.Utilities')
1547             optionsForHLT = {}
1548             if self._options.scenario == 'HeavyIons':
1549                 optionsForHLT['type'] = 'HIon'
1550             else:
1551                 optionsForHLT['type'] = 'GRun'
1552             optionsForHLTConfig = ', '.join('%s=%s' % (key, repr(val)) for (key, val) in optionsForHLT.items())
1553             if sequence == 'run,fromSource':
1554                 if hasattr(self.process.source,'firstRun'):
1555                     self.executeAndRemember('process.loadHltConfiguration("run:%%d"%%(process.source.firstRun.value()),%s)'%(optionsForHLTConfig))
1556                 elif hasattr(self.process.source,'setRunNumber'):
1557                     self.executeAndRemember('process.loadHltConfiguration("run:%%d"%%(process.source.setRunNumber.value()),%s)'%(optionsForHLTConfig))
1558                 else:
1559                     raise Exception('Cannot replace menu to load %s'%(sequence))
1560             else:
1561                 self.executeAndRemember('process.loadHltConfiguration("%s",%s)'%(sequence.replace(',',':'),optionsForHLTConfig))
1562         else:
1563             self.loadAndRemember('HLTrigger/Configuration/HLT_%s_cff' % sequence)
1564
1565         if self._options.isMC:
1566             self._options.customisation_file.append("HLTrigger/Configuration/customizeHLTforMC.customizeHLTforMC")
1567
1568         if self._options.name != 'HLT':
1569             self.additionalCommands.append('from HLTrigger.Configuration.CustomConfigs import ProcessName')
1570             self.additionalCommands.append('process = ProcessName(process)')
1571             self.additionalCommands.append('')
1572             from HLTrigger.Configuration.CustomConfigs import ProcessName
1573             self.process = ProcessName(self.process)
1574
1575         if self.process.schedule == None:
1576             raise Exception('the HLT step did not attach a valid schedule to the process')
1577
1578         self.scheduleIndexOfFirstHLTPath = len(self.schedule)
1579         [self.blacklist_paths.append(path) for path in self.process.schedule if isinstance(path,(cms.Path,cms.EndPath))]
1580
1581         # this is a fake, to be removed with fastim migration and HLT menu dump
1582         if self._options.fast:
1583             if not hasattr(self.process,'HLTEndSequence'):
1584                 self.executeAndRemember("process.HLTEndSequence = cms.Sequence( process.dummyModule )")
1585
1586
1587     def prepare_RAW2RECO(self, sequence = None):
1588         if ','in sequence:
1589             seqReco=sequence.split(',')[1]
1590             seqDigi=sequence.split(',')[0]
1591         else:
1592             print("RAW2RECO requires two specifications",sequence,"insufficient")
1593
1594         self.prepare_RAW2DIGI(seqDigi)
1595         self.prepare_RECO(seqReco)
1596         return
1597
1598     def prepare_RAW2DIGI(self, sequence = "RawToDigi"):
1599         self.loadDefaultOrSpecifiedCFF(sequence,self.RAW2DIGIDefaultCFF)
1600         self.scheduleSequence(sequence,'raw2digi_step')
1601          #          if self._options.isRepacked:
1602         #self.renameInputTagsInSequence(sequence)
1603         return
1604
1605     def prepare_PATFILTER(self, sequence=None):
1606         self.loadAndRemember("PhysicsTools/PatAlgos/slimming/metFilterPaths_cff")
1607         from PhysicsTools.PatAlgos.slimming.metFilterPaths_cff import allMetFilterPaths
1608         for filt in allMetFilterPaths:
1609             self.schedule.append(getattr(self.process,'Flag_'+filt))
1610
1611     def prepare_L1HwVal(self, sequence = 'L1HwVal'):
1612         ''' Enrich the schedule with L1 HW validation '''
1613         self.loadDefaultOrSpecifiedCFF(sequence,self.L1HwValDefaultCFF)
1614         #self.scheduleSequence(sequence.split('.')[-1],'l1hwval_step')
1615         print('\n\n\n DEPRECATED this has no action \n\n\n')
1616         return
1617
1618     def prepare_L1Reco(self, sequence = "L1Reco"):
1619         ''' Enrich the schedule with L1 reconstruction '''
1620         self.loadDefaultOrSpecifiedCFF(sequence,self.L1RecoDefaultCFF)
1621         self.scheduleSequence(sequence.split('.')[-1],'L1Reco_step')
1622         return
1623
1624     def prepare_L1TrackTrigger(self, sequence = "L1TrackTrigger"):
1625         ''' Enrich the schedule with L1 reconstruction '''
1626         self.loadDefaultOrSpecifiedCFF(sequence,self.L1TrackTriggerDefaultCFF)
1627         self.scheduleSequence(sequence.split('.')[-1],'L1TrackTrigger_step')
1628         return
1629
1630     def prepare_FILTER(self, sequence = None):
1631         ''' Enrich the schedule with a user defined filter sequence '''
1632         ## load the relevant part
1633         filterConfig=self.load(sequence.split('.')[0])
1634         filterSeq=sequence.split('.')[-1]
1635         ## print it in the configuration
1636         class PrintAllModules(object):
1637             def __init__(self):
1638                 self.inliner=''
1639                 pass
1640             def enter(self,visitee):
1641                 try:
1642                     label=visitee.label()
1643                     ##needs to be in reverse order
1644                     self.inliner=label+','+self.inliner
1645                 except:
1646                     pass
1647             def leave(self,v): pass
1648
1649         expander=PrintAllModules()
1650         getattr(self.process,filterSeq).visit( expander )
1651         self._options.inlineObjets+=','+expander.inliner
1652         self._options.inlineObjets+=','+filterSeq
1653
1654         ## put the filtering path in the schedule
1655         self.scheduleSequence(filterSeq,'filtering_step')
1656         self.nextScheduleIsConditional=True
1657         ## put it before all the other paths
1658         self.productionFilterSequence = filterSeq
1659
1660         return
1661
1662     def prepare_RECO(self, sequence = "reconstruction"):
1663         ''' Enrich the schedule with reconstruction '''
1664         self.loadDefaultOrSpecifiedCFF(sequence,self.RECODefaultCFF)
1665         self.scheduleSequence(sequence.split('.')[-1],'reconstruction_step')
1666         return
1667
1668     def prepare_RECOSIM(self, sequence = "recosim"):
1669         ''' Enrich the schedule with reconstruction '''
1670         self.loadDefaultOrSpecifiedCFF(sequence,self.RECOSIMDefaultCFF)
1671         self.scheduleSequence(sequence.split('.')[-1],'recosim_step')
1672         return
1673
1674     def prepare_RECOBEFMIX(self, sequence = "reconstruction"):
1675         ''' Enrich the schedule with the part of reconstruction that is done before mixing in FastSim'''
1676         if not self._options.fast:
1677             print("ERROR: this step is only implemented for FastSim")
1678             sys.exit()
1679         self.loadDefaultOrSpecifiedCFF(self.RECOBEFMIXDefaultSeq,self.RECOBEFMIXDefaultCFF)
1680         self.scheduleSequence(sequence.split('.')[-1],'reconstruction_befmix_step')
1681         return
1682
1683     def prepare_PAT(self, sequence = "miniAOD"):
1684         ''' Enrich the schedule with PAT '''
1685         self.prepare_PATFILTER(self)
1686         self.loadDefaultOrSpecifiedCFF(sequence,self.PATDefaultCFF)
1687         self.labelsToAssociate.append('patTask')
1688         if self._options.isData:
1689             self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllData")
1690         else:
1691             if self._options.fast:
1692                 self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllMCFastSim")
1693             else:
1694                 self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllMC")
1695
1696         if self._options.hltProcess:
1697             if len(self._options.customise_commands) > 1:
1698                 self._options.customise_commands = self._options.customise_commands + " \n"
1699             self._options.customise_commands = self._options.customise_commands + "process.patTrigger.processName = \""+self._options.hltProcess+"\"\n"
1700             self._options.customise_commands = self._options.customise_commands + "process.slimmedPatTrigger.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1701             self._options.customise_commands = self._options.customise_commands + "process.patMuons.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1702
1703 #            self.renameHLTprocessInSequence(sequence)
1704
1705         return
1706
1707     def prepare_PATGEN(self, sequence = "miniGEN"):
1708         ''' Enrich the schedule with PATGEN '''
1709         self.loadDefaultOrSpecifiedCFF(sequence,self.PATGENDefaultCFF) #this is unscheduled
1710         self.labelsToAssociate.append('patGENTask')
1711         if self._options.isData:
1712             raise Exception("PATGEN step can only run on MC")
1713         return
1714
1715     def prepare_NANO(self, sequence = "nanoAOD"):
1716         ''' Enrich the schedule with NANO '''
1717         self.loadDefaultOrSpecifiedCFF(sequence,self.NANODefaultCFF)
1718         self.scheduleSequence(sequence.split('.')[-1],'nanoAOD_step')
1719         custom = "nanoAOD_customizeData" if self._options.isData else "nanoAOD_customizeMC"
1720         self._options.customisation_file.insert(0,"PhysicsTools/NanoAOD/nano_cff."+custom)
1721         if self._options.hltProcess:
1722             if len(self._options.customise_commands) > 1:
1723                 self._options.customise_commands = self._options.customise_commands + " \n"
1724             self._options.customise_commands = self._options.customise_commands + "process.unpackedPatTrigger.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1725
1726     def prepare_NANOGEN(self, sequence = "nanoAOD"):
1727         ''' Enrich the schedule with NANOGEN '''
1728         # TODO: Need to modify this based on the input file type
1729         fromGen = any([x in self.stepMap for x in ['LHE', 'GEN', 'AOD']])
1730         self.loadDefaultOrSpecifiedCFF(sequence,self.NANOGENDefaultCFF)
1731         self.scheduleSequence(sequence.split('.')[-1],'nanoAOD_step')
1732         custom = "customizeNanoGEN" if fromGen else "customizeNanoGENFromMini"
1733         if self._options.runUnscheduled:
1734             self._options.customisation_file_unsch.insert(0, '.'.join([self.NANOGENDefaultCFF, custom]))
1735         else:
1736             self._options.customisation_file.insert(0, '.'.join([self.NANOGENDefaultCFF, custom]))
1737
1738     def prepare_SKIM(self, sequence = "all"):
1739         ''' Enrich the schedule with skimming fragments'''
1740         skimConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.SKIMDefaultCFF)
1741         sequence = sequence.split('.')[-1]
1742
1743         skimlist=sequence.split('+')
1744         ## support @Mu+DiJet+@Electron configuration via autoSkim.py
1745         from Configuration.Skimming.autoSkim import autoSkim
1746         self.expandMapping(skimlist,autoSkim)
1747
1748         #print "dictionnary for skims:",skimConfig.__dict__
1749         for skim in skimConfig.__dict__:
1750             skimstream = getattr(skimConfig,skim)
1751             if isinstance(skimstream,cms.Path):
1752             #black list the alca path so that they do not appear in the cfg
1753                 self.blacklist_paths.append(skimstream)
1754             if (not isinstance(skimstream,cms.FilteredStream)):
1755                 continue
1756             shortname = skim.replace('SKIMStream','')
1757             if (sequence=="all"):
1758                 self.addExtraStream(skim,skimstream)
1759             elif (shortname in skimlist):
1760                 self.addExtraStream(skim,skimstream)
1761                 #add a DQM eventcontent for this guy
1762                 if self._options.datatier=='DQM':
1763                     self.process.load(self.EVTCONTDefaultCFF)
1764                     skimstreamDQM = cms.FilteredStream(
1765                             responsible = skimstream.responsible,
1766                             name = skimstream.name+'DQM',
1767                             paths = skimstream.paths,
1768                             selectEvents = skimstream.selectEvents,
1769                             content = self._options.datatier+'EventContent',
1770                             dataTier = cms.untracked.string(self._options.datatier)
1771                             )
1772                     self.addExtraStream(skim+'DQM',skimstreamDQM)
1773                 for i in range(skimlist.count(shortname)):
1774                     skimlist.remove(shortname)
1775
1776
1777
1778         if (skimlist.__len__()!=0 and sequence!="all"):
1779             print('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
1780             raise Exception('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
1781
1782     def prepare_USER(self, sequence = None):
1783         ''' Enrich the schedule with a user defined sequence '''
1784         self.loadDefaultOrSpecifiedCFF(sequence,self.USERDefaultCFF)
1785         self.scheduleSequence(sequence.split('.')[-1],'user_step')
1786         return
1787
1788     def prepare_POSTRECO(self, sequence = None):
1789         """ Enrich the schedule with the postreco step """
1790         self.loadAndRemember(self.POSTRECODefaultCFF)
1791         self.scheduleSequence('postreco_generator','postreco_step')
1792         return
1793
1794
1795     def prepare_VALIDATION(self, sequence = 'validation'):
1796         print(sequence,"in preparing validation")
1797         self.loadDefaultOrSpecifiedCFF(sequence,self.VALIDATIONDefaultCFF)
1798         from Validation.Configuration.autoValidation import autoValidation
1799         #in case VALIDATION:something:somethingelse -> something,somethingelse
1800         sequence=sequence.split('.')[-1]
1801         if sequence.find(',')!=-1:
1802             prevalSeqName=sequence.split(',')[0].split('+')
1803             valSeqName=sequence.split(',')[1].split('+')
1804             self.expandMapping(prevalSeqName,autoValidation,index=0)
1805             self.expandMapping(valSeqName,autoValidation,index=1)
1806         else:
1807             if '@' in sequence:
1808                 prevalSeqName=sequence.split('+')
1809                 valSeqName=sequence.split('+')
1810                 self.expandMapping(prevalSeqName,autoValidation,index=0)
1811                 self.expandMapping(valSeqName,autoValidation,index=1)
1812             else:
1813                 postfix=''
1814                 if sequence:
1815                     postfix='_'+sequence
1816                 prevalSeqName=['prevalidation'+postfix]
1817                 valSeqName=['validation'+postfix]
1818                 if not hasattr(self.process,valSeqName[0]):
1819                     prevalSeqName=['']
1820                     valSeqName=[sequence]
1821
1822         def NFI(index):
1823             ##name from index, required to keep backward compatibility
1824             if index==0:
1825                 return ''
1826             else:
1827                 return '%s'%index
1828
1829
1830         #rename the HLT process in validation steps
1831         if ('HLT' in self.stepMap and not self._options.fast) or self._options.hltProcess:
1832             for s in valSeqName+prevalSeqName:
1833                 if s:
1834                     self.renameHLTprocessInSequence(s)
1835         for (i,s) in enumerate(prevalSeqName):
1836             if s:
1837                 setattr(self.process,'prevalidation_step%s'%NFI(i),  cms.Path( getattr(self.process, s)) )
1838                 self.schedule.append(getattr(self.process,'prevalidation_step%s'%NFI(i)))
1839
1840         for (i,s) in enumerate(valSeqName):
1841             setattr(self.process,'validation_step%s'%NFI(i), cms.EndPath( getattr(self.process, s)))
1842             self.schedule.append(getattr(self.process,'validation_step%s'%NFI(i)))
1843
1844         #needed in case the miniAODValidation sequence is run starting from AODSIM
1845         if 'PAT' in self.stepMap and not 'RECO' in self.stepMap:
1846             return
1847
1848         if not 'DIGI' in self.stepMap and not self._options.fast and not any(map( lambda s : s.startswith('genvalid'), valSeqName)):
1849             if self._options.restoreRNDSeeds==False and not self._options.restoreRNDSeeds==True:
1850                 self._options.restoreRNDSeeds=True
1851
1852         if not 'DIGI' in self.stepMap and not self._options.isData and not self._options.fast:
1853             self.executeAndRemember("process.mix.playback = True")
1854             self.executeAndRemember("process.mix.digitizers = cms.PSet()")
1855             self.executeAndRemember("for a in process.aliases: delattr(process, a)")
1856             self._options.customisation_file.append("SimGeneral/MixingModule/fullMixCustomize_cff.setCrossingFrameOn")
1857
1858         if hasattr(self.process,"genstepfilter") and len(self.process.genstepfilter.triggerConditions):
1859             #will get in the schedule, smoothly
1860             for (i,s) in enumerate(valSeqName):
1861                 getattr(self.process,'validation_step%s'%NFI(i)).insert(0, self.process.genstepfilter)
1862
1863         return
1864
1865
1866     class MassSearchReplaceProcessNameVisitor(object):
1867         """Visitor that travels within a cms.Sequence, looks for a parameter and replace its value
1868         It will climb down within PSets, VPSets and VInputTags to find its target"""
1869         def __init__(self, paramSearch, paramReplace, verbose=False, whitelist=()):
1870             self._paramReplace = paramReplace
1871             self._paramSearch = paramSearch
1872             self._verbose = verbose
1873             self._whitelist = whitelist
1874
1875         def doIt(self,pset,base):
1876             if isinstance(pset, cms._Parameterizable):
1877                 for name in pset.parameters_().keys():
1878                     # skip whitelisted parameters
1879                     if name in self._whitelist:
1880                         continue
1881                     # if I use pset.parameters_().items() I get copies of the parameter values
1882                     # so I can't modify the nested pset
1883                     value = getattr(pset,name)
1884                     type = value.pythonTypeName()
1885                     if type in ('cms.PSet', 'cms.untracked.PSet'):
1886                         self.doIt(value,base+"."+name)
1887                     elif type in ('cms.VPSet', 'cms.untracked.VPSet'):
1888                         for (i,ps) in enumerate(value): self.doIt(ps, "%s.%s[%d]"%(base,name,i) )
1889                     elif type in ('cms.string', 'cms.untracked.string'):
1890                         if value.value() == self._paramSearch:
1891                             if self._verbose: print("set string process name %s.%s %s ==> %s"% (base, name, value, self._paramReplace))
1892                             setattr(pset, name,self._paramReplace)
1893                     elif type in ('cms.VInputTag', 'cms.untracked.VInputTag'):
1894                         for (i,n) in enumerate(value):
1895                             if not isinstance(n, cms.InputTag):
1896                                 n=cms.InputTag(n)
1897                             if n.processName == self._paramSearch:
1898                                 # VInputTag can be declared as a list of strings, so ensure that n is formatted correctly
1899                                 if self._verbose:print("set process name %s.%s[%d] %s ==> %s " % (base, name, i, n, self._paramReplace))
1900                                 setattr(n,"processName",self._paramReplace)
1901                                 value[i]=n
1902                     elif type in ('cms.vstring', 'cms.untracked.vstring'):
1903                         for (i,n) in enumerate(value):
1904                             if n==self._paramSearch:
1905                                 getattr(pset,name)[i]=self._paramReplace
1906                     elif type in ('cms.InputTag', 'cms.untracked.InputTag'):
1907                         if value.processName == self._paramSearch:
1908                             if self._verbose: print("set process name %s.%s %s ==> %s " % (base, name, value, self._paramReplace))
1909                             setattr(getattr(pset, name),"processName",self._paramReplace)
1910
1911         def enter(self,visitee):
1912             label = ''
1913             try:
1914                 label = visitee.label()
1915             except AttributeError:
1916                 label = '<Module not in a Process>'
1917             except:
1918                 label = 'other execption'
1919             self.doIt(visitee, label)
1920
1921         def leave(self,visitee):
1922             pass
1923
1924     #visit a sequence to repalce all input tags
1925     def renameInputTagsInSequence(self,sequence,oldT="rawDataCollector",newT="rawDataRepacker"):
1926         print("Replacing all InputTag %s => %s"%(oldT,newT))
1927         from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag
1928         massSearchReplaceAnyInputTag(getattr(self.process,sequence),oldT,newT)
1929         loadMe='from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag'
1930         if not loadMe in self.additionalCommands:
1931             self.additionalCommands.append(loadMe)
1932         self.additionalCommands.append('massSearchReplaceAnyInputTag(process.%s,"%s","%s",False,True)'%(sequence,oldT,newT))
1933
1934     #change the process name used to address HLT results in any sequence
1935     def renameHLTprocessInSequence(self,sequence,proc=None,HLTprocess='HLT'):
1936         if self._options.hltProcess:
1937             proc=self._options.hltProcess
1938         else:
1939             proc=self.process.name_()
1940         if proc==HLTprocess:    return
1941         # look up all module in dqm sequence
1942         print("replacing %s process name - sequence %s will use '%s'" % (HLTprocess,sequence, proc))
1943         getattr(self.process,sequence).visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor(HLTprocess,proc,whitelist = ("subSystemFolder",)))
1944         if 'from Configuration.Applications.ConfigBuilder import ConfigBuilder' not in self.additionalCommands:
1945             self.additionalCommands.append('from Configuration.Applications.ConfigBuilder import ConfigBuilder')
1946         self.additionalCommands.append('process.%s.visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor("%s", "%s", whitelist = ("subSystemFolder",)))'% (sequence,HLTprocess, proc))
1947
1948
1949     def expandMapping(self,seqList,mapping,index=None):
1950         maxLevel=30
1951         level=0
1952         while '@' in repr(seqList) and level<maxLevel:
1953             level+=1
1954             for specifiedCommand in seqList:
1955                 if specifiedCommand.startswith('@'):
1956                     location=specifiedCommand[1:]
1957                     if not location in mapping:
1958                         raise Exception("Impossible to map "+location+" from "+repr(mapping))
1959                     mappedTo=mapping[location]
1960                     if index!=None:
1961                         mappedTo=mappedTo[index]
1962                     seqList.remove(specifiedCommand)
1963                     seqList.extend(mappedTo.split('+'))
1964                     break;
1965         if level==maxLevel:
1966             raise Exception("Could not fully expand "+repr(seqList)+" from "+repr(mapping))
1967
1968     def prepare_DQM(self, sequence = 'DQMOffline'):
1969         # this one needs replacement
1970
1971         # any 'DQM' job should use DQMStore in non-legacy mode (but not HARVESTING)
1972         self.loadAndRemember("DQMServices/Core/DQMStoreNonLegacy_cff")
1973         self.loadDefaultOrSpecifiedCFF(sequence,self.DQMOFFLINEDefaultCFF)
1974         sequenceList=sequence.split('.')[-1].split('+')
1975         postSequenceList=sequence.split('.')[-1].split('+')
1976         from DQMOffline.Configuration.autoDQM import autoDQM
1977         self.expandMapping(sequenceList,autoDQM,index=0)
1978         self.expandMapping(postSequenceList,autoDQM,index=1)
1979
1980         if len(set(sequenceList))!=len(sequenceList):
1981             sequenceList=list(set(sequenceList))
1982             print("Duplicate entries for DQM:, using",sequenceList)
1983
1984         pathName='dqmoffline_step'
1985         for (i,sequence) in enumerate(sequenceList):
1986             if (i!=0):
1987                 pathName='dqmoffline_%d_step'%(i)
1988
1989             if 'HLT' in self.stepMap.keys() or self._options.hltProcess:
1990                 self.renameHLTprocessInSequence(sequence)
1991
1992             setattr(self.process,pathName, cms.EndPath( getattr(self.process,sequence ) ) )
1993             self.schedule.append(getattr(self.process,pathName))
1994
1995             if hasattr(self.process,"genstepfilter") and len(self.process.genstepfilter.triggerConditions):
1996                 #will get in the schedule, smoothly
1997                 getattr(self.process,pathName).insert(0,self.process.genstepfilter)
1998
1999
2000         pathName='dqmofflineOnPAT_step'
2001         for (i,sequence) in enumerate(postSequenceList):
2002         #Fix needed to avoid duplication of sequences not defined in autoDQM or without a PostDQM
2003             if (sequenceList[i]==postSequenceList[i]):
2004                       continue
2005             if (i!=0):
2006                 pathName='dqmofflineOnPAT_%d_step'%(i)
2007
2008             setattr(self.process,pathName, cms.EndPath( getattr(self.process, sequence ) ) )
2009             self.schedule.append(getattr(self.process,pathName))
2010
2011     def prepare_HARVESTING(self, sequence = None):
2012         """ Enrich the process with harvesting step """
2013         self.DQMSaverCFF='Configuration/StandardSequences/DQMSaver'+self._options.harvesting+'_cff'
2014         self.loadAndRemember(self.DQMSaverCFF)
2015
2016         harvestingConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.HARVESTINGDefaultCFF)
2017         sequence = sequence.split('.')[-1]
2018
2019         # decide which HARVESTING paths to use
2020         harvestingList = sequence.split("+")
2021         from DQMOffline.Configuration.autoDQM import autoDQM
2022         from Validation.Configuration.autoValidation import autoValidation
2023         import copy
2024         combined_mapping = copy.deepcopy( autoDQM )
2025         combined_mapping.update( autoValidation )
2026         self.expandMapping(harvestingList,combined_mapping,index=-1)
2027
2028         if len(set(harvestingList))!=len(harvestingList):
2029             harvestingList=list(set(harvestingList))
2030             print("Duplicate entries for HARVESTING, using",harvestingList)
2031
2032         for name in harvestingList:
2033             if not name in harvestingConfig.__dict__:
2034                 print(name,"is not a possible harvesting type. Available are",harvestingConfig.__dict__.keys())
2035                 # trigger hard error, like for other sequence types
2036                 getattr(self.process, name)
2037                 continue
2038             harvestingstream = getattr(harvestingConfig,name)
2039             if isinstance(harvestingstream,cms.Path):
2040                 self.schedule.append(harvestingstream)
2041                 self.blacklist_paths.append(harvestingstream)
2042             if isinstance(harvestingstream,cms.Sequence):
2043                 setattr(self.process,name+"_step",cms.Path(harvestingstream))
2044                 self.schedule.append(getattr(self.process,name+"_step"))
2045
2046         self.scheduleSequence('DQMSaver','dqmsave_step')
2047         return
2048
2049     def prepare_ALCAHARVEST(self, sequence = None):
2050         """ Enrich the process with AlCaHarvesting step """
2051         harvestingConfig = self.loadAndRemember(self.ALCAHARVESTDefaultCFF)
2052         sequence=sequence.split(".")[-1]
2053
2054         # decide which AlcaHARVESTING paths to use
2055         harvestingList = sequence.split("+")
2056
2057
2058
2059         from Configuration.AlCa.autoPCL import autoPCL
2060         self.expandMapping(harvestingList,autoPCL)
2061
2062         for name in harvestingConfig.__dict__:
2063             harvestingstream = getattr(harvestingConfig,name)
2064             if name in harvestingList and isinstance(harvestingstream,cms.Path):
2065                 self.schedule.append(harvestingstream)
2066                 if isinstance(getattr(harvestingConfig,"ALCAHARVEST" + name + "_dbOutput"), cms.VPSet) and \
2067                    isinstance(getattr(harvestingConfig,"ALCAHARVEST" + name + "_metadata"), cms.VPSet):
2068                     self.executeAndRemember("process.PoolDBOutputService.toPut.extend(process.ALCAHARVEST" + name + "_dbOutput)")
2069                     self.executeAndRemember("process.pclMetadataWriter.recordsToMap.extend(process.ALCAHARVEST" + name + "_metadata)")
2070                 else:
2071                     self.executeAndRemember("process.PoolDBOutputService.toPut.append(process.ALCAHARVEST" + name + "_dbOutput)")
2072                     self.executeAndRemember("process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVEST" + name + "_metadata)")
2073                 harvestingList.remove(name)
2074         # append the common part at the end of the sequence
2075         lastStep = getattr(harvestingConfig,"ALCAHARVESTDQMSaveAndMetadataWriter")
2076         self.schedule.append(lastStep)
2077
2078         if len(harvestingList) != 0 and 'dummyHarvesting' not in harvestingList :
2079             print("The following harvesting could not be found : ", harvestingList)
2080             raise Exception("The following harvesting could not be found : "+str(harvestingList))
2081
2082
2083
2084     def prepare_ENDJOB(self, sequence = 'endOfProcess'):
2085         self.loadDefaultOrSpecifiedCFF(sequence,self.ENDJOBDefaultCFF)
2086         self.scheduleSequenceAtEnd(sequence.split('.')[-1],'endjob_step')
2087         return
2088
2089     def finalizeFastSimHLT(self):
2090         self.process.reconstruction = cms.Path(self.process.reconstructionWithFamos)
2091         self.schedule.append(self.process.reconstruction)
2092
2093
2094     def build_production_info(self, evt_type, evtnumber):
2095         """ Add useful info for the production. """
2096         self.process.configurationMetadata=cms.untracked.PSet\
2097                                             (version=cms.untracked.string("$Revision: 1.19 $"),
2098                                              name=cms.untracked.string("Applications"),
2099                                              annotation=cms.untracked.string(evt_type+ " nevts:"+str(evtnumber))
2100                                              )
2101
2102         self.addedObjects.append(("Production Info","configurationMetadata"))
2103
2104
2105     def create_process(self):
2106         self.pythonCfgCode =  "# Auto generated configuration file\n"
2107         self.pythonCfgCode += "# using: \n# "+__version__[1:-1]+"\n# "+__source__[1:-1]+'\n'
2108         self.pythonCfgCode += "# with command line options: "+self._options.arguments+'\n'
2109         self.pythonCfgCode += "import FWCore.ParameterSet.Config as cms\n\n"
2110
2111         # now set up the modifies
2112         modifiers=[]
2113         modifierStrings=[]
2114         modifierImports=[]
2115
2116         if hasattr(self._options,"era") and self._options.era :
2117         # Multiple eras can be specified in a comma seperated list
2118             from Configuration.StandardSequences.Eras import eras
2119             for requestedEra in self._options.era.split(",") :
2120                 modifierStrings.append(requestedEra)
2121                 modifierImports.append(eras.pythonCfgLines[requestedEra])
2122                 modifiers.append(getattr(eras,requestedEra))
2123
2124
2125         if hasattr(self._options,"procModifiers") and self._options.procModifiers:
2126             import importlib
2127             thingsImported=[]
2128             for c in self._options.procModifiers:
2129                 thingsImported.extend(c.split(","))
2130             for pm in thingsImported:
2131                 modifierStrings.append(pm)
2132                 modifierImports.append('from Configuration.ProcessModifiers.'+pm+'_cff import '+pm)
2133                 modifiers.append(getattr(importlib.import_module('Configuration.ProcessModifiers.'+pm+'_cff'),pm))
2134
2135         self.pythonCfgCode += '\n'.join(modifierImports)+'\n\n'
2136         self.pythonCfgCode += "process = cms.Process('"+self._options.name+"'" # Start of the line, finished after the loop
2137
2138
2139         if len(modifierStrings)>0:
2140             self.pythonCfgCode+= ','+','.join(modifierStrings)
2141         self.pythonCfgCode+=')\n\n'
2142
2143         #yes, the cfg code gets out of sync here if a process is passed in. That could be fixed in the future
2144         #assuming there is some way for the fwk to get the list of modifiers (and their stringified name)
2145         if self.process == None:
2146             if len(modifiers)>0:
2147                 self.process = cms.Process(self._options.name,*modifiers)
2148             else:
2149                 self.process = cms.Process(self._options.name)
2150
2151
2152
2153
2154     def prepare(self, doChecking = False):
2155         """ Prepare the configuration string and add missing pieces."""
2156
2157         self.loadAndRemember(self.EVTCONTDefaultCFF)  #load the event contents regardless
2158         self.addMaxEvents()
2159         if self.with_input:
2160             self.addSource()
2161         self.addStandardSequences()
2162         ##adding standard sequences might change the inputEventContent option and therefore needs to be finalized after
2163         self.completeInputCommand()
2164         self.addConditions()
2165
2166
2167         outputModuleCfgCode=""
2168         if not 'HARVESTING' in self.stepMap.keys() and not 'ALCAHARVEST' in self.stepMap.keys() and not 'ALCAOUTPUT' in self.stepMap.keys() and self.with_output:
2169             outputModuleCfgCode=self.addOutput()
2170
2171         self.addCommon()
2172
2173         self.pythonCfgCode += "# import of standard configurations\n"
2174         for module in self.imports:
2175             self.pythonCfgCode += ("process.load('"+module+"')\n")
2176
2177         # production info
2178         if not hasattr(self.process,"configurationMetadata"):
2179             self.build_production_info(self._options.evt_type, self._options.number)
2180         else:
2181             #the PSet was added via a load
2182             self.addedObjects.append(("Production Info","configurationMetadata"))
2183
2184         self.pythonCfgCode +="\n"
2185         for comment,object in self.addedObjects:
2186             if comment!="":
2187                 self.pythonCfgCode += "\n# "+comment+"\n"
2188             self.pythonCfgCode += dumpPython(self.process,object)
2189
2190         # dump the output definition
2191         self.pythonCfgCode += "\n# Output definition\n"
2192         self.pythonCfgCode += outputModuleCfgCode
2193
2194         # dump all additional outputs (e.g. alca or skim streams)
2195         self.pythonCfgCode += "\n# Additional output definition\n"
2196         #I do not understand why the keys are not normally ordered.
2197         nl=sorted(self.additionalOutputs.keys())
2198         for name in nl:
2199             output = self.additionalOutputs[name]
2200             self.pythonCfgCode += "process.%s = %s" %(name, output.dumpPython())
2201             tmpOut = cms.EndPath(output)
2202             setattr(self.process,name+'OutPath',tmpOut)
2203             self.schedule.append(tmpOut)
2204
2205         # dump all additional commands
2206         self.pythonCfgCode += "\n# Other statements\n"
2207         for command in self.additionalCommands:
2208             self.pythonCfgCode += command + "\n"
2209
2210         #comma separated list of objects that deserve to be inlined in the configuration (typically from a modified config deep down)
2211         for object in self._options.inlineObjets.split(','):
2212             if not object:
2213                 continue
2214             if not hasattr(self.process,object):
2215                 print('cannot inline -'+object+'- : not known')
2216             else:
2217                 self.pythonCfgCode +='\n'
2218                 self.pythonCfgCode +=dumpPython(self.process,object)
2219
2220         if self._options.pileup=='HiMixEmbGEN':
2221             self.pythonCfgCode += "\nprocess.generator.embeddingMode=cms.bool(True)\n"
2222
2223         # dump all paths
2224         self.pythonCfgCode += "\n# Path and EndPath definitions\n"
2225         for path in self.process.paths:
2226             if getattr(self.process,path) not in self.blacklist_paths:
2227                 self.pythonCfgCode += dumpPython(self.process,path)
2228
2229         for endpath in self.process.endpaths:
2230             if getattr(self.process,endpath) not in self.blacklist_paths:
2231                 self.pythonCfgCode += dumpPython(self.process,endpath)
2232
2233         # dump the schedule
2234         self.pythonCfgCode += "\n# Schedule definition\n"
2235
2236         # handling of the schedule
2237         pathNames = ['process.'+p.label_() for p in self.schedule]
2238         if self.process.schedule == None:
2239             self.process.schedule = cms.Schedule()
2240             for item in self.schedule:
2241                 self.process.schedule.append(item)
2242             result = 'process.schedule = cms.Schedule('+','.join(pathNames)+')\n'
2243         else:
2244             if not isinstance(self.scheduleIndexOfFirstHLTPath, int):
2245                 raise Exception('the schedule was imported from a cff in HLTrigger.Configuration, but the final index of the first HLT path is undefined')
2246
2247             for index, item in enumerate(self.schedule):
2248                 if index < self.scheduleIndexOfFirstHLTPath:
2249                     self.process.schedule.insert(index, item)
2250                 else:
2251                     self.process.schedule.append(item)
2252
2253             result = "# process.schedule imported from cff in HLTrigger.Configuration\n"
2254             for index, item in enumerate(pathNames[:self.scheduleIndexOfFirstHLTPath]):
2255                 result += 'process.schedule.insert('+str(index)+', '+item+')\n'
2256             if self.scheduleIndexOfFirstHLTPath < len(pathNames):
2257                 result += 'process.schedule.extend(['+','.join(pathNames[self.scheduleIndexOfFirstHLTPath:])+'])\n'
2258
2259         self.pythonCfgCode += result
2260
2261         for labelToAssociate in self.labelsToAssociate:
2262             self.process.schedule.associate(getattr(self.process, labelToAssociate))
2263             self.pythonCfgCode += 'process.schedule.associate(process.' + labelToAssociate + ')\n'
2264
2265         from PhysicsTools.PatAlgos.tools.helpers import associatePatAlgosToolsTask
2266         associatePatAlgosToolsTask(self.process)
2267         self.pythonCfgCode+="from PhysicsTools.PatAlgos.tools.helpers import associatePatAlgosToolsTask\n"
2268         self.pythonCfgCode+="associatePatAlgosToolsTask(process)\n"
2269
2270         if self._options.nThreads != "1":
2271             self.pythonCfgCode +="\n"
2272             self.pythonCfgCode +="#Setup FWK for multithreaded\n"
2273             self.pythonCfgCode +="process.options.numberOfThreads = "+self._options.nThreads+"\n"
2274             self.pythonCfgCode +="process.options.numberOfStreams = "+self._options.nStreams+"\n"
2275             self.pythonCfgCode +="process.options.numberOfConcurrentLuminosityBlocks = "+self._options.nConcurrentLumis+"\n"
2276             self.pythonCfgCode +="process.options.eventSetup.numberOfConcurrentIOVs = "+self._options.nConcurrentIOVs+"\n"
2277             if int(self._options.nConcurrentLumis) > 1:
2278               self.pythonCfgCode +="if hasattr(process, 'DQMStore'): process.DQMStore.assertLegacySafe=cms.untracked.bool(False)\n"
2279             self.process.options.numberOfThreads = int(self._options.nThreads)
2280             self.process.options.numberOfStreams = int(self._options.nStreams)
2281             self.process.options.numberOfConcurrentLuminosityBlocks = int(self._options.nConcurrentLumis)
2282             self.process.options.eventSetup.numberOfConcurrentIOVs = int(self._options.nConcurrentIOVs)
2283         #repacked version
2284         if self._options.isRepacked:
2285             self.pythonCfgCode +="\n"
2286             self.pythonCfgCode +="from Configuration.Applications.ConfigBuilder import MassReplaceInputTag\n"
2287             self.pythonCfgCode +="MassReplaceInputTag(process, new=\"rawDataMapperByLabel\", old=\"rawDataCollector\")\n"
2288             MassReplaceInputTag(self.process, new="rawDataMapperByLabel", old="rawDataCollector")
2289
2290         # special treatment in case of production filter sequence 2/2
2291         if self.productionFilterSequence and not (self._options.pileup=='HiMixEmbGEN'):
2292             self.pythonCfgCode +='# filter all path with the production filter sequence\n'
2293             self.pythonCfgCode +='for path in process.paths:\n'
2294             if len(self.conditionalPaths):
2295                 self.pythonCfgCode +='\tif not path in %s: continue\n'%str(self.conditionalPaths)
2296             if len(self.excludedPaths):
2297                 self.pythonCfgCode +='\tif path in %s: continue\n'%str(self.excludedPaths)
2298             self.pythonCfgCode +='\tgetattr(process,path).insert(0, process.%s)\n'%(self.productionFilterSequence,)
2299             pfs = getattr(self.process,self.productionFilterSequence)
2300             for path in self.process.paths:
2301                 if not path in self.conditionalPaths: continue
2302                 if path in self.excludedPaths: continue
2303                 getattr(self.process,path).insert(0, pfs)
2304
2305
2306         # dump customise fragment
2307         self.pythonCfgCode += self.addCustomise()
2308
2309         if self._options.runUnscheduled:
2310             print("--runUnscheduled is deprecated and not necessary anymore, and will be removed soon. Please update your command line.")
2311         # Keep the "unscheduled customise functions" separate for now,
2312         # there are customize functions given by users (in our unit
2313         # tests) that need to be run before the "unscheduled customise
2314         # functions"
2315         self.pythonCfgCode += self.addCustomise(1)
2316
2317         self.pythonCfgCode += self.addCustomiseCmdLine()
2318
2319         if hasattr(self.process,"logErrorHarvester"):
2320             #configure logErrorHarvester to wait for same EDProducers to finish as the OutputModules
2321             self.pythonCfgCode +="\n#Have logErrorHarvester wait for the same EDProducers to finish as those providing data for the OutputModule\n"
2322             self.pythonCfgCode +="from FWCore.Modules.logErrorHarvester_cff import customiseLogErrorHarvesterUsingOutputCommands\n"
2323             self.pythonCfgCode +="process = customiseLogErrorHarvesterUsingOutputCommands(process)\n"
2324             from FWCore.Modules.logErrorHarvester_cff import customiseLogErrorHarvesterUsingOutputCommands
2325             self.process = customiseLogErrorHarvesterUsingOutputCommands(self.process)
2326
2327         # Temporary hack to put the early delete customization after
2328         # everything else
2329         #
2330         # FIXME: remove when no longer needed
2331         self.pythonCfgCode += "\n# Add early deletion of temporary data products to reduce peak memory need\n"
2332         self.pythonCfgCode += "from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete\n"
2333         self.pythonCfgCode += "process = customiseEarlyDelete(process)\n"
2334         self.pythonCfgCode += "# End adding early deletion\n"
2335         from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete
2336         self.process = customiseEarlyDelete(self.process)
2337
2338         imports = cms.specialImportRegistry.getSpecialImports()
2339         if len(imports) > 0:
2340             #need to inject this at the top
2341             index = self.pythonCfgCode.find("import FWCore.ParameterSet.Config")
2342             #now find the end of line
2343             index = self.pythonCfgCode.find("\n",index)
2344             self.pythonCfgCode = self.pythonCfgCode[:index]+ "\n" + "\n".join(imports)+"\n" +self.pythonCfgCode[index:]
2345
2346
2347         # make the .io file
2348
2349         if self._options.io:
2350             #io=open(self._options.python_filename.replace('.py','.io'),'w')
2351             if not self._options.io.endswith('.io'): self._option.io+='.io'
2352             io=open(self._options.io,'w')
2353             ioJson={}
2354             if hasattr(self.process.source,"fileNames"):
2355                 if len(self.process.source.fileNames.value()):
2356                     ioJson['primary']=self.process.source.fileNames.value()
2357             if hasattr(self.process.source,"secondaryFileNames"):
2358                 if len(self.process.source.secondaryFileNames.value()):
2359                     ioJson['secondary']=self.process.source.secondaryFileNames.value()
2360             if self._options.pileup_input and (self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:')):
2361                 ioJson['pileup']=self._options.pileup_input[4:]
2362             for (o,om) in self.process.outputModules_().items():
2363                 ioJson[o]=om.fileName.value()
2364             ioJson['GT']=self.process.GlobalTag.globaltag.value()
2365             if self.productionFilterSequence:
2366                 ioJson['filter']=self.productionFilterSequence
2367             import json
2368             io.write(json.dumps(ioJson))
2369         return
2370