Applications/python/ConfigBuilder.py

0001 #! /usr/bin/env python3
0002
0003 from __future__ import print_function
0004 __version__ = "$Revision: 1.19 $"
0005 __source__ = "$Source: /local/reps/CMSSW/CMSSW/Configuration/Applications/python/ConfigBuilder.py,v $"
0006
0007 import FWCore.ParameterSet.Config as cms
0008 from FWCore.ParameterSet.Modules import _Module
0009 # The following import is provided for backward compatibility reasons.
0010 # The function used to be defined in this file.
0011 from FWCore.ParameterSet.MassReplace import massReplaceInputTag as MassReplaceInputTag
0012
0013 import hashlib
0014 import sys
0015 import re
0016 import collections
0017 from subprocess import Popen,PIPE
0018 import FWCore.ParameterSet.DictTypes as DictTypes
0019 class Options:
0020     pass
0021
0022 # the canonical defaults
0023 defaultOptions = Options()
0024 defaultOptions.datamix = 'DataOnSim'
0025 defaultOptions.isMC=False
0026 defaultOptions.isData=True
0027 defaultOptions.step=''
0028 defaultOptions.pileup='NoPileUp'
0029 defaultOptions.pileup_input = None
0030 defaultOptions.pileup_dasoption = ''
0031 defaultOptions.geometry = 'SimDB'
0032 defaultOptions.geometryExtendedOptions = ['ExtendedGFlash','Extended','NoCastor']
0033 defaultOptions.magField = ''
0034 defaultOptions.conditions = None
0035 defaultOptions.scenarioOptions=['pp','cosmics','nocoll','HeavyIons']
0036 defaultOptions.harvesting= 'AtRunEnd'
0037 defaultOptions.gflash = False
0038 defaultOptions.number = -1
0039 defaultOptions.number_out = None
0040 defaultOptions.arguments = ""
0041 defaultOptions.name = "NO NAME GIVEN"
0042 defaultOptions.evt_type = ""
0043 defaultOptions.filein = ""
0044 defaultOptions.dasquery=""
0045 defaultOptions.dasoption=""
0046 defaultOptions.secondfilein = ""
0047 defaultOptions.customisation_file = []
0048 defaultOptions.customisation_file_unsch = []
0049 defaultOptions.customise_commands = ""
0050 defaultOptions.inline_custom=False
0051 defaultOptions.particleTable = 'pythiapdt'
0052 defaultOptions.particleTableList = ['pythiapdt','pdt']
0053 defaultOptions.dirin = ''
0054 defaultOptions.dirout = ''
0055 defaultOptions.filetype = 'EDM'
0056 defaultOptions.fileout = 'output.root'
0057 defaultOptions.filtername = ''
0058 defaultOptions.lazy_download = False
0059 defaultOptions.custom_conditions = ''
0060 defaultOptions.hltProcess = ''
0061 defaultOptions.eventcontent = None
0062 defaultOptions.datatier = None
0063 defaultOptions.inlineEventContent = True
0064 defaultOptions.inlineObjets =''
0065 defaultOptions.hideGen=False
0066 from Configuration.StandardSequences.VtxSmeared import VtxSmearedDefaultKey,VtxSmearedHIDefaultKey
0067 defaultOptions.beamspot=None
0068 defaultOptions.outputDefinition =''
0069 defaultOptions.inputCommands = None
0070 defaultOptions.outputCommands = None
0071 defaultOptions.inputEventContent = ''
0072 defaultOptions.dropDescendant = False
0073 defaultOptions.relval = None
0074 defaultOptions.profile = None
0075 defaultOptions.isRepacked = False
0076 defaultOptions.restoreRNDSeeds = False
0077 defaultOptions.donotDropOnInput = ''
0078 defaultOptions.python_filename =''
0079 defaultOptions.io=None
0080 defaultOptions.lumiToProcess=None
0081 defaultOptions.fast=False
0082 defaultOptions.runsAndWeightsForMC = None
0083 defaultOptions.runsScenarioForMC = None
0084 defaultOptions.runsAndWeightsForMCIntegerWeights = None
0085 defaultOptions.runsScenarioForMCIntegerWeights = None
0086 defaultOptions.runUnscheduled = False
0087 defaultOptions.timeoutOutput = False
0088 defaultOptions.nThreads = '1'
0089 defaultOptions.nStreams = '0'
0090 defaultOptions.nConcurrentLumis = '0'
0091 defaultOptions.nConcurrentIOVs = '1'
0092
0093 # some helper routines
0094 def dumpPython(process,name):
0095     theObject = getattr(process,name)
0096     if isinstance(theObject,cms.Path) or isinstance(theObject,cms.EndPath) or isinstance(theObject,cms.Sequence):
0097         return "process."+name+" = " + theObject.dumpPython()
0098     elif isinstance(theObject,_Module) or isinstance(theObject,cms.ESProducer):
0099         return "process."+name+" = " + theObject.dumpPython()+"\n"
0100     else:
0101         return "process."+name+" = " + theObject.dumpPython()+"\n"
0102 def filesFromList(fileName,s=None):
0103     import os
0104     import FWCore.ParameterSet.Config as cms
0105     prim=[]
0106     sec=[]
0107     for line in open(fileName,'r'):
0108         if line.count(".root")>=2:
0109             #two files solution...
0110             entries=line.replace("\n","").split()
0111             prim.append(entries[0])
0112             sec.append(entries[1])
0113         elif (line.find(".root")!=-1):
0114             entry=line.replace("\n","")
0115             prim.append(entry)
0116     # remove any duplicates
0117     prim = sorted(list(set(prim)))
0118     sec = sorted(list(set(sec)))
0119     if s:
0120         if not hasattr(s,"fileNames"):
0121             s.fileNames=cms.untracked.vstring(prim)
0122         else:
0123             s.fileNames.extend(prim)
0124         if len(sec)!=0:
0125             if not hasattr(s,"secondaryFileNames"):
0126                 s.secondaryFileNames=cms.untracked.vstring(sec)
0127             else:
0128                 s.secondaryFileNames.extend(sec)
0129     print("found files: ",prim)
0130     if len(prim)==0:
0131         raise Exception("There are not files in input from the file list")
0132     if len(sec)!=0:
0133         print("found parent files:",sec)
0134     return (prim,sec)
0135
0136 def filesFromDASQuery(query,option="",s=None):
0137     import os,time
0138     import FWCore.ParameterSet.Config as cms
0139     prim=[]
0140     sec=[]
0141     print("the query is",query)
0142     eC=5
0143     count=0
0144     while eC!=0 and count<3:
0145         if count!=0:
0146             print('Sleeping, then retrying DAS')
0147             time.sleep(100)
0148         p = Popen('dasgoclient %s --query "%s"'%(option,query), stdout=PIPE,shell=True, universal_newlines=True)
0149         pipe=p.stdout.read()
0150         tupleP = os.waitpid(p.pid, 0)
0151         eC=tupleP[1]
0152         count=count+1
0153     if eC==0:
0154         print("DAS succeeded after",count,"attempts",eC)
0155     else:
0156         print("DAS failed 3 times- I give up")
0157     for line in pipe.split('\n'):
0158         if line.count(".root")>=2:
0159             #two files solution...
0160             entries=line.replace("\n","").split()
0161             prim.append(entries[0])
0162             sec.append(entries[1])
0163         elif (line.find(".root")!=-1):
0164             entry=line.replace("\n","")
0165             prim.append(entry)
0166     # remove any duplicates
0167     prim = sorted(list(set(prim)))
0168     sec = sorted(list(set(sec)))
0169     if s:
0170         if not hasattr(s,"fileNames"):
0171             s.fileNames=cms.untracked.vstring(prim)
0172         else:
0173             s.fileNames.extend(prim)
0174         if len(sec)!=0:
0175             if not hasattr(s,"secondaryFileNames"):
0176                 s.secondaryFileNames=cms.untracked.vstring(sec)
0177             else:
0178                 s.secondaryFileNames.extend(sec)
0179     print("found files: ",prim)
0180     if len(sec)!=0:
0181         print("found parent files:",sec)
0182     return (prim,sec)
0183
0184 def anyOf(listOfKeys,dict,opt=None):
0185     for k in listOfKeys:
0186         if k in dict:
0187             toReturn=dict[k]
0188             dict.pop(k)
0189             return toReturn
0190     if opt!=None:
0191         return opt
0192     else:
0193         raise Exception("any of "+','.join(listOfKeys)+" are mandatory entries of --output options")
0194
0195 class ConfigBuilder(object):
0196     """The main building routines """
0197
0198     def __init__(self, options, process = None, with_output = False, with_input = False ):
0199         """options taken from old cmsDriver and optparse """
0200
0201         options.outfile_name = options.dirout+options.fileout
0202
0203         self._options = options
0204
0205         if self._options.isData and options.isMC:
0206             raise Exception("ERROR: You may specify only --data or --mc, not both")
0207         #if not self._options.conditions:
0208         #        raise Exception("ERROR: No conditions given!\nPlease specify conditions. E.g. via --conditions=IDEAL_30X::All")
0209
0210         # check that MEtoEDMConverter (running in ENDJOB) and DQMIO don't run in the same job
0211         if 'ENDJOB' in self._options.step:
0212             if  (hasattr(self._options,"outputDefinition") and \
0213                 self._options.outputDefinition != '' and \
0214                 any(anyOf(['t','tier','dataTier'],outdic) == 'DQMIO' for outdic in eval(self._options.outputDefinition))) or \
0215                 (hasattr(self._options,"datatier") and \
0216                 self._options.datatier and \
0217                 'DQMIO' in self._options.datatier):
0218                 print("removing ENDJOB from steps since not compatible with DQMIO dataTier")
0219                 self._options.step=self._options.step.replace(',ENDJOB','')
0220
0221
0222
0223         # what steps are provided by this class?
0224         stepList = [re.sub(r'^prepare_', '', methodName) for methodName in ConfigBuilder.__dict__ if methodName.startswith('prepare_')]
0225         self.stepMap={}
0226         self.stepKeys=[]
0227         for step in self._options.step.split(","):
0228             if step=='': continue
0229             stepParts = step.split(":")
0230             stepName = stepParts[0]
0231             if stepName not in stepList and not stepName.startswith('re'):
0232                 raise ValueError("Step "+stepName+" unknown")
0233             if len(stepParts)==1:
0234                 self.stepMap[stepName]=""
0235             elif len(stepParts)==2:
0236                 self.stepMap[stepName]=stepParts[1].split('+')
0237             elif len(stepParts)==3:
0238                 self.stepMap[stepName]=(stepParts[2].split('+'),stepParts[1])
0239             else:
0240                 raise ValueError("Step definition "+step+" invalid")
0241             self.stepKeys.append(stepName)
0242
0243         #print "map of steps is:",self.stepMap
0244
0245         self.with_output = with_output
0246         self.process=process
0247
0248         if hasattr(self._options,"no_output_flag") and self._options.no_output_flag:
0249             self.with_output = False
0250         self.with_input = with_input
0251         self.imports = []
0252         self.create_process()
0253         self.define_Configs()
0254         self.schedule = list()
0255
0256         # we are doing three things here:
0257         # creating a process to catch errors
0258         # building the code to re-create the process
0259
0260         self.additionalCommands = []
0261         # TODO: maybe a list of to be dumped objects would help as well
0262         self.blacklist_paths = []
0263         self.addedObjects = []
0264         self.additionalOutputs = {}
0265
0266         self.productionFilterSequence = None
0267         self.labelsToAssociate=[]
0268         self.nextScheduleIsConditional=False
0269         self.conditionalPaths=[]
0270         self.excludedPaths=[]
0271
0272     def profileOptions(self):
0273         """
0274         addIgProfService
0275         Function to add the igprof profile service so that you can dump in the middle
0276         of the run.
0277         """
0278         profileOpts = self._options.profile.split(':')
0279         profilerStart = 1
0280         profilerInterval = 100
0281         profilerFormat = None
0282         profilerJobFormat = None
0283
0284         if len(profileOpts):
0285             #type, given as first argument is unused here
0286             profileOpts.pop(0)
0287         if len(profileOpts):
0288             startEvent = profileOpts.pop(0)
0289             if not startEvent.isdigit():
0290                 raise Exception("%s is not a number" % startEvent)
0291             profilerStart = int(startEvent)
0292         if len(profileOpts):
0293             eventInterval = profileOpts.pop(0)
0294             if not eventInterval.isdigit():
0295                 raise Exception("%s is not a number" % eventInterval)
0296             profilerInterval = int(eventInterval)
0297         if len(profileOpts):
0298             profilerFormat = profileOpts.pop(0)
0299
0300
0301         if not profilerFormat:
0302             profilerFormat = "%s___%s___%%I.gz" % (
0303                 self._options.evt_type.replace("_cfi", ""),
0304                 hashlib.md5(
0305                     str(self._options.step) + str(self._options.pileup) + str(self._options.conditions) +
0306                     str(self._options.datatier) + str(self._options.profileTypeLabel)
0307                 ).hexdigest()
0308             )
0309         if not profilerJobFormat and profilerFormat.endswith(".gz"):
0310             profilerJobFormat = profilerFormat.replace(".gz", "_EndOfJob.gz")
0311         elif not profilerJobFormat:
0312             profilerJobFormat = profilerFormat + "_EndOfJob.gz"
0313
0314         return (profilerStart,profilerInterval,profilerFormat,profilerJobFormat)
0315
0316     def load(self,includeFile):
0317         includeFile = includeFile.replace('/','.')
0318         self.process.load(includeFile)
0319         return sys.modules[includeFile]
0320
0321     def loadAndRemember(self, includeFile):
0322         """helper routine to load am memorize imports"""
0323         # we could make the imports a on-the-fly data method of the process instance itself
0324         # not sure if the latter is a good idea
0325         includeFile = includeFile.replace('/','.')
0326         self.imports.append(includeFile)
0327         self.process.load(includeFile)
0328         return sys.modules[includeFile]
0329
0330     def executeAndRemember(self, command):
0331         """helper routine to remember replace statements"""
0332         self.additionalCommands.append(command)
0333         if not command.strip().startswith("#"):
0334         # substitute: process.foo = process.bar -> self.process.foo = self.process.bar
0335             import re
0336             exec(re.sub(r"([^a-zA-Z_0-9]|^)(process)([^a-zA-Z_0-9])",r"\1self.process\3",command))
0337             #exec(command.replace("process.","self.process."))
0338
0339     def addCommon(self):
0340         if 'HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys():
0341             self.process.options.Rethrow = ['ProductNotFound']
0342             self.process.options.fileMode = 'FULLMERGE'
0343
0344         self.addedObjects.append(("","options"))
0345
0346         if self._options.lazy_download:
0347             self.process.AdaptorConfig = cms.Service("AdaptorConfig",
0348                                                      stats = cms.untracked.bool(True),
0349                                                      enable = cms.untracked.bool(True),
0350                                                      cacheHint = cms.untracked.string("lazy-download"),
0351                                                      readHint = cms.untracked.string("read-ahead-buffered")
0352                                                      )
0353             self.addedObjects.append(("Setup lazy download","AdaptorConfig"))
0354
0355         #self.process.cmsDriverCommand = cms.untracked.PSet( command=cms.untracked.string('cmsDriver.py '+self._options.arguments) )
0356         #self.addedObjects.append(("what cmsDriver command was used","cmsDriverCommand"))
0357
0358         if self._options.profile:
0359             (start, interval, eventFormat, jobFormat)=self.profileOptions()
0360             self.process.IgProfService = cms.Service("IgProfService",
0361                                                      reportFirstEvent            = cms.untracked.int32(start),
0362                                                      reportEventInterval         = cms.untracked.int32(interval),
0363                                                      reportToFileAtPostEvent     = cms.untracked.string("| gzip -c > %s"%(eventFormat)),
0364                                                      reportToFileAtPostEndJob    = cms.untracked.string("| gzip -c > %s"%(jobFormat)))
0365             self.addedObjects.append(("Setup IGProf Service for profiling","IgProfService"))
0366
0367     def addMaxEvents(self):
0368         """Here we decide how many evts will be processed"""
0369         self.process.maxEvents.input = int(self._options.number)
0370         if self._options.number_out:
0371             self.process.maxEvents.output = int(self._options.number_out)
0372         self.addedObjects.append(("","maxEvents"))
0373
0374     def addSource(self):
0375         """Here the source is built. Priority: file, generator"""
0376         self.addedObjects.append(("Input source","source"))
0377
0378         def filesFromOption(self):
0379             for entry in self._options.filein.split(','):
0380                 print("entry",entry)
0381                 if entry.startswith("filelist:"):
0382                     filesFromList(entry[9:],self.process.source)
0383                 elif entry.startswith("dbs:") or entry.startswith("das:"):
0384                     filesFromDASQuery('file dataset = %s'%(entry[4:]),self._options.dasoption,self.process.source)
0385                 else:
0386                     self.process.source.fileNames.append(self._options.dirin+entry)
0387             if self._options.secondfilein:
0388                 if not hasattr(self.process.source,"secondaryFileNames"):
0389                     raise Exception("--secondfilein not compatible with "+self._options.filetype+"input type")
0390                 for entry in self._options.secondfilein.split(','):
0391                     print("entry",entry)
0392                     if entry.startswith("filelist:"):
0393                         self.process.source.secondaryFileNames.extend((filesFromList(entry[9:]))[0])
0394                     elif entry.startswith("dbs:") or entry.startswith("das:"):
0395                         self.process.source.secondaryFileNames.extend((filesFromDASQuery('file dataset = %s'%(entry[4:]),self._options.dasoption))[0])
0396                     else:
0397                         self.process.source.secondaryFileNames.append(self._options.dirin+entry)
0398
0399         if self._options.filein or self._options.dasquery:
0400             if self._options.filetype == "EDM":
0401                 self.process.source=cms.Source("PoolSource",
0402                                                fileNames = cms.untracked.vstring(),
0403                                                secondaryFileNames= cms.untracked.vstring())
0404                 filesFromOption(self)
0405             elif self._options.filetype == "DAT":
0406                 self.process.source=cms.Source("NewEventStreamFileReader",fileNames = cms.untracked.vstring())
0407                 filesFromOption(self)
0408             elif self._options.filetype == "LHE":
0409                 self.process.source=cms.Source("LHESource", fileNames = cms.untracked.vstring())
0410                 if self._options.filein.startswith("lhe:"):
0411                     #list the article directory automatically
0412                     args=self._options.filein.split(':')
0413                     article=args[1]
0414                     print('LHE input from article ',article)
0415                     location='/store/lhe/'
0416                     import os
0417                     textOfFiles=os.popen('cmsLHEtoEOSManager.py -l '+article)
0418                     for line in textOfFiles:
0419                         for fileName in [x for x in line.split() if '.lhe' in x]:
0420                             self.process.source.fileNames.append(location+article+'/'+fileName)
0421                     #check first if list of LHE files is loaded (not empty)
0422                     if len(line)<2:
0423                         print('Issue to load LHE files, please check and try again.')
0424                         sys.exit(-1)
0425                     #Additional check to protect empty fileNames in process.source
0426                     if len(self.process.source.fileNames)==0:
0427                         print('Issue with empty filename, but can pass line check')
0428                         sys.exit(-1)
0429                     if len(args)>2:
0430                         self.process.source.skipEvents = cms.untracked.uint32(int(args[2]))
0431                 else:
0432                     filesFromOption(self)
0433
0434             elif self._options.filetype == "DQM":
0435                 self.process.source=cms.Source("DQMRootSource",
0436                                                fileNames = cms.untracked.vstring())
0437                 filesFromOption(self)
0438
0439             elif self._options.filetype == "DQMDAQ":
0440                 # FIXME: how to configure it if there are no input files specified?
0441                 self.process.source=cms.Source("DQMStreamerReader")
0442
0443
0444             if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"):
0445                 self.process.source.processingMode = cms.untracked.string("RunsAndLumis")
0446
0447         if self._options.dasquery!='':
0448             self.process.source=cms.Source("PoolSource", fileNames = cms.untracked.vstring(),secondaryFileNames = cms.untracked.vstring())
0449             filesFromDASQuery(self._options.dasquery,self._options.dasoption,self.process.source)
0450
0451             if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"):
0452                 self.process.source.processingMode = cms.untracked.string("RunsAndLumis")
0453
0454         ##drop LHEXMLStringProduct on input to save memory if appropriate
0455         if 'GEN' in self.stepMap.keys() and not self._options.filetype == "LHE":
0456             if self._options.inputCommands:
0457                 self._options.inputCommands+=',drop LHEXMLStringProduct_*_*_*,'
0458             else:
0459                 self._options.inputCommands='keep *, drop LHEXMLStringProduct_*_*_*,'
0460
0461         if self.process.source and self._options.inputCommands and not self._options.filetype == "LHE":
0462             if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring()
0463             for command in self._options.inputCommands.split(','):
0464                 # remove whitespace around the keep/drop statements
0465                 command = command.strip()
0466                 if command=='': continue
0467                 self.process.source.inputCommands.append(command)
0468             if not self._options.dropDescendant:
0469                 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False)
0470
0471         if self._options.lumiToProcess:
0472             import FWCore.PythonUtilities.LumiList as LumiList
0473             self.process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange( LumiList.LumiList(self._options.lumiToProcess).getCMSSWString().split(',') )
0474
0475         if 'GEN' in self.stepMap.keys() or 'LHE' in self.stepMap or (not self._options.filein and hasattr(self._options, "evt_type")):
0476             if self.process.source is None:
0477                 self.process.source=cms.Source("EmptySource")
0478
0479         # modify source in case of run-dependent MC
0480         self.runsAndWeights=None
0481         if self._options.runsAndWeightsForMC or self._options.runsScenarioForMC :
0482             if not self._options.isMC :
0483                 raise Exception("options --runsAndWeightsForMC and --runsScenarioForMC are only valid for MC")
0484             if self._options.runsAndWeightsForMC:
0485                 self.runsAndWeights = eval(self._options.runsAndWeightsForMC)
0486             else:
0487                 from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights
0488                 if isinstance(RunsAndWeights[self._options.runsScenarioForMC], str):
0489                     __import__(RunsAndWeights[self._options.runsScenarioForMC])
0490                     self.runsAndWeights = sys.modules[RunsAndWeights[self._options.runsScenarioForMC]].runProbabilityDistribution
0491                 else:
0492                     self.runsAndWeights = RunsAndWeights[self._options.runsScenarioForMC]
0493
0494         if self.runsAndWeights:
0495             import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun
0496             ThrowAndSetRandomRun.throwAndSetRandomRun(self.process.source,self.runsAndWeights)
0497             self.additionalCommands.append('import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun')
0498             self.additionalCommands.append('ThrowAndSetRandomRun.throwAndSetRandomRun(process.source,%s)'%(self.runsAndWeights))
0499
0500         # modify source in case of run-dependent MC (Run-3 method)
0501         self.runsAndWeightsInt=None
0502         if self._options.runsAndWeightsForMCIntegerWeights or self._options.runsScenarioForMCIntegerWeights:
0503             if not self._options.isMC :
0504                 raise Exception("options --runsAndWeightsForMCIntegerWeights and --runsScenarioForMCIntegerWeights are only valid for MC")
0505             if self._options.runsAndWeightsForMCIntegerWeights:
0506                 self.runsAndWeightsInt = eval(self._options.runsAndWeightsForMCIntegerWeights)
0507             else:
0508                 from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights
0509                 if isinstance(RunsAndWeights[self._options.runsScenarioForMCIntegerWeights], str):
0510                     __import__(RunsAndWeights[self._options.runsScenarioForMCIntegerWeights])
0511                     self.runsAndWeightsInt = sys.modules[RunsAndWeights[self._options.runsScenarioForMCIntegerWeights]].runProbabilityDistribution
0512                 else:
0513                     self.runsAndWeightsInt = RunsAndWeights[self._options.runsScenarioForMCIntegerWeights]
0514
0515         if self.runsAndWeightsInt:
0516             if not self._options.relval:
0517                 raise Exception("--relval option required when using --runsAndWeightsInt")
0518             if 'DATAMIX' in self._options.step:
0519                 from SimGeneral.Configuration.LumiToRun import lumi_to_run
0520                 total_events, events_per_job  = self._options.relval.split(',')
0521                 lumi_to_run_mapping = lumi_to_run(self.runsAndWeightsInt, int(total_events), int(events_per_job))
0522                 self.additionalCommands.append("process.source.firstLuminosityBlockForEachRun = cms.untracked.VLuminosityBlockID(*[cms.LuminosityBlockID(x,y) for x,y in " + str(lumi_to_run_mapping) + "])")
0523
0524         return
0525
0526     def addOutput(self):
0527         """ Add output module to the process """
0528         result=""
0529         if self._options.outputDefinition:
0530             if self._options.datatier:
0531                 print("--datatier & --eventcontent options ignored")
0532
0533             #new output convention with a list of dict
0534             outList = eval(self._options.outputDefinition)
0535             for (id,outDefDict) in enumerate(outList):
0536                 outDefDictStr=outDefDict.__str__()
0537                 if not isinstance(outDefDict,dict):
0538                     raise Exception("--output needs to be passed a list of dict"+self._options.outputDefinition+" is invalid")
0539                 #requires option: tier
0540                 theTier=anyOf(['t','tier','dataTier'],outDefDict)
0541                 #optional option: eventcontent, filtername, selectEvents, moduleLabel, filename
0542                 ## event content
0543                 theStreamType=anyOf(['e','ec','eventContent','streamType'],outDefDict,theTier)
0544                 theFilterName=anyOf(['f','ftN','filterName'],outDefDict,'')
0545                 theSelectEvent=anyOf(['s','sE','selectEvents'],outDefDict,'')
0546                 theModuleLabel=anyOf(['l','mL','moduleLabel'],outDefDict,'')
0547                 theExtraOutputCommands=anyOf(['o','oC','outputCommands'],outDefDict,'')
0548                 # module label has a particular role
0549                 if not theModuleLabel:
0550                     tryNames=[theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+'output',
0551                               theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+'output',
0552                               theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+theSelectEvent.split(',')[0].replace(':','for').replace(' ','')+'output'
0553                               ]
0554                     for name in tryNames:
0555                         if not hasattr(self.process,name):
0556                             theModuleLabel=name
0557                             break
0558                 if not theModuleLabel:
0559                     raise Exception("cannot find a module label for specification: "+outDefDictStr)
0560                 if id==0:
0561                     defaultFileName=self._options.outfile_name
0562                 else:
0563                     defaultFileName=self._options.outfile_name.replace('.root','_in'+theTier+'.root')
0564
0565                 theFileName=self._options.dirout+anyOf(['fn','fileName'],outDefDict,defaultFileName)
0566                 if not theFileName.endswith('.root'):
0567                     theFileName+='.root'
0568
0569                 if len(outDefDict):
0570                     raise Exception("unused keys from --output options: "+','.join(outDefDict.keys()))
0571                 if theStreamType=='DQMIO': theStreamType='DQM'
0572                 if theStreamType=='ALL':
0573                     theEventContent = cms.PSet(outputCommands = cms.untracked.vstring('keep *'))
0574                 else:
0575                     theEventContent = getattr(self.process, theStreamType+"EventContent")
0576
0577
0578                 addAlCaSelects=False
0579                 if theStreamType=='ALCARECO' and not theFilterName:
0580                     theFilterName='StreamALCACombined'
0581                     addAlCaSelects=True
0582
0583                 CppType='PoolOutputModule'
0584                 if self._options.timeoutOutput:
0585                     CppType='TimeoutPoolOutputModule'
0586                 if theStreamType=='DQM' and theTier=='DQMIO': CppType='DQMRootOutputModule'
0587                 output = cms.OutputModule(CppType,
0588                                           theEventContent.clone(),
0589                                           fileName = cms.untracked.string(theFileName),
0590                                           dataset = cms.untracked.PSet(
0591                                              dataTier = cms.untracked.string(theTier),
0592                                              filterName = cms.untracked.string(theFilterName))
0593                                           )
0594                 if not theSelectEvent and hasattr(self.process,'generation_step') and theStreamType!='LHE':
0595                     output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step'))
0596                 if not theSelectEvent and hasattr(self.process,'filtering_step'):
0597                     output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step'))
0598                 if theSelectEvent:
0599                     output.SelectEvents =cms.untracked.PSet(SelectEvents = cms.vstring(theSelectEvent))
0600
0601                 if addAlCaSelects:
0602                     if not hasattr(output,'SelectEvents'):
0603                         output.SelectEvents=cms.untracked.PSet(SelectEvents=cms.vstring())
0604                     for alca in self.AlCaPaths:
0605                         output.SelectEvents.SelectEvents.extend(getattr(self.process,'OutALCARECO'+alca).SelectEvents.SelectEvents)
0606
0607
0608                 if hasattr(self.process,theModuleLabel):
0609                     raise Exception("the current process already has a module "+theModuleLabel+" defined")
0610                 #print "creating output module ",theModuleLabel
0611                 setattr(self.process,theModuleLabel,output)
0612                 outputModule=getattr(self.process,theModuleLabel)
0613                 setattr(self.process,theModuleLabel+'_step',cms.EndPath(outputModule))
0614                 path=getattr(self.process,theModuleLabel+'_step')
0615                 self.schedule.append(path)
0616
0617                 if not self._options.inlineEventContent and hasattr(self.process,theStreamType+"EventContent"):
0618                     def doNotInlineEventContent(instance,label = "cms.untracked.vstring(process."+theStreamType+"EventContent.outputCommands)"):
0619                         return label
0620                     outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
0621                 if theExtraOutputCommands:
0622                     if not isinstance(theExtraOutputCommands,list):
0623                         raise Exception("extra ouput command in --option must be a list of strings")
0624                     if hasattr(self.process,theStreamType+"EventContent"):
0625                         self.executeAndRemember('process.%s.outputCommands.extend(%s)'%(theModuleLabel,theExtraOutputCommands))
0626                     else:
0627                         outputModule.outputCommands.extend(theExtraOutputCommands)
0628
0629                 result+="\nprocess."+theModuleLabel+" = "+outputModule.dumpPython()
0630
0631             ##ends the --output options model
0632             return result
0633
0634         streamTypes=self._options.eventcontent.split(',')
0635         tiers=self._options.datatier.split(',')
0636         if not self._options.outputDefinition and len(streamTypes)!=len(tiers):
0637             raise Exception("number of event content arguments does not match number of datatier arguments")
0638
0639         # if the only step is alca we don't need to put in an output
0640         if self._options.step.split(',')[0].split(':')[0] == 'ALCA':
0641             return "\n"
0642
0643         for i,(streamType,tier) in enumerate(zip(streamTypes,tiers)):
0644             if streamType=='': continue
0645             if streamType == 'ALCARECO' and not 'ALCAPRODUCER' in self._options.step: continue
0646             if streamType=='DQMIO': streamType='DQM'
0647             eventContent=streamType
0648             ## override streamType to eventContent in case NANOEDM
0649             if streamType == "NANOEDMAOD" :
0650                 eventContent = "NANOAOD"
0651             elif streamType == "NANOEDMAODSIM" :
0652                 eventContent = "NANOAODSIM"
0653             theEventContent = getattr(self.process, eventContent+"EventContent")
0654             if i==0:
0655                 theFileName=self._options.outfile_name
0656                 theFilterName=self._options.filtername
0657             else:
0658                 theFileName=self._options.outfile_name.replace('.root','_in'+streamType+'.root')
0659                 theFilterName=self._options.filtername
0660             CppType='PoolOutputModule'
0661             if self._options.timeoutOutput:
0662                 CppType='TimeoutPoolOutputModule'
0663             if streamType=='DQM' and tier=='DQMIO': CppType='DQMRootOutputModule'
0664             if "NANOAOD" in streamType : CppType='NanoAODOutputModule'
0665             output = cms.OutputModule(CppType,
0666                                       theEventContent,
0667                                       fileName = cms.untracked.string(theFileName),
0668                                       dataset = cms.untracked.PSet(dataTier = cms.untracked.string(tier),
0669                                                                    filterName = cms.untracked.string(theFilterName)
0670                                                                    )
0671                                       )
0672             if hasattr(self.process,"generation_step") and streamType!='LHE':
0673                 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step'))
0674             if hasattr(self.process,"filtering_step"):
0675                 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step'))
0676
0677             if streamType=='ALCARECO':
0678                 output.dataset.filterName = cms.untracked.string('StreamALCACombined')
0679
0680             if "MINIAOD" in streamType:
0681                 from PhysicsTools.PatAlgos.slimming.miniAOD_tools import miniAOD_customizeOutput
0682                 miniAOD_customizeOutput(output)
0683
0684             outputModuleName=streamType+'output'
0685             setattr(self.process,outputModuleName,output)
0686             outputModule=getattr(self.process,outputModuleName)
0687             setattr(self.process,outputModuleName+'_step',cms.EndPath(outputModule))
0688             path=getattr(self.process,outputModuleName+'_step')
0689             self.schedule.append(path)
0690
0691             if self._options.outputCommands and streamType!='DQM':
0692                 for evct in self._options.outputCommands.split(','):
0693                     if not evct: continue
0694                     self.executeAndRemember("process.%s.outputCommands.append('%s')"%(outputModuleName,evct.strip()))
0695
0696             if not self._options.inlineEventContent:
0697                 tmpstreamType=streamType
0698                 if "NANOEDM" in tmpstreamType :
0699                     tmpstreamType=tmpstreamType.replace("NANOEDM","NANO")
0700                 def doNotInlineEventContent(instance,label = "process."+tmpstreamType+"EventContent.outputCommands"):
0701                     return label
0702                 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
0703
0704             result+="\nprocess."+outputModuleName+" = "+outputModule.dumpPython()
0705
0706         return result
0707
0708     def addStandardSequences(self):
0709         """
0710         Add selected standard sequences to the process
0711         """
0712         # load the pile up file
0713         if self._options.pileup:
0714             pileupSpec=self._options.pileup.split(',')[0]
0715
0716             # Does the requested pile-up scenario exist?
0717             from Configuration.StandardSequences.Mixing import Mixing,defineMixing
0718             if not pileupSpec in Mixing and '.' not in pileupSpec and 'file:' not in pileupSpec:
0719                 message = pileupSpec+' is not a know mixing scenario:\n available are: '+'\n'.join(Mixing.keys())
0720                 raise Exception(message)
0721
0722             # Put mixing parameters in a dictionary
0723             if '.' in pileupSpec:
0724                 mixingDict={'file':pileupSpec}
0725             elif pileupSpec.startswith('file:'):
0726                 mixingDict={'file':pileupSpec[5:]}
0727             else:
0728                 import copy
0729                 mixingDict=copy.copy(Mixing[pileupSpec])
0730             if len(self._options.pileup.split(','))>1:
0731                 mixingDict.update(eval(self._options.pileup[self._options.pileup.find(',')+1:]))
0732
0733             # Load the pu cfg file corresponding to the requested pu scenario
0734             if 'file:' in pileupSpec:
0735                 #the file is local
0736                 self.process.load(mixingDict['file'])
0737                 print("inlining mixing module configuration")
0738                 self._options.inlineObjets+=',mix'
0739             else:
0740                 self.loadAndRemember(mixingDict['file'])
0741
0742             mixingDict.pop('file')
0743             if not "DATAMIX" in self.stepMap.keys(): # when DATAMIX is present, pileup_input refers to pre-mixed GEN-RAW
0744                 if self._options.pileup_input:
0745                     if self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:'):
0746                         mixingDict['F']=filesFromDASQuery('file dataset = %s'%(self._options.pileup_input[4:],),self._options.pileup_dasoption)[0]
0747                     elif self._options.pileup_input.startswith("filelist:"):
0748                         mixingDict['F']=(filesFromList(self._options.pileup_input[9:]))[0]
0749                     else:
0750                         mixingDict['F']=self._options.pileup_input.split(',')
0751                 specialization=defineMixing(mixingDict)
0752                 for command in specialization:
0753                     self.executeAndRemember(command)
0754                 if len(mixingDict)!=0:
0755                     raise Exception('unused mixing specification: '+mixingDict.keys().__str__())
0756
0757
0758         # load the geometry file
0759         try:
0760             if len(self.stepMap):
0761                 self.loadAndRemember(self.GeometryCFF)
0762                 if ('SIM' in self.stepMap or 'reSIM' in self.stepMap) and not self._options.fast:
0763                     self.loadAndRemember(self.SimGeometryCFF)
0764                     if self.geometryDBLabel:
0765                         self.executeAndRemember('if hasattr(process, "XMLFromDBSource"): process.XMLFromDBSource.label="%s"'%(self.geometryDBLabel))
0766                         self.executeAndRemember('if hasattr(process, "DDDetectorESProducerFromDB"): process.DDDetectorESProducerFromDB.label="%s"'%(self.geometryDBLabel))
0767
0768         except ImportError:
0769             print("Geometry option",self._options.geometry,"unknown.")
0770             raise
0771
0772         if len(self.stepMap):
0773             self.loadAndRemember(self.magFieldCFF)
0774
0775         for stepName in self.stepKeys:
0776             stepSpec = self.stepMap[stepName]
0777             print("Step:", stepName,"Spec:",stepSpec)
0778             if stepName.startswith('re'):
0779                 ##add the corresponding input content
0780                 if stepName[2:] not in self._options.donotDropOnInput:
0781                     self._options.inputEventContent='%s,%s'%(stepName.upper(),self._options.inputEventContent)
0782                 stepName=stepName[2:]
0783             if stepSpec=="":
0784                 getattr(self,"prepare_"+stepName)(sequence = getattr(self,stepName+"DefaultSeq"))
0785             elif isinstance(stepSpec, list):
0786                 getattr(self,"prepare_"+stepName)(sequence = '+'.join(stepSpec))
0787             elif isinstance(stepSpec, tuple):
0788                 getattr(self,"prepare_"+stepName)(sequence = ','.join([stepSpec[1],'+'.join(stepSpec[0])]))
0789             else:
0790                 raise ValueError("Invalid step definition")
0791
0792         if self._options.restoreRNDSeeds!=False:
0793             #it is either True, or a process name
0794             if self._options.restoreRNDSeeds==True:
0795                 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateLabel=cms.untracked.string("randomEngineStateProducer")')
0796             else:
0797                 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateTag=cms.untracked.InputTag("randomEngineStateProducer","","%s")'%(self._options.restoreRNDSeeds))
0798             if self._options.inputEventContent or self._options.inputCommands:
0799                 if self._options.inputCommands:
0800                     self._options.inputCommands+='keep *_randomEngineStateProducer_*_*,'
0801                 else:
0802                     self._options.inputCommands='keep *_randomEngineStateProducer_*_*,'
0803
0804
0805     def completeInputCommand(self):
0806         if self._options.inputEventContent:
0807             import copy
0808             def dropSecondDropStar(iec):
0809                 #drop occurence of 'drop *' in the list
0810                 count=0
0811                 for item in iec:
0812                     if item=='drop *':
0813                         if count!=0:
0814                             iec.remove(item)
0815                         count+=1
0816
0817             ## allow comma separated input eventcontent
0818             if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring()
0819             for evct in self._options.inputEventContent.split(','):
0820                 if evct=='': continue
0821                 theEventContent = getattr(self.process, evct+"EventContent")
0822                 if hasattr(theEventContent,'outputCommands'):
0823                     self.process.source.inputCommands.extend(copy.copy(theEventContent.outputCommands))
0824                 if hasattr(theEventContent,'inputCommands'):
0825                     self.process.source.inputCommands.extend(copy.copy(theEventContent.inputCommands))
0826
0827             dropSecondDropStar(self.process.source.inputCommands)
0828
0829             if not self._options.dropDescendant:
0830                 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False)
0831
0832
0833         return
0834
0835     def addConditions(self):
0836         """Add conditions to the process"""
0837         if not self._options.conditions: return
0838
0839         if 'FrontierConditions_GlobalTag' in self._options.conditions:
0840             print('using FrontierConditions_GlobalTag in --conditions is not necessary anymore and will be deprecated soon. please update your command line')
0841             self._options.conditions = self._options.conditions.replace("FrontierConditions_GlobalTag,",'')
0842
0843         self.loadAndRemember(self.ConditionsDefaultCFF)
0844         from Configuration.AlCa.GlobalTag import GlobalTag
0845         self.process.GlobalTag = GlobalTag(self.process.GlobalTag, self._options.conditions, self._options.custom_conditions)
0846         self.additionalCommands.append('from Configuration.AlCa.GlobalTag import GlobalTag')
0847         self.additionalCommands.append('process.GlobalTag = GlobalTag(process.GlobalTag, %s, %s)' % (repr(self._options.conditions), repr(self._options.custom_conditions)))
0848
0849
0850     def addCustomise(self,unsch=0):
0851         """Include the customise code """
0852
0853         custOpt=[]
0854         if unsch==0:
0855             for c in self._options.customisation_file:
0856                 custOpt.extend(c.split(","))
0857         else:
0858             for c in self._options.customisation_file_unsch:
0859                 custOpt.extend(c.split(","))
0860
0861         custMap=DictTypes.SortedKeysDict()
0862         for opt in custOpt:
0863             if opt=='': continue
0864             if opt.count('.')>1:
0865                 raise Exception("more than . in the specification:"+opt)
0866             fileName=opt.split('.')[0]
0867             if opt.count('.')==0:   rest='customise'
0868             else:
0869                 rest=opt.split('.')[1]
0870                 if rest=='py': rest='customise' #catch the case of --customise file.py
0871
0872             if fileName in custMap:
0873                 custMap[fileName].extend(rest.split('+'))
0874             else:
0875                 custMap[fileName]=rest.split('+')
0876
0877         if len(custMap)==0:
0878             final_snippet='\n'
0879         else:
0880             final_snippet='\n# customisation of the process.\n'
0881
0882         allFcn=[]
0883         for opt in custMap:
0884             allFcn.extend(custMap[opt])
0885         for fcn in allFcn:
0886             if allFcn.count(fcn)!=1:
0887                 raise Exception("cannot specify twice "+fcn+" as a customisation method")
0888
0889         for f in custMap:
0890             # let python search for that package and do syntax checking at the same time
0891             packageName = f.replace(".py","").replace("/",".")
0892             __import__(packageName)
0893             package = sys.modules[packageName]
0894
0895             # now ask the package for its definition and pick .py instead of .pyc
0896             customiseFile = re.sub(r'\.pyc$', '.py', package.__file__)
0897
0898             final_snippet+='\n# Automatic addition of the customisation function from '+packageName+'\n'
0899             if self._options.inline_custom:
0900                 for line in file(customiseFile,'r'):
0901                     if "import FWCore.ParameterSet.Config" in line:
0902                         continue
0903                     final_snippet += line
0904             else:
0905                 final_snippet += 'from %s import %s \n'%(packageName,','.join(custMap[f]))
0906             for fcn in custMap[f]:
0907                 print("customising the process with",fcn,"from",f)
0908                 if not hasattr(package,fcn):
0909                     #bound to fail at run time
0910                     raise Exception("config "+f+" has no function "+fcn)
0911                 #execute the command
0912                 self.process=getattr(package,fcn)(self.process)
0913                 #and print it in the configuration
0914                 final_snippet += "\n#call to customisation function "+fcn+" imported from "+packageName
0915                 final_snippet += "\nprocess = %s(process)\n"%(fcn,)
0916
0917         if len(custMap)!=0:
0918             final_snippet += '\n# End of customisation functions\n'
0919
0920         ### now for a useful command
0921         return final_snippet
0922
0923     def addCustomiseCmdLine(self):
0924         final_snippet='\n# Customisation from command line\n'
0925         if self._options.customise_commands:
0926             import string
0927             for com in self._options.customise_commands.split('\\n'):
0928                 com=com.lstrip()
0929                 self.executeAndRemember(com)
0930                 final_snippet +='\n'+com
0931
0932         return final_snippet
0933
0934     #----------------------------------------------------------------------------
0935     # here the methods to define the python includes for each step or
0936     # conditions
0937     #----------------------------------------------------------------------------
0938     def define_Configs(self):
0939         if len(self.stepMap):
0940             self.loadAndRemember('Configuration/StandardSequences/Services_cff')
0941         if self._options.particleTable not in defaultOptions.particleTableList:
0942             print('Invalid particle table provided. Options are:')
0943             print(defaultOptions.particleTable)
0944             sys.exit(-1)
0945         else:
0946             if len(self.stepMap):
0947                 self.loadAndRemember('SimGeneral.HepPDTESSource.'+self._options.particleTable+'_cfi')
0948
0949         self.loadAndRemember('FWCore/MessageService/MessageLogger_cfi')
0950
0951         self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreams_cff"
0952         self.GENDefaultCFF="Configuration/StandardSequences/Generator_cff"
0953         self.SIMDefaultCFF="Configuration/StandardSequences/Sim_cff"
0954         self.DIGIDefaultCFF="Configuration/StandardSequences/Digi_cff"
0955         self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRaw_cff"
0956         self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1Emulator_cff'
0957         self.L1MENUDefaultCFF="Configuration/StandardSequences/L1TriggerDefaultMenu_cff"
0958         self.HLTDefaultCFF="Configuration/StandardSequences/HLTtable_cff"
0959         self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_Data_cff"
0960         if self._options.isRepacked: self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_DataMapper_cff"
0961         self.L1RecoDefaultCFF="Configuration/StandardSequences/L1Reco_cff"
0962         self.L1TrackTriggerDefaultCFF="Configuration/StandardSequences/L1TrackTrigger_cff"
0963         self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_Data_cff"
0964         self.RECOSIMDefaultCFF="Configuration/StandardSequences/RecoSim_cff"
0965         self.PATDefaultCFF="Configuration/StandardSequences/PAT_cff"
0966         self.NANODefaultCFF="PhysicsTools/NanoAOD/nano_cff"
0967         self.NANOGENDefaultCFF="PhysicsTools/NanoAOD/nanogen_cff"
0968         self.EIDefaultCFF=None
0969         self.SKIMDefaultCFF="Configuration/StandardSequences/Skims_cff"
0970         self.POSTRECODefaultCFF="Configuration/StandardSequences/PostRecoGenerator_cff"
0971         self.VALIDATIONDefaultCFF="Configuration/StandardSequences/Validation_cff"
0972         self.L1HwValDefaultCFF = "Configuration/StandardSequences/L1HwVal_cff"
0973         self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOffline_cff"
0974         self.HARVESTINGDefaultCFF="Configuration/StandardSequences/Harvesting_cff"
0975         self.ALCAHARVESTDefaultCFF="Configuration/StandardSequences/AlCaHarvesting_cff"
0976         self.ENDJOBDefaultCFF="Configuration/StandardSequences/EndOfProcess_cff"
0977         self.ConditionsDefaultCFF = "Configuration/StandardSequences/FrontierConditions_GlobalTag_cff"
0978         self.CFWRITERDefaultCFF = "Configuration/StandardSequences/CrossingFrameWriter_cff"
0979         self.REPACKDefaultCFF="Configuration/StandardSequences/DigiToRaw_Repack_cff"
0980
0981         if "DATAMIX" in self.stepMap.keys():
0982             self.DATAMIXDefaultCFF="Configuration/StandardSequences/DataMixer"+self._options.datamix+"_cff"
0983             self.DIGIDefaultCFF="Configuration/StandardSequences/DigiDM_cff"
0984             self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRawDM_cff"
0985             self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1EmulatorDM_cff'
0986
0987         self.ALCADefaultSeq=None
0988         self.LHEDefaultSeq='externalLHEProducer'
0989         self.GENDefaultSeq='pgen'
0990         self.SIMDefaultSeq='psim'
0991         self.DIGIDefaultSeq='pdigi'
0992         self.DATAMIXDefaultSeq=None
0993         self.DIGI2RAWDefaultSeq='DigiToRaw'
0994         self.HLTDefaultSeq='GRun'
0995         self.L1DefaultSeq=None
0996         self.L1REPACKDefaultSeq='GT'
0997         self.HARVESTINGDefaultSeq=None
0998         self.ALCAHARVESTDefaultSeq=None
0999         self.CFWRITERDefaultSeq=None
1000         self.RAW2DIGIDefaultSeq='RawToDigi'
1001         self.L1RecoDefaultSeq='L1Reco'
1002         self.L1TrackTriggerDefaultSeq='L1TrackTrigger'
1003         if self._options.fast or ('RAW2DIGI' in self.stepMap and 'RECO' in self.stepMap):
1004             self.RECODefaultSeq='reconstruction'
1005         else:
1006             self.RECODefaultSeq='reconstruction_fromRECO'
1007         self.RECOSIMDefaultSeq='recosim'
1008         self.EIDefaultSeq='top'
1009         self.POSTRECODefaultSeq=None
1010         self.L1HwValDefaultSeq='L1HwVal'
1011         self.DQMDefaultSeq='DQMOffline'
1012         self.VALIDATIONDefaultSeq=''
1013         self.ENDJOBDefaultSeq='endOfProcess'
1014         self.REPACKDefaultSeq='DigiToRawRepack'
1015         self.PATDefaultSeq='miniAOD'
1016         self.PATGENDefaultSeq='miniGEN'
1017         #TODO: Check based of file input
1018         self.NANOGENDefaultSeq='nanogenSequence'
1019         self.NANODefaultSeq='nanoSequence'
1020
1021         self.EVTCONTDefaultCFF="Configuration/EventContent/EventContent_cff"
1022
1023         if not self._options.beamspot:
1024             self._options.beamspot=VtxSmearedDefaultKey
1025
1026         # if its MC then change the raw2digi
1027         if self._options.isMC==True:
1028             self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_cff"
1029             self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_cff"
1030             self.PATDefaultCFF="Configuration/StandardSequences/PATMC_cff"
1031             self.PATGENDefaultCFF="Configuration/StandardSequences/PATGEN_cff"
1032             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineMC_cff"
1033             self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreamsMC_cff"
1034             self.NANODefaultSeq='nanoSequenceMC'
1035         else:
1036             self._options.beamspot = None
1037
1038         #patch for gen, due to backward incompatibility
1039         if 'reGEN' in self.stepMap:
1040             self.GENDefaultSeq='fixGenInfo'
1041
1042         if self._options.scenario=='cosmics':
1043             self._options.pileup='Cosmics'
1044             self.DIGIDefaultCFF="Configuration/StandardSequences/DigiCosmics_cff"
1045             self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionCosmics_cff"
1046             self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsCosmics_cff"
1047             self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentCosmics_cff"
1048             self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationCosmics_cff"
1049             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmics_cff"
1050             if self._options.isMC==True:
1051                 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmicsMC_cff"
1052             self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingCosmics_cff"
1053             self.RECODefaultSeq='reconstructionCosmics'
1054             self.DQMDefaultSeq='DQMOfflineCosmics'
1055
1056         if self._options.scenario=='HeavyIons':
1057             if not self._options.beamspot:
1058                 self._options.beamspot=VtxSmearedHIDefaultKey
1059             self.HLTDefaultSeq = 'HIon'
1060             self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationHeavyIons_cff"
1061             self.VALIDATIONDefaultSeq=''
1062             self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentHeavyIons_cff"
1063             self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionHeavyIons_cff"
1064             self.RECODefaultSeq='reconstructionHeavyIons'
1065             self.ALCADefaultCFF = "Configuration/StandardSequences/AlCaRecoStreamsHeavyIons_cff"
1066             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIons_cff"
1067             self.DQMDefaultSeq='DQMOfflineHeavyIons'
1068             self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsHeavyIons_cff"
1069             self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingHeavyIons_cff"
1070             if self._options.isMC==True:
1071                 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIonsMC_cff"
1072
1073
1074         self.RAW2RECODefaultSeq=','.join([self.RAW2DIGIDefaultSeq,self.RECODefaultSeq])
1075
1076         self.USERDefaultSeq='user'
1077         self.USERDefaultCFF=None
1078
1079         # the magnetic field
1080         self.magFieldCFF = 'Configuration/StandardSequences/MagneticField_'+self._options.magField.replace('.','')+'_cff'
1081         self.magFieldCFF = self.magFieldCFF.replace("__",'_')
1082
1083         # the geometry
1084         self.GeometryCFF='Configuration/StandardSequences/GeometryRecoDB_cff'
1085         self.geometryDBLabel=None
1086         simGeometry=''
1087         if self._options.fast:
1088             if 'start' in self._options.conditions.lower():
1089                 self.GeometryCFF='FastSimulation/Configuration/Geometries_START_cff'
1090             else:
1091                 self.GeometryCFF='FastSimulation/Configuration/Geometries_MC_cff'
1092         else:
1093             def inGeometryKeys(opt):
1094                 from Configuration.StandardSequences.GeometryConf import GeometryConf
1095                 if opt in GeometryConf:
1096                     return GeometryConf[opt]
1097                 else:
1098                     return opt
1099
1100             geoms=self._options.geometry.split(',')
1101             if len(geoms)==1: geoms=inGeometryKeys(geoms[0]).split(',')
1102             if len(geoms)==2:
1103                 #may specify the reco geometry
1104                 if '/' in geoms[1] or '_cff' in geoms[1]:
1105                     self.GeometryCFF=geoms[1]
1106                 else:
1107                     self.GeometryCFF='Configuration/Geometry/Geometry'+geoms[1]+'_cff'
1108
1109             if (geoms[0].startswith('DB:')):
1110                 self.SimGeometryCFF='Configuration/StandardSequences/GeometrySimDB_cff'
1111                 self.geometryDBLabel=geoms[0][3:]
1112                 print("with DB:")
1113             else:
1114                 if '/' in geoms[0] or '_cff' in geoms[0]:
1115                     self.SimGeometryCFF=geoms[0]
1116                 else:
1117                     simGeometry=geoms[0]
1118                     if self._options.gflash==True:
1119                         self.SimGeometryCFF='Configuration/Geometry/Geometry'+geoms[0]+'GFlash_cff'
1120                     else:
1121                         self.SimGeometryCFF='Configuration/Geometry/Geometry'+geoms[0]+'_cff'
1122
1123         # synchronize the geometry configuration and the FullSimulation sequence to be used
1124         if simGeometry not in defaultOptions.geometryExtendedOptions:
1125             self.SIMDefaultCFF="Configuration/StandardSequences/SimIdeal_cff"
1126
1127         if self._options.scenario=='nocoll' or self._options.scenario=='cosmics':
1128             self.SIMDefaultCFF="Configuration/StandardSequences/SimNOBEAM_cff"
1129             self._options.beamspot='NoSmear'
1130
1131         # fastsim requires some changes to the default cff files and sequences
1132         if self._options.fast:
1133             self.SIMDefaultCFF = 'FastSimulation.Configuration.SimIdeal_cff'
1134             self.RECODefaultCFF= 'FastSimulation.Configuration.Reconstruction_AftMix_cff'
1135             self.RECOBEFMIXDefaultCFF = 'FastSimulation.Configuration.Reconstruction_BefMix_cff'
1136             self.RECOBEFMIXDefaultSeq = 'reconstruction_befmix'
1137             self.NANODefaultSeq = 'nanoSequenceFS'
1138             self.DQMOFFLINEDefaultCFF="DQMOffline.Configuration.DQMOfflineFS_cff"
1139
1140         # Mixing
1141         if self._options.pileup=='default':
1142             from Configuration.StandardSequences.Mixing import MixingDefaultKey
1143             self._options.pileup=MixingDefaultKey
1144
1145
1146         #not driven by a default cff anymore
1147         if self._options.isData:
1148             self._options.pileup=None
1149
1150
1151         self.REDIGIDefaultSeq=self.DIGIDefaultSeq
1152
1153     # for alca, skims, etc
1154     def addExtraStream(self, name, stream, workflow='full', cppType="PoolOutputModule"):
1155             # define output module and go from there
1156         output = cms.OutputModule(cppType)
1157         if stream.selectEvents.parameters_().__len__()!=0:
1158             output.SelectEvents = stream.selectEvents
1159         else:
1160             output.SelectEvents = cms.untracked.PSet()
1161             output.SelectEvents.SelectEvents=cms.vstring()
1162             if isinstance(stream.paths,tuple):
1163                 for path in stream.paths:
1164                     output.SelectEvents.SelectEvents.append(path.label())
1165             else:
1166                 output.SelectEvents.SelectEvents.append(stream.paths.label())
1167
1168
1169
1170         if isinstance(stream.content,str):
1171             evtPset=getattr(self.process,stream.content)
1172             for p in evtPset.parameters_():
1173                 setattr(output,p,getattr(evtPset,p))
1174             if not self._options.inlineEventContent:
1175                 def doNotInlineEventContent(instance,label = "process."+stream.content+".outputCommands"):
1176                     return label
1177                 output.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
1178         else:
1179             output.outputCommands = stream.content
1180
1181
1182         output.fileName = cms.untracked.string(self._options.dirout+stream.name+'.root')
1183
1184         output.dataset  = cms.untracked.PSet( dataTier = stream.dataTier,
1185                                               filterName = cms.untracked.string(stream.name))
1186
1187         if self._options.filtername:
1188             output.dataset.filterName= cms.untracked.string(self._options.filtername+"_"+stream.name)
1189
1190         if cppType == "PoolOutputModule":
1191             #add an automatic flushing to limit memory consumption
1192             output.eventAutoFlushCompressedSize=cms.untracked.int32(5*1024*1024)
1193
1194         if workflow in ("producers,full"):
1195             if isinstance(stream.paths,tuple):
1196                 for path in stream.paths:
1197                     self.schedule.append(path)
1198             else:
1199                 self.schedule.append(stream.paths)
1200
1201
1202         # in case of relvals we don't want to have additional outputs
1203         if (not self._options.relval) and workflow in ("full","output"):
1204             self.additionalOutputs[name] = output
1205             setattr(self.process,name,output)
1206
1207         if workflow == 'output':
1208             # adjust the select events to the proper trigger results from previous process
1209             filterList = output.SelectEvents.SelectEvents
1210             for i, filter in enumerate(filterList):
1211                 filterList[i] = filter+":"+self._options.triggerResultsProcess
1212
1213         return output
1214
1215     #----------------------------------------------------------------------------
1216     # here the methods to create the steps. Of course we are doing magic here ;)
1217     # prepare_STEPNAME modifies self.process and what else's needed.
1218     #----------------------------------------------------------------------------
1219
1220     def loadDefaultOrSpecifiedCFF(self, sequence,defaultCFF):
1221         if ( len(sequence.split('.'))==1 ):
1222             l=self.loadAndRemember(defaultCFF)
1223         elif ( len(sequence.split('.'))==2 ):
1224             l=self.loadAndRemember(sequence.split('.')[0])
1225             sequence=sequence.split('.')[1]
1226         else:
1227             print("sub sequence configuration must be of the form dir/subdir/cff.a+b+c or cff.a")
1228             print(sequence,"not recognized")
1229             raise
1230         return l
1231
1232     def scheduleSequence(self,seq,prefix,what='Path'):
1233         if '*' in seq:
1234             #create only one path with all sequences in it
1235             for i,s in enumerate(seq.split('*')):
1236                 if i==0:
1237                     setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, s) ))
1238                 else:
1239                     p=getattr(self.process,prefix)
1240                     tmp = getattr(self.process, s)
1241                     if isinstance(tmp, cms.Task):
1242                         p.associate(tmp)
1243                     else:
1244                         p+=tmp
1245             self.schedule.append(getattr(self.process,prefix))
1246             return
1247         else:
1248             #create as many path as many sequences
1249             if not '+' in seq:
1250                 if self.nextScheduleIsConditional:
1251                     self.conditionalPaths.append(prefix)
1252                 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, seq) ))
1253                 self.schedule.append(getattr(self.process,prefix))
1254             else:
1255                 for i,s in enumerate(seq.split('+')):
1256                     sn=prefix+'%d'%(i)
1257                     setattr(self.process,sn,getattr(cms,what)( getattr(self.process, s) ))
1258                     self.schedule.append(getattr(self.process,sn))
1259             return
1260
1261     def scheduleSequenceAtEnd(self,seq,prefix):
1262         self.scheduleSequence(seq,prefix,what='EndPath')
1263         return
1264
1265     def prepare_ALCAPRODUCER(self, sequence = None):
1266         self.prepare_ALCA(sequence, workflow = "producers")
1267
1268     def prepare_ALCAOUTPUT(self, sequence = None):
1269         self.prepare_ALCA(sequence, workflow = "output")
1270
1271     def prepare_ALCA(self, sequence = None, workflow = 'full'):
1272         """ Enrich the process with alca streams """
1273         alcaConfig=self.loadDefaultOrSpecifiedCFF(sequence,self.ALCADefaultCFF)
1274         sequence = sequence.split('.')[-1]
1275
1276         # decide which ALCA paths to use
1277         alcaList = sequence.split("+")
1278         maxLevel=0
1279         from Configuration.AlCa.autoAlca import autoAlca, AlCaNoConcurrentLumis
1280         # support @X from autoAlca.py, and recursion support: i.e T0:@Mu+@EG+...
1281         self.expandMapping(alcaList,autoAlca)
1282         self.AlCaPaths=[]
1283         for name in alcaConfig.__dict__:
1284             alcastream = getattr(alcaConfig,name)
1285             shortName = name.replace('ALCARECOStream','')
1286             if shortName in alcaList and isinstance(alcastream,cms.FilteredStream):
1287                 if shortName in AlCaNoConcurrentLumis:
1288                     print("Setting numberOfConcurrentLuminosityBlocks=1 because of AlCa sequence {}".format(shortName))
1289                     self._options.nConcurrentLumis = "1"
1290                     self._options.nConcurrentIOVs = "1"
1291                 isNano = (alcastream.dataTier == "NANOAOD")
1292                 output = self.addExtraStream(name, alcastream, workflow=workflow,
1293                         cppType=("NanoAODOutputModule" if isNano else "PoolOutputModule"))
1294                 self.executeAndRemember('process.ALCARECOEventContent.outputCommands.extend(process.OutALCARECO'+shortName+'_noDrop.outputCommands)')
1295                 self.AlCaPaths.append(shortName)
1296                 if 'DQM' in alcaList and not isNano:
1297                     if not self._options.inlineEventContent and hasattr(self.process,name):
1298                         self.executeAndRemember('process.' + name + '.outputCommands.append("keep *_MEtoEDMConverter_*_*")')
1299                     else:
1300                         output.outputCommands.append("keep *_MEtoEDMConverter_*_*")
1301
1302                 #rename the HLT process name in the alca modules
1303                 if self._options.hltProcess or 'HLT' in self.stepMap:
1304                     if isinstance(alcastream.paths,tuple):
1305                         for path in alcastream.paths:
1306                             self.renameHLTprocessInSequence(path.label())
1307                     else:
1308                         self.renameHLTprocessInSequence(alcastream.paths.label())
1309
1310                 for i in range(alcaList.count(shortName)):
1311                     alcaList.remove(shortName)
1312
1313             # DQM needs a special handling
1314             elif name == 'pathALCARECODQM' and 'DQM' in alcaList:
1315                 path = getattr(alcaConfig,name)
1316                 self.schedule.append(path)
1317                 alcaList.remove('DQM')
1318
1319             if isinstance(alcastream,cms.Path):
1320                 #black list the alca path so that they do not appear in the cfg
1321                 self.blacklist_paths.append(alcastream)
1322
1323
1324         if len(alcaList) != 0:
1325             available=[]
1326             for name in alcaConfig.__dict__:
1327                 alcastream = getattr(alcaConfig,name)
1328                 if isinstance(alcastream,cms.FilteredStream):
1329                     available.append(name.replace('ALCARECOStream',''))
1330             print("The following alcas could not be found "+str(alcaList))
1331             print("available ",available)
1332             #print "verify your configuration, ignoring for now"
1333             raise Exception("The following alcas could not be found "+str(alcaList))
1334
1335     def prepare_LHE(self, sequence = None):
1336             #load the fragment
1337             ##make it loadable
1338         loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','').replace('/','.')
1339         print("Loading lhe fragment from",loadFragment)
1340         __import__(loadFragment)
1341         self.process.load(loadFragment)
1342         ##inline the modules
1343         self._options.inlineObjets+=','+sequence
1344
1345         getattr(self.process,sequence).nEvents = int(self._options.number)
1346
1347         #schedule it
1348         self.process.lhe_step = cms.Path( getattr( self.process,sequence)  )
1349         self.excludedPaths.append("lhe_step")
1350         self.schedule.append( self.process.lhe_step )
1351
1352     def prepare_GEN(self, sequence = None):
1353         """ load the fragment of generator configuration """
1354         loadFailure=False
1355         #remove trailing .py
1356         #support old style .cfi by changing into something.cfi into something_cfi
1357         #remove python/ from the name
1358         loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','')
1359         #standard location of fragments
1360         if not '/' in loadFragment:
1361             loadFragment='Configuration.Generator.'+loadFragment
1362         else:
1363             loadFragment=loadFragment.replace('/','.')
1364         try:
1365             print("Loading generator fragment from",loadFragment)
1366             __import__(loadFragment)
1367         except:
1368             loadFailure=True
1369             #if self.process.source and self.process.source.type_()=='EmptySource':
1370             if not (self._options.filein or self._options.dasquery):
1371                 raise Exception("Neither gen fragment of input files provided: this is an inconsistent GEN step configuration")
1372
1373         if not loadFailure:
1374             from Configuration.Generator.concurrentLumisDisable import noConcurrentLumiGenerators
1375
1376             generatorModule=sys.modules[loadFragment]
1377             genModules=generatorModule.__dict__
1378             #remove lhe producer module since this should have been
1379             #imported instead in the LHE step
1380             if self.LHEDefaultSeq in genModules:
1381                 del genModules[self.LHEDefaultSeq]
1382
1383             if self._options.hideGen:
1384                 self.loadAndRemember(loadFragment)
1385             else:
1386                 self.process.load(loadFragment)
1387                 # expose the objects from that fragment to the configuration
1388                 import FWCore.ParameterSet.Modules as cmstypes
1389                 for name in genModules:
1390                     theObject = getattr(generatorModule,name)
1391                     if isinstance(theObject, cmstypes._Module):
1392                         self._options.inlineObjets=name+','+self._options.inlineObjets
1393                         if theObject.type_() in noConcurrentLumiGenerators:
1394                             print("Setting numberOfConcurrentLuminosityBlocks=1 because of generator {}".format(theObject.type_()))
1395                             self._options.nConcurrentLumis = "1"
1396                             self._options.nConcurrentIOVs = "1"
1397                     elif isinstance(theObject, cms.Sequence) or isinstance(theObject, cmstypes.ESProducer):
1398                         self._options.inlineObjets+=','+name
1399
1400             if sequence == self.GENDefaultSeq or sequence == 'pgen_genonly':
1401                 if 'ProductionFilterSequence' in genModules and ('generator' in genModules):
1402                     self.productionFilterSequence = 'ProductionFilterSequence'
1403                 elif 'generator' in genModules:
1404                     self.productionFilterSequence = 'generator'
1405
1406         """ Enrich the schedule with the rest of the generation step """
1407         self.loadDefaultOrSpecifiedCFF(sequence,self.GENDefaultCFF)
1408         genSeqName=sequence.split('.')[-1]
1409
1410         if True:
1411             try:
1412                 from Configuration.StandardSequences.VtxSmeared import VtxSmeared
1413                 cffToBeLoaded=VtxSmeared[self._options.beamspot]
1414                 self.loadAndRemember(cffToBeLoaded)
1415             except ImportError:
1416                 raise Exception("VertexSmearing type or beamspot "+self._options.beamspot+" unknown.")
1417
1418             if self._options.scenario == 'HeavyIons':
1419                 if self._options.pileup=='HiMixGEN':
1420                     self.loadAndRemember("Configuration/StandardSequences/GeneratorMix_cff")
1421                 elif self._options.pileup=='HiMixEmbGEN':
1422                     self.loadAndRemember("Configuration/StandardSequences/GeneratorEmbMix_cff")
1423                 else:
1424                     self.loadAndRemember("Configuration/StandardSequences/GeneratorHI_cff")
1425
1426         self.process.generation_step = cms.Path( getattr(self.process,genSeqName) )
1427         self.schedule.append(self.process.generation_step)
1428
1429         #register to the genstepfilter the name of the path (static right now, but might evolve)
1430         self.executeAndRemember('process.genstepfilter.triggerConditions=cms.vstring("generation_step")')
1431
1432         if 'reGEN' in self.stepMap:
1433             #stop here
1434             return
1435
1436         """ Enrich the schedule with the summary of the filter step """
1437         #the gen filter in the endpath
1438         self.loadAndRemember("GeneratorInterface/Core/genFilterSummary_cff")
1439         self.scheduleSequenceAtEnd('genFilterSummary','genfiltersummary_step')
1440         return
1441
1442     def prepare_SIM(self, sequence = None):
1443         """ Enrich the schedule with the simulation step"""
1444         self.loadDefaultOrSpecifiedCFF(sequence,self.SIMDefaultCFF)
1445         if not self._options.fast:
1446             if self._options.gflash==True:
1447                 self.loadAndRemember("Configuration/StandardSequences/GFlashSIM_cff")
1448
1449             if self._options.magField=='0T':
1450                 self.executeAndRemember("process.g4SimHits.UseMagneticField = cms.bool(False)")
1451         else:
1452             if self._options.magField=='0T':
1453                 self.executeAndRemember("process.fastSimProducer.detectorDefinition.magneticFieldZ = cms.untracked.double(0.)")
1454
1455         self.scheduleSequence(sequence.split('.')[-1],'simulation_step')
1456         return
1457
1458     def prepare_DIGI(self, sequence = None):
1459         """ Enrich the schedule with the digitisation step"""
1460         self.loadDefaultOrSpecifiedCFF(sequence,self.DIGIDefaultCFF)
1461
1462         if self._options.gflash==True:
1463             self.loadAndRemember("Configuration/StandardSequences/GFlashDIGI_cff")
1464
1465         if sequence == 'pdigi_valid' or sequence == 'pdigi_hi':
1466             self.executeAndRemember("process.mix.digitizers = cms.PSet(process.theDigitizersValid)")
1467
1468         if sequence != 'pdigi_nogen' and sequence != 'pdigi_valid_nogen' and sequence != 'pdigi_hi_nogen' and not self.process.source.type_()=='EmptySource' and not self._options.filetype == "LHE":
1469             if self._options.inputEventContent=='':
1470                 self._options.inputEventContent='REGEN'
1471             else:
1472                 self._options.inputEventContent=self._options.inputEventContent+',REGEN'
1473
1474
1475         self.scheduleSequence(sequence.split('.')[-1],'digitisation_step')
1476         return
1477
1478     def prepare_CFWRITER(self, sequence = None):
1479         """ Enrich the schedule with the crossing frame writer step"""
1480         self.loadAndRemember(self.CFWRITERDefaultCFF)
1481         self.scheduleSequence('pcfw','cfwriter_step')
1482         return
1483
1484     def prepare_DATAMIX(self, sequence = None):
1485         """ Enrich the schedule with the digitisation step"""
1486         self.loadAndRemember(self.DATAMIXDefaultCFF)
1487         self.scheduleSequence('pdatamix','datamixing_step')
1488
1489         if self._options.pileup_input:
1490             theFiles=''
1491             if self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:'):
1492                 theFiles=filesFromDASQuery('file dataset = %s'%(self._options.pileup_input[4:],),self._options.pileup_dasoption)[0]
1493             elif self._options.pileup_input.startswith("filelist:"):
1494                 theFiles= (filesFromList(self._options.pileup_input[9:]))[0]
1495             else:
1496                 theFiles=self._options.pileup_input.split(',')
1497             #print theFiles
1498             self.executeAndRemember( "process.mixData.input.fileNames = cms.untracked.vstring(%s)"%(  theFiles ) )
1499
1500         return
1501
1502     def prepare_DIGI2RAW(self, sequence = None):
1503         self.loadDefaultOrSpecifiedCFF(sequence,self.DIGI2RAWDefaultCFF)
1504         self.scheduleSequence(sequence.split('.')[-1],'digi2raw_step')
1505         return
1506
1507     def prepare_REPACK(self, sequence = None):
1508         self.loadDefaultOrSpecifiedCFF(sequence,self.REPACKDefaultCFF)
1509         self.scheduleSequence(sequence.split('.')[-1],'digi2repack_step')
1510         return
1511
1512     def prepare_L1(self, sequence = None):
1513         """ Enrich the schedule with the L1 simulation step"""
1514         assert(sequence == None)
1515         self.loadAndRemember(self.L1EMDefaultCFF)
1516         self.scheduleSequence('SimL1Emulator','L1simulation_step')
1517         return
1518
1519     def prepare_L1REPACK(self, sequence = None):
1520         """ Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection"""
1521         supported = ['GT','GT1','GT2','GCTGT','Full','FullSimTP','FullMC','Full2015Data','uGT','CalouGT']
1522         if sequence in supported:
1523             self.loadAndRemember('Configuration/StandardSequences/SimL1EmulatorRepack_%s_cff'%sequence)
1524             if self._options.scenario == 'HeavyIons':
1525                 self.renameInputTagsInSequence("SimL1Emulator","rawDataCollector","rawDataRepacker")
1526             self.scheduleSequence('SimL1Emulator','L1RePack_step')
1527         else:
1528             print("L1REPACK with '",sequence,"' is not supported! Supported choices are: ",supported)
1529             raise Exception('unsupported feature')
1530
1531
1532     def prepare_HLT(self, sequence = None):
1533         """ Enrich the schedule with the HLT simulation step"""
1534         if not sequence:
1535             print("no specification of the hlt menu has been given, should never happen")
1536             raise  Exception('no HLT sequence provided')
1537
1538         if '@' in sequence:
1539             # case where HLT:@something was provided
1540             from Configuration.HLT.autoHLT import autoHLT
1541             key = sequence[1:]
1542             if key in autoHLT:
1543                 sequence = autoHLT[key]
1544             else:
1545                 raise ValueError('no HLT mapping key "%s" found in autoHLT' % key)
1546
1547         if ',' in sequence:
1548             #case where HLT:something:something was provided
1549             self.executeAndRemember('import HLTrigger.Configuration.Utilities')
1550             optionsForHLT = {}
1551             if self._options.scenario == 'HeavyIons':
1552                 optionsForHLT['type'] = 'HIon'
1553             else:
1554                 optionsForHLT['type'] = 'GRun'
1555             optionsForHLTConfig = ', '.join('%s=%s' % (key, repr(val)) for (key, val) in optionsForHLT.items())
1556             if sequence == 'run,fromSource':
1557                 if hasattr(self.process.source,'firstRun'):
1558                     self.executeAndRemember('process.loadHltConfiguration("run:%%d"%%(process.source.firstRun.value()),%s)'%(optionsForHLTConfig))
1559                 elif hasattr(self.process.source,'setRunNumber'):
1560                     self.executeAndRemember('process.loadHltConfiguration("run:%%d"%%(process.source.setRunNumber.value()),%s)'%(optionsForHLTConfig))
1561                 else:
1562                     raise Exception('Cannot replace menu to load %s'%(sequence))
1563             else:
1564                 self.executeAndRemember('process.loadHltConfiguration("%s",%s)'%(sequence.replace(',',':'),optionsForHLTConfig))
1565         else:
1566             self.loadAndRemember('HLTrigger/Configuration/HLT_%s_cff'       % sequence)
1567
1568         if self._options.isMC:
1569             self._options.customisation_file.append("HLTrigger/Configuration/customizeHLTforMC.customizeHLTforMC")
1570
1571         if self._options.name != 'HLT':
1572             self.additionalCommands.append('from HLTrigger.Configuration.CustomConfigs import ProcessName')
1573             self.additionalCommands.append('process = ProcessName(process)')
1574             self.additionalCommands.append('')
1575             from HLTrigger.Configuration.CustomConfigs import ProcessName
1576             self.process = ProcessName(self.process)
1577
1578         self.schedule.append(self.process.HLTSchedule)
1579         [self.blacklist_paths.append(path) for path in self.process.HLTSchedule if isinstance(path,(cms.Path,cms.EndPath))]
1580
1581         #this is a fake, to be removed with fastim migration and HLT menu dump
1582         if self._options.fast:
1583             if not hasattr(self.process,'HLTEndSequence'):
1584                 self.executeAndRemember("process.HLTEndSequence = cms.Sequence( process.dummyModule )")
1585
1586
1587     def prepare_RAW2RECO(self, sequence = None):
1588         if ','in sequence:
1589             seqReco=sequence.split(',')[1]
1590             seqDigi=sequence.split(',')[0]
1591         else:
1592             print("RAW2RECO requires two specifications",sequence,"insufficient")
1593
1594         self.prepare_RAW2DIGI(seqDigi)
1595         self.prepare_RECO(seqReco)
1596         return
1597
1598     def prepare_RAW2DIGI(self, sequence = "RawToDigi"):
1599         self.loadDefaultOrSpecifiedCFF(sequence,self.RAW2DIGIDefaultCFF)
1600         self.scheduleSequence(sequence,'raw2digi_step')
1601          #          if self._options.isRepacked:
1602         #self.renameInputTagsInSequence(sequence)
1603         return
1604
1605     def prepare_PATFILTER(self, sequence=None):
1606         self.loadAndRemember("PhysicsTools/PatAlgos/slimming/metFilterPaths_cff")
1607         from PhysicsTools.PatAlgos.slimming.metFilterPaths_cff import allMetFilterPaths
1608         for filt in allMetFilterPaths:
1609             self.schedule.append(getattr(self.process,'Flag_'+filt))
1610
1611     def prepare_L1HwVal(self, sequence = 'L1HwVal'):
1612         ''' Enrich the schedule with L1 HW validation '''
1613         self.loadDefaultOrSpecifiedCFF(sequence,self.L1HwValDefaultCFF)
1614         #self.scheduleSequence(sequence.split('.')[-1],'l1hwval_step')
1615         print('\n\n\n DEPRECATED this has no action \n\n\n')
1616         return
1617
1618     def prepare_L1Reco(self, sequence = "L1Reco"):
1619         ''' Enrich the schedule with L1 reconstruction '''
1620         self.loadDefaultOrSpecifiedCFF(sequence,self.L1RecoDefaultCFF)
1621         self.scheduleSequence(sequence.split('.')[-1],'L1Reco_step')
1622         return
1623
1624     def prepare_L1TrackTrigger(self, sequence = "L1TrackTrigger"):
1625         ''' Enrich the schedule with L1 reconstruction '''
1626         self.loadDefaultOrSpecifiedCFF(sequence,self.L1TrackTriggerDefaultCFF)
1627         self.scheduleSequence(sequence.split('.')[-1],'L1TrackTrigger_step')
1628         return
1629
1630     def prepare_FILTER(self, sequence = None):
1631         ''' Enrich the schedule with a user defined filter sequence '''
1632         ## load the relevant part
1633         filterConfig=self.load(sequence.split('.')[0])
1634         filterSeq=sequence.split('.')[-1]
1635         ## print it in the configuration
1636         class PrintAllModules(object):
1637             def __init__(self):
1638                 self.inliner=''
1639                 pass
1640             def enter(self,visitee):
1641                 try:
1642                     label=visitee.label()
1643                     ##needs to be in reverse order
1644                     self.inliner=label+','+self.inliner
1645                 except:
1646                     pass
1647             def leave(self,v): pass
1648
1649         expander=PrintAllModules()
1650         getattr(self.process,filterSeq).visit( expander )
1651         self._options.inlineObjets+=','+expander.inliner
1652         self._options.inlineObjets+=','+filterSeq
1653
1654         ## put the filtering path in the schedule
1655         self.scheduleSequence(filterSeq,'filtering_step')
1656         self.nextScheduleIsConditional=True
1657         ## put it before all the other paths
1658         self.productionFilterSequence = filterSeq
1659
1660         return
1661
1662     def prepare_RECO(self, sequence = "reconstruction"):
1663         ''' Enrich the schedule with reconstruction '''
1664         self.loadDefaultOrSpecifiedCFF(sequence,self.RECODefaultCFF)
1665         self.scheduleSequence(sequence.split('.')[-1],'reconstruction_step')
1666         return
1667
1668     def prepare_RECOSIM(self, sequence = "recosim"):
1669         ''' Enrich the schedule with reconstruction '''
1670         self.loadDefaultOrSpecifiedCFF(sequence,self.RECOSIMDefaultCFF)
1671         self.scheduleSequence(sequence.split('.')[-1],'recosim_step')
1672         return
1673
1674     def prepare_RECOBEFMIX(self, sequence = "reconstruction"):
1675         ''' Enrich the schedule with the part of reconstruction that is done before mixing in FastSim'''
1676         if not self._options.fast:
1677             print("ERROR: this step is only implemented for FastSim")
1678             sys.exit()
1679         self.loadDefaultOrSpecifiedCFF(self.RECOBEFMIXDefaultSeq,self.RECOBEFMIXDefaultCFF)
1680         self.scheduleSequence(sequence.split('.')[-1],'reconstruction_befmix_step')
1681         return
1682
1683     def prepare_PAT(self, sequence = "miniAOD"):
1684         ''' Enrich the schedule with PAT '''
1685         self.prepare_PATFILTER(self)
1686         self.loadDefaultOrSpecifiedCFF(sequence,self.PATDefaultCFF)
1687         self.labelsToAssociate.append('patTask')
1688         if self._options.isData:
1689             self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllData")
1690         else:
1691             if self._options.fast:
1692                 self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllMCFastSim")
1693             else:
1694                 self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllMC")
1695
1696         if self._options.hltProcess:
1697             if len(self._options.customise_commands) > 1:
1698                 self._options.customise_commands = self._options.customise_commands + " \n"
1699             self._options.customise_commands = self._options.customise_commands + "process.patTrigger.processName = \""+self._options.hltProcess+"\"\n"
1700             self._options.customise_commands = self._options.customise_commands + "process.slimmedPatTrigger.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1701             self._options.customise_commands = self._options.customise_commands + "process.patMuons.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1702
1703 #            self.renameHLTprocessInSequence(sequence)
1704
1705         return
1706
1707     def prepare_PATGEN(self, sequence = "miniGEN"):
1708         ''' Enrich the schedule with PATGEN '''
1709         self.loadDefaultOrSpecifiedCFF(sequence,self.PATGENDefaultCFF) #this is unscheduled
1710         self.labelsToAssociate.append('patGENTask')
1711         if self._options.isData:
1712             raise Exception("PATGEN step can only run on MC")
1713         return
1714
1715     def prepare_NANO(self, sequence = "nanoAOD"):
1716         ''' Enrich the schedule with NANO '''
1717         self.loadDefaultOrSpecifiedCFF(sequence,self.NANODefaultCFF)
1718         self.scheduleSequence(sequence.split('.')[-1],'nanoAOD_step')
1719         custom = "nanoAOD_customizeData" if self._options.isData else "nanoAOD_customizeMC"
1720         self._options.customisation_file.insert(0,"PhysicsTools/NanoAOD/nano_cff."+custom)
1721         if self._options.hltProcess:
1722             if len(self._options.customise_commands) > 1:
1723                 self._options.customise_commands = self._options.customise_commands + " \n"
1724             self._options.customise_commands = self._options.customise_commands + "process.unpackedPatTrigger.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1725
1726     def prepare_NANOGEN(self, sequence = "nanoAOD"):
1727         ''' Enrich the schedule with NANOGEN '''
1728         # TODO: Need to modify this based on the input file type
1729         fromGen = any([x in self.stepMap for x in ['LHE', 'GEN', 'AOD']])
1730         self.loadDefaultOrSpecifiedCFF(sequence,self.NANOGENDefaultCFF)
1731         self.scheduleSequence(sequence.split('.')[-1],'nanoAOD_step')
1732         custom = "customizeNanoGEN" if fromGen else "customizeNanoGENFromMini"
1733         if self._options.runUnscheduled:
1734             self._options.customisation_file_unsch.insert(0, '.'.join([self.NANOGENDefaultCFF, custom]))
1735         else:
1736             self._options.customisation_file.insert(0, '.'.join([self.NANOGENDefaultCFF, custom]))
1737
1738     def prepare_EI(self, sequence = None):
1739         ''' Enrich the schedule with event interpretation '''
1740         from Configuration.StandardSequences.EventInterpretation import EventInterpretation
1741         if sequence in EventInterpretation:
1742             self.EIDefaultCFF = EventInterpretation[sequence]
1743             sequence = 'EIsequence'
1744         else:
1745             raise Exception('Cannot set %s event interpretation'%( sequence) )
1746         self.loadDefaultOrSpecifiedCFF(sequence,self.EIDefaultCFF)
1747         self.scheduleSequence(sequence.split('.')[-1],'eventinterpretaion_step')
1748         return
1749
1750     def prepare_SKIM(self, sequence = "all"):
1751         ''' Enrich the schedule with skimming fragments'''
1752         skimConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.SKIMDefaultCFF)
1753         sequence = sequence.split('.')[-1]
1754
1755         skimlist=sequence.split('+')
1756         ## support @Mu+DiJet+@Electron configuration via autoSkim.py
1757         from Configuration.Skimming.autoSkim import autoSkim
1758         self.expandMapping(skimlist,autoSkim)
1759
1760         #print "dictionnary for skims:",skimConfig.__dict__
1761         for skim in skimConfig.__dict__:
1762             skimstream = getattr(skimConfig,skim)
1763             if isinstance(skimstream,cms.Path):
1764             #black list the alca path so that they do not appear in the cfg
1765                 self.blacklist_paths.append(skimstream)
1766             if (not isinstance(skimstream,cms.FilteredStream)):
1767                 continue
1768             shortname = skim.replace('SKIMStream','')
1769             if (sequence=="all"):
1770                 self.addExtraStream(skim,skimstream)
1771             elif (shortname in skimlist):
1772                 self.addExtraStream(skim,skimstream)
1773                 #add a DQM eventcontent for this guy
1774                 if self._options.datatier=='DQM':
1775                     self.process.load(self.EVTCONTDefaultCFF)
1776                     skimstreamDQM = cms.FilteredStream(
1777                             responsible = skimstream.responsible,
1778                             name = skimstream.name+'DQM',
1779                             paths = skimstream.paths,
1780                             selectEvents = skimstream.selectEvents,
1781                             content = self._options.datatier+'EventContent',
1782                             dataTier = cms.untracked.string(self._options.datatier)
1783                             )
1784                     self.addExtraStream(skim+'DQM',skimstreamDQM)
1785                 for i in range(skimlist.count(shortname)):
1786                     skimlist.remove(shortname)
1787
1788
1789
1790         if (skimlist.__len__()!=0 and sequence!="all"):
1791             print('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
1792             raise Exception('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
1793
1794     def prepare_USER(self, sequence = None):
1795         ''' Enrich the schedule with a user defined sequence '''
1796         self.loadDefaultOrSpecifiedCFF(sequence,self.USERDefaultCFF)
1797         self.scheduleSequence(sequence.split('.')[-1],'user_step')
1798         return
1799
1800     def prepare_POSTRECO(self, sequence = None):
1801         """ Enrich the schedule with the postreco step """
1802         self.loadAndRemember(self.POSTRECODefaultCFF)
1803         self.scheduleSequence('postreco_generator','postreco_step')
1804         return
1805
1806
1807     def prepare_VALIDATION(self, sequence = 'validation'):
1808         print(sequence,"in preparing validation")
1809         self.loadDefaultOrSpecifiedCFF(sequence,self.VALIDATIONDefaultCFF)
1810         from Validation.Configuration.autoValidation import autoValidation
1811         #in case VALIDATION:something:somethingelse -> something,somethingelse
1812         sequence=sequence.split('.')[-1]
1813         if sequence.find(',')!=-1:
1814             prevalSeqName=sequence.split(',')[0].split('+')
1815             valSeqName=sequence.split(',')[1].split('+')
1816             self.expandMapping(prevalSeqName,autoValidation,index=0)
1817             self.expandMapping(valSeqName,autoValidation,index=1)
1818         else:
1819             if '@' in sequence:
1820                 prevalSeqName=sequence.split('+')
1821                 valSeqName=sequence.split('+')
1822                 self.expandMapping(prevalSeqName,autoValidation,index=0)
1823                 self.expandMapping(valSeqName,autoValidation,index=1)
1824             else:
1825                 postfix=''
1826                 if sequence:
1827                     postfix='_'+sequence
1828                 prevalSeqName=['prevalidation'+postfix]
1829                 valSeqName=['validation'+postfix]
1830                 if not hasattr(self.process,valSeqName[0]):
1831                     prevalSeqName=['']
1832                     valSeqName=[sequence]
1833
1834         def NFI(index):
1835             ##name from index, required to keep backward compatibility
1836             if index==0:
1837                 return ''
1838             else:
1839                 return '%s'%index
1840
1841
1842         #rename the HLT process in validation steps
1843         if ('HLT' in self.stepMap and not self._options.fast) or self._options.hltProcess:
1844             for s in valSeqName+prevalSeqName:
1845                 if s:
1846                     self.renameHLTprocessInSequence(s)
1847         for (i,s) in enumerate(prevalSeqName):
1848             if s:
1849                 setattr(self.process,'prevalidation_step%s'%NFI(i),  cms.Path( getattr(self.process, s)) )
1850                 self.schedule.append(getattr(self.process,'prevalidation_step%s'%NFI(i)))
1851
1852         for (i,s) in enumerate(valSeqName):
1853             setattr(self.process,'validation_step%s'%NFI(i), cms.EndPath( getattr(self.process, s)))
1854             self.schedule.append(getattr(self.process,'validation_step%s'%NFI(i)))
1855
1856         #needed in case the miniAODValidation sequence is run starting from AODSIM
1857         if 'PAT' in self.stepMap and not 'RECO' in self.stepMap:
1858             return
1859
1860         if not 'DIGI' in self.stepMap and not self._options.fast and not any(map( lambda s : s.startswith('genvalid'), valSeqName)):
1861             if self._options.restoreRNDSeeds==False and not self._options.restoreRNDSeeds==True:
1862                 self._options.restoreRNDSeeds=True
1863
1864         if not 'DIGI' in self.stepMap and not self._options.isData and not self._options.fast:
1865             self.executeAndRemember("process.mix.playback = True")
1866             self.executeAndRemember("process.mix.digitizers = cms.PSet()")
1867             self.executeAndRemember("for a in process.aliases: delattr(process, a)")
1868             self._options.customisation_file.append("SimGeneral/MixingModule/fullMixCustomize_cff.setCrossingFrameOn")
1869
1870         if hasattr(self.process,"genstepfilter") and len(self.process.genstepfilter.triggerConditions):
1871             #will get in the schedule, smoothly
1872             for (i,s) in enumerate(valSeqName):
1873                 getattr(self.process,'validation_step%s'%NFI(i)).insert(0, self.process.genstepfilter)
1874
1875         return
1876
1877
1878     class MassSearchReplaceProcessNameVisitor(object):
1879         """Visitor that travels within a cms.Sequence, looks for a parameter and replace its value
1880         It will climb down within PSets, VPSets and VInputTags to find its target"""
1881         def __init__(self, paramSearch, paramReplace, verbose=False, whitelist=()):
1882             self._paramReplace = paramReplace
1883             self._paramSearch = paramSearch
1884             self._verbose = verbose
1885             self._whitelist = whitelist
1886
1887         def doIt(self,pset,base):
1888             if isinstance(pset, cms._Parameterizable):
1889                 for name in pset.parameters_().keys():
1890                     # skip whitelisted parameters
1891                     if name in self._whitelist:
1892                         continue
1893                     # if I use pset.parameters_().items() I get copies of the parameter values
1894                     # so I can't modify the nested pset
1895                     value = getattr(pset,name)
1896                     type = value.pythonTypeName()
1897                     if type in ('cms.PSet', 'cms.untracked.PSet'):
1898                         self.doIt(value,base+"."+name)
1899                     elif type in ('cms.VPSet', 'cms.untracked.VPSet'):
1900                         for (i,ps) in enumerate(value): self.doIt(ps, "%s.%s[%d]"%(base,name,i) )
1901                     elif type in ('cms.string', 'cms.untracked.string'):
1902                         if value.value() == self._paramSearch:
1903                             if self._verbose: print("set string process name %s.%s %s ==> %s"% (base, name, value, self._paramReplace))
1904                             setattr(pset, name,self._paramReplace)
1905                     elif type in ('cms.VInputTag', 'cms.untracked.VInputTag'):
1906                         for (i,n) in enumerate(value):
1907                             if not isinstance(n, cms.InputTag):
1908                                 n=cms.InputTag(n)
1909                             if n.processName == self._paramSearch:
1910                                 # VInputTag can be declared as a list of strings, so ensure that n is formatted correctly
1911                                 if self._verbose:print("set process name %s.%s[%d] %s ==> %s " % (base, name, i, n, self._paramReplace))
1912                                 setattr(n,"processName",self._paramReplace)
1913                                 value[i]=n
1914                     elif type in ('cms.vstring', 'cms.untracked.vstring'):
1915                         for (i,n) in enumerate(value):
1916                             if n==self._paramSearch:
1917                                 getattr(pset,name)[i]=self._paramReplace
1918                     elif type in ('cms.InputTag', 'cms.untracked.InputTag'):
1919                         if value.processName == self._paramSearch:
1920                             if self._verbose: print("set process name %s.%s %s ==> %s " % (base, name, value, self._paramReplace))
1921                             setattr(getattr(pset, name),"processName",self._paramReplace)
1922
1923         def enter(self,visitee):
1924             label = ''
1925             try:
1926                 label = visitee.label()
1927             except AttributeError:
1928                 label = '<Module not in a Process>'
1929             except:
1930                 label = 'other execption'
1931             self.doIt(visitee, label)
1932
1933         def leave(self,visitee):
1934             pass
1935
1936     #visit a sequence to repalce all input tags
1937     def renameInputTagsInSequence(self,sequence,oldT="rawDataCollector",newT="rawDataRepacker"):
1938         print("Replacing all InputTag %s => %s"%(oldT,newT))
1939         from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag
1940         massSearchReplaceAnyInputTag(getattr(self.process,sequence),oldT,newT)
1941         loadMe='from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag'
1942         if not loadMe in self.additionalCommands:
1943             self.additionalCommands.append(loadMe)
1944         self.additionalCommands.append('massSearchReplaceAnyInputTag(process.%s,"%s","%s",False,True)'%(sequence,oldT,newT))
1945
1946     #change the process name used to address HLT results in any sequence
1947     def renameHLTprocessInSequence(self,sequence,proc=None,HLTprocess='HLT'):
1948         if self._options.hltProcess:
1949             proc=self._options.hltProcess
1950         else:
1951             proc=self.process.name_()
1952         if proc==HLTprocess:    return
1953         # look up all module in dqm sequence
1954         print("replacing %s process name - sequence %s will use '%s'" % (HLTprocess,sequence, proc))
1955         getattr(self.process,sequence).visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor(HLTprocess,proc,whitelist = ("subSystemFolder",)))
1956         if 'from Configuration.Applications.ConfigBuilder import ConfigBuilder' not in self.additionalCommands:
1957             self.additionalCommands.append('from Configuration.Applications.ConfigBuilder import ConfigBuilder')
1958         self.additionalCommands.append('process.%s.visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor("%s", "%s", whitelist = ("subSystemFolder",)))'% (sequence,HLTprocess, proc))
1959
1960
1961     def expandMapping(self,seqList,mapping,index=None):
1962         maxLevel=30
1963         level=0
1964         while '@' in repr(seqList) and level<maxLevel:
1965             level+=1
1966             for specifiedCommand in seqList:
1967                 if specifiedCommand.startswith('@'):
1968                     location=specifiedCommand[1:]
1969                     if not location in mapping:
1970                         raise Exception("Impossible to map "+location+" from "+repr(mapping))
1971                     mappedTo=mapping[location]
1972                     if index!=None:
1973                         mappedTo=mappedTo[index]
1974                     seqList.remove(specifiedCommand)
1975                     seqList.extend(mappedTo.split('+'))
1976                     break;
1977         if level==maxLevel:
1978             raise Exception("Could not fully expand "+repr(seqList)+" from "+repr(mapping))
1979
1980     def prepare_DQM(self, sequence = 'DQMOffline'):
1981         # this one needs replacement
1982
1983         # any 'DQM' job should use DQMStore in non-legacy mode (but not HARVESTING)
1984         self.loadAndRemember("DQMServices/Core/DQMStoreNonLegacy_cff")
1985         self.loadDefaultOrSpecifiedCFF(sequence,self.DQMOFFLINEDefaultCFF)
1986         sequenceList=sequence.split('.')[-1].split('+')
1987         postSequenceList=sequence.split('.')[-1].split('+')
1988         from DQMOffline.Configuration.autoDQM import autoDQM
1989         self.expandMapping(sequenceList,autoDQM,index=0)
1990         self.expandMapping(postSequenceList,autoDQM,index=1)
1991
1992         if len(set(sequenceList))!=len(sequenceList):
1993             sequenceList=list(set(sequenceList))
1994             print("Duplicate entries for DQM:, using",sequenceList)
1995
1996         pathName='dqmoffline_step'
1997         for (i,sequence) in enumerate(sequenceList):
1998             if (i!=0):
1999                 pathName='dqmoffline_%d_step'%(i)
2000
2001             if 'HLT' in self.stepMap.keys() or self._options.hltProcess:
2002                 self.renameHLTprocessInSequence(sequence)
2003
2004             setattr(self.process,pathName, cms.EndPath( getattr(self.process,sequence ) ) )
2005             self.schedule.append(getattr(self.process,pathName))
2006
2007             if hasattr(self.process,"genstepfilter") and len(self.process.genstepfilter.triggerConditions):
2008                 #will get in the schedule, smoothly
2009                 getattr(self.process,pathName).insert(0,self.process.genstepfilter)
2010
2011
2012         pathName='dqmofflineOnPAT_step'
2013         for (i,sequence) in enumerate(postSequenceList):
2014         #Fix needed to avoid duplication of sequences not defined in autoDQM or without a PostDQM
2015             if (sequenceList[i]==postSequenceList[i]):
2016                       continue
2017             if (i!=0):
2018                 pathName='dqmofflineOnPAT_%d_step'%(i)
2019
2020             setattr(self.process,pathName, cms.EndPath( getattr(self.process, sequence ) ) )
2021             self.schedule.append(getattr(self.process,pathName))
2022
2023     def prepare_HARVESTING(self, sequence = None):
2024         """ Enrich the process with harvesting step """
2025         self.DQMSaverCFF='Configuration/StandardSequences/DQMSaver'+self._options.harvesting+'_cff'
2026         self.loadAndRemember(self.DQMSaverCFF)
2027
2028         harvestingConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.HARVESTINGDefaultCFF)
2029         sequence = sequence.split('.')[-1]
2030
2031         # decide which HARVESTING paths to use
2032         harvestingList = sequence.split("+")
2033         from DQMOffline.Configuration.autoDQM import autoDQM
2034         from Validation.Configuration.autoValidation import autoValidation
2035         import copy
2036         combined_mapping = copy.deepcopy( autoDQM )
2037         combined_mapping.update( autoValidation )
2038         self.expandMapping(harvestingList,combined_mapping,index=-1)
2039
2040         if len(set(harvestingList))!=len(harvestingList):
2041             harvestingList=list(set(harvestingList))
2042             print("Duplicate entries for HARVESTING, using",harvestingList)
2043
2044         for name in harvestingList:
2045             if not name in harvestingConfig.__dict__:
2046                 print(name,"is not a possible harvesting type. Available are",harvestingConfig.__dict__.keys())
2047                 # trigger hard error, like for other sequence types
2048                 getattr(self.process, name)
2049                 continue
2050             harvestingstream = getattr(harvestingConfig,name)
2051             if isinstance(harvestingstream,cms.Path):
2052                 self.schedule.append(harvestingstream)
2053                 self.blacklist_paths.append(harvestingstream)
2054             if isinstance(harvestingstream,cms.Sequence):
2055                 setattr(self.process,name+"_step",cms.Path(harvestingstream))
2056                 self.schedule.append(getattr(self.process,name+"_step"))
2057
2058         self.scheduleSequence('DQMSaver','dqmsave_step')
2059         return
2060
2061     def prepare_ALCAHARVEST(self, sequence = None):
2062         """ Enrich the process with AlCaHarvesting step """
2063         harvestingConfig = self.loadAndRemember(self.ALCAHARVESTDefaultCFF)
2064         sequence=sequence.split(".")[-1]
2065
2066         # decide which AlcaHARVESTING paths to use
2067         harvestingList = sequence.split("+")
2068
2069
2070
2071         from Configuration.AlCa.autoPCL import autoPCL
2072         self.expandMapping(harvestingList,autoPCL)
2073
2074         for name in harvestingConfig.__dict__:
2075             harvestingstream = getattr(harvestingConfig,name)
2076             if name in harvestingList and isinstance(harvestingstream,cms.Path):
2077                 self.schedule.append(harvestingstream)
2078                 if isinstance(getattr(harvestingConfig,"ALCAHARVEST" + name + "_dbOutput"), cms.VPSet) and \
2079                    isinstance(getattr(harvestingConfig,"ALCAHARVEST" + name + "_metadata"), cms.VPSet):
2080                     self.executeAndRemember("process.PoolDBOutputService.toPut.extend(process.ALCAHARVEST" + name + "_dbOutput)")
2081                     self.executeAndRemember("process.pclMetadataWriter.recordsToMap.extend(process.ALCAHARVEST" + name + "_metadata)")
2082                 else:
2083                     self.executeAndRemember("process.PoolDBOutputService.toPut.append(process.ALCAHARVEST" + name + "_dbOutput)")
2084                     self.executeAndRemember("process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVEST" + name + "_metadata)")
2085                 harvestingList.remove(name)
2086         # append the common part at the end of the sequence
2087         lastStep = getattr(harvestingConfig,"ALCAHARVESTDQMSaveAndMetadataWriter")
2088         self.schedule.append(lastStep)
2089
2090         if len(harvestingList) != 0 and 'dummyHarvesting' not in harvestingList :
2091             print("The following harvesting could not be found : ", harvestingList)
2092             raise Exception("The following harvesting could not be found : "+str(harvestingList))
2093
2094
2095
2096     def prepare_ENDJOB(self, sequence = 'endOfProcess'):
2097         self.loadDefaultOrSpecifiedCFF(sequence,self.ENDJOBDefaultCFF)
2098         self.scheduleSequenceAtEnd(sequence.split('.')[-1],'endjob_step')
2099         return
2100
2101     def finalizeFastSimHLT(self):
2102         self.process.reconstruction = cms.Path(self.process.reconstructionWithFamos)
2103         self.schedule.append(self.process.reconstruction)
2104
2105
2106     def build_production_info(self, evt_type, evtnumber):
2107         """ Add useful info for the production. """
2108         self.process.configurationMetadata=cms.untracked.PSet\
2109                                             (version=cms.untracked.string("$Revision: 1.19 $"),
2110                                              name=cms.untracked.string("Applications"),
2111                                              annotation=cms.untracked.string(evt_type+ " nevts:"+str(evtnumber))
2112                                              )
2113
2114         self.addedObjects.append(("Production Info","configurationMetadata"))
2115
2116
2117     def create_process(self):
2118         self.pythonCfgCode =  "# Auto generated configuration file\n"
2119         self.pythonCfgCode += "# using: \n# "+__version__[1:-1]+"\n# "+__source__[1:-1]+'\n'
2120         self.pythonCfgCode += "# with command line options: "+self._options.arguments+'\n'
2121         self.pythonCfgCode += "import FWCore.ParameterSet.Config as cms\n\n"
2122
2123         # now set up the modifies
2124         modifiers=[]
2125         modifierStrings=[]
2126         modifierImports=[]
2127
2128         if hasattr(self._options,"era") and self._options.era :
2129         # Multiple eras can be specified in a comma seperated list
2130             from Configuration.StandardSequences.Eras import eras
2131             for requestedEra in self._options.era.split(",") :
2132                 modifierStrings.append(requestedEra)
2133                 modifierImports.append(eras.pythonCfgLines[requestedEra])
2134                 modifiers.append(getattr(eras,requestedEra))
2135
2136
2137         if hasattr(self._options,"procModifiers") and self._options.procModifiers:
2138             import importlib
2139             thingsImported=[]
2140             for pm in self._options.procModifiers.split(','):
2141                 modifierStrings.append(pm)
2142                 modifierImports.append('from Configuration.ProcessModifiers.'+pm+'_cff import '+pm)
2143                 modifiers.append(getattr(importlib.import_module('Configuration.ProcessModifiers.'+pm+'_cff'),pm))
2144
2145         self.pythonCfgCode += '\n'.join(modifierImports)+'\n\n'
2146         self.pythonCfgCode += "process = cms.Process('"+self._options.name+"'" # Start of the line, finished after the loop
2147
2148
2149         if len(modifierStrings)>0:
2150             self.pythonCfgCode+= ','+','.join(modifierStrings)
2151         self.pythonCfgCode+=')\n\n'
2152
2153         #yes, the cfg code gets out of sync here if a process is passed in. That could be fixed in the future
2154         #assuming there is some way for the fwk to get the list of modifiers (and their stringified name)
2155         if self.process == None:
2156             if len(modifiers)>0:
2157                 self.process = cms.Process(self._options.name,*modifiers)
2158             else:
2159                 self.process = cms.Process(self._options.name)
2160
2161
2162
2163
2164     def prepare(self, doChecking = False):
2165         """ Prepare the configuration string and add missing pieces."""
2166
2167         self.loadAndRemember(self.EVTCONTDefaultCFF)  #load the event contents regardless
2168         self.addMaxEvents()
2169         if self.with_input:
2170             self.addSource()
2171         self.addStandardSequences()
2172         ##adding standard sequences might change the inputEventContent option and therefore needs to be finalized after
2173         self.completeInputCommand()
2174         self.addConditions()
2175
2176
2177         outputModuleCfgCode=""
2178         if not 'HARVESTING' in self.stepMap.keys() and not 'ALCAHARVEST' in self.stepMap.keys() and not 'ALCAOUTPUT' in self.stepMap.keys() and self.with_output:
2179             outputModuleCfgCode=self.addOutput()
2180
2181         self.addCommon()
2182
2183         self.pythonCfgCode += "# import of standard configurations\n"
2184         for module in self.imports:
2185             self.pythonCfgCode += ("process.load('"+module+"')\n")
2186
2187         # production info
2188         if not hasattr(self.process,"configurationMetadata"):
2189             self.build_production_info(self._options.evt_type, self._options.number)
2190         else:
2191             #the PSet was added via a load
2192             self.addedObjects.append(("Production Info","configurationMetadata"))
2193
2194         self.pythonCfgCode +="\n"
2195         for comment,object in self.addedObjects:
2196             if comment!="":
2197                 self.pythonCfgCode += "\n# "+comment+"\n"
2198             self.pythonCfgCode += dumpPython(self.process,object)
2199
2200         # dump the output definition
2201         self.pythonCfgCode += "\n# Output definition\n"
2202         self.pythonCfgCode += outputModuleCfgCode
2203
2204         # dump all additional outputs (e.g. alca or skim streams)
2205         self.pythonCfgCode += "\n# Additional output definition\n"
2206         #I do not understand why the keys are not normally ordered.
2207         nl=sorted(self.additionalOutputs.keys())
2208         for name in nl:
2209             output = self.additionalOutputs[name]
2210             self.pythonCfgCode += "process.%s = %s" %(name, output.dumpPython())
2211             tmpOut = cms.EndPath(output)
2212             setattr(self.process,name+'OutPath',tmpOut)
2213             self.schedule.append(tmpOut)
2214
2215         # dump all additional commands
2216         self.pythonCfgCode += "\n# Other statements\n"
2217         for command in self.additionalCommands:
2218             self.pythonCfgCode += command + "\n"
2219
2220         #comma separated list of objects that deserve to be inlined in the configuration (typically from a modified config deep down)
2221         for object in self._options.inlineObjets.split(','):
2222             if not object:
2223                 continue
2224             if not hasattr(self.process,object):
2225                 print('cannot inline -'+object+'- : not known')
2226             else:
2227                 self.pythonCfgCode +='\n'
2228                 self.pythonCfgCode +=dumpPython(self.process,object)
2229
2230         if self._options.pileup=='HiMixEmbGEN':
2231             self.pythonCfgCode += "\nprocess.generator.embeddingMode=cms.bool(True)\n"
2232
2233         # dump all paths
2234         self.pythonCfgCode += "\n# Path and EndPath definitions\n"
2235         for path in self.process.paths:
2236             if getattr(self.process,path) not in self.blacklist_paths:
2237                 self.pythonCfgCode += dumpPython(self.process,path)
2238
2239         for endpath in self.process.endpaths:
2240             if getattr(self.process,endpath) not in self.blacklist_paths:
2241                 self.pythonCfgCode += dumpPython(self.process,endpath)
2242
2243         # dump the schedule
2244         self.pythonCfgCode += "\n# Schedule definition\n"
2245         result = "process.schedule = cms.Schedule("
2246
2247         # handling of the schedule
2248         self.process.schedule = cms.Schedule()
2249         for item in self.schedule:
2250             if not isinstance(item, cms.Schedule):
2251                 self.process.schedule.append(item)
2252             else:
2253                 self.process.schedule.extend(item)
2254
2255         if hasattr(self.process,"HLTSchedule"):
2256             beforeHLT = self.schedule[:self.schedule.index(self.process.HLTSchedule)]
2257             afterHLT = self.schedule[self.schedule.index(self.process.HLTSchedule)+1:]
2258             pathNames = ['process.'+p.label_() for p in beforeHLT]
2259             result += ','.join(pathNames)+')\n'
2260             result += 'process.schedule.extend(process.HLTSchedule)\n'
2261             pathNames = ['process.'+p.label_() for p in afterHLT]
2262             result += 'process.schedule.extend(['+','.join(pathNames)+'])\n'
2263         else:
2264             pathNames = ['process.'+p.label_() for p in self.schedule]
2265             result ='process.schedule = cms.Schedule('+','.join(pathNames)+')\n'
2266
2267         self.pythonCfgCode += result
2268
2269         for labelToAssociate in self.labelsToAssociate:
2270             self.process.schedule.associate(getattr(self.process, labelToAssociate))
2271             self.pythonCfgCode += 'process.schedule.associate(process.' + labelToAssociate + ')\n'
2272
2273         from PhysicsTools.PatAlgos.tools.helpers import associatePatAlgosToolsTask
2274         associatePatAlgosToolsTask(self.process)
2275         self.pythonCfgCode+="from PhysicsTools.PatAlgos.tools.helpers import associatePatAlgosToolsTask\n"
2276         self.pythonCfgCode+="associatePatAlgosToolsTask(process)\n"
2277
2278         if self._options.nThreads != "1":
2279             self.pythonCfgCode +="\n"
2280             self.pythonCfgCode +="#Setup FWK for multithreaded\n"
2281             self.pythonCfgCode +="process.options.numberOfThreads = "+self._options.nThreads+"\n"
2282             self.pythonCfgCode +="process.options.numberOfStreams = "+self._options.nStreams+"\n"
2283             self.pythonCfgCode +="process.options.numberOfConcurrentLuminosityBlocks = "+self._options.nConcurrentLumis+"\n"
2284             self.pythonCfgCode +="process.options.eventSetup.numberOfConcurrentIOVs = "+self._options.nConcurrentIOVs+"\n"
2285             if int(self._options.nConcurrentLumis) > 1:
2286               self.pythonCfgCode +="if hasattr(process, 'DQMStore'): process.DQMStore.assertLegacySafe=cms.untracked.bool(False)\n"
2287             self.process.options.numberOfThreads = int(self._options.nThreads)
2288             self.process.options.numberOfStreams = int(self._options.nStreams)
2289             self.process.options.numberOfConcurrentLuminosityBlocks = int(self._options.nConcurrentLumis)
2290             self.process.options.eventSetup.numberOfConcurrentIOVs = int(self._options.nConcurrentIOVs)
2291         #repacked version
2292         if self._options.isRepacked:
2293             self.pythonCfgCode +="\n"
2294             self.pythonCfgCode +="from Configuration.Applications.ConfigBuilder import MassReplaceInputTag\n"
2295             self.pythonCfgCode +="MassReplaceInputTag(process, new=\"rawDataMapperByLabel\", old=\"rawDataCollector\")\n"
2296             MassReplaceInputTag(self.process, new="rawDataMapperByLabel", old="rawDataCollector")
2297
2298         # special treatment in case of production filter sequence 2/2
2299         if self.productionFilterSequence and not (self._options.pileup=='HiMixEmbGEN'):
2300             self.pythonCfgCode +='# filter all path with the production filter sequence\n'
2301             self.pythonCfgCode +='for path in process.paths:\n'
2302             if len(self.conditionalPaths):
2303                 self.pythonCfgCode +='\tif not path in %s: continue\n'%str(self.conditionalPaths)
2304             if len(self.excludedPaths):
2305                 self.pythonCfgCode +='\tif path in %s: continue\n'%str(self.excludedPaths)
2306             self.pythonCfgCode +='\tgetattr(process,path).insert(0, process.%s)\n'%(self.productionFilterSequence,)
2307             pfs = getattr(self.process,self.productionFilterSequence)
2308             for path in self.process.paths:
2309                 if not path in self.conditionalPaths: continue
2310                 if path in self.excludedPaths: continue
2311                 getattr(self.process,path).insert(0, pfs)
2312
2313
2314         # dump customise fragment
2315         self.pythonCfgCode += self.addCustomise()
2316
2317         if self._options.runUnscheduled:
2318             print("--runUnscheduled is deprecated and not necessary anymore, and will be removed soon. Please update your command line.")
2319         # Keep the "unscheduled customise functions" separate for now,
2320         # there are customize functions given by users (in our unit
2321         # tests) that need to be run before the "unscheduled customise
2322         # functions"
2323         self.pythonCfgCode += self.addCustomise(1)
2324
2325         self.pythonCfgCode += self.addCustomiseCmdLine()
2326
2327         if hasattr(self.process,"logErrorHarvester"):
2328             #configure logErrorHarvester to wait for same EDProducers to finish as the OutputModules
2329             self.pythonCfgCode +="\n#Have logErrorHarvester wait for the same EDProducers to finish as those providing data for the OutputModule\n"
2330             self.pythonCfgCode +="from FWCore.Modules.logErrorHarvester_cff import customiseLogErrorHarvesterUsingOutputCommands\n"
2331             self.pythonCfgCode +="process = customiseLogErrorHarvesterUsingOutputCommands(process)\n"
2332             from FWCore.Modules.logErrorHarvester_cff import customiseLogErrorHarvesterUsingOutputCommands
2333             self.process = customiseLogErrorHarvesterUsingOutputCommands(self.process)
2334
2335         # Temporary hack to put the early delete customization after
2336         # everything else
2337         #
2338         # FIXME: remove when no longer needed
2339         self.pythonCfgCode += "\n# Add early deletion of temporary data products to reduce peak memory need\n"
2340         self.pythonCfgCode += "from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete\n"
2341         self.pythonCfgCode += "process = customiseEarlyDelete(process)\n"
2342         self.pythonCfgCode += "# End adding early deletion\n"
2343         from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete
2344         self.process = customiseEarlyDelete(self.process)
2345
2346         imports = cms.specialImportRegistry.getSpecialImports()
2347         if len(imports) > 0:
2348             #need to inject this at the top
2349             index = self.pythonCfgCode.find("import FWCore.ParameterSet.Config")
2350             #now find the end of line
2351             index = self.pythonCfgCode.find("\n",index)
2352             self.pythonCfgCode = self.pythonCfgCode[:index]+ "\n" + "\n".join(imports)+"\n" +self.pythonCfgCode[index:]
2353
2354
2355         # make the .io file
2356
2357         if self._options.io:
2358             #io=open(self._options.python_filename.replace('.py','.io'),'w')
2359             if not self._options.io.endswith('.io'): self._option.io+='.io'
2360             io=open(self._options.io,'w')
2361             ioJson={}
2362             if hasattr(self.process.source,"fileNames"):
2363                 if len(self.process.source.fileNames.value()):
2364                     ioJson['primary']=self.process.source.fileNames.value()
2365             if hasattr(self.process.source,"secondaryFileNames"):
2366                 if len(self.process.source.secondaryFileNames.value()):
2367                     ioJson['secondary']=self.process.source.secondaryFileNames.value()
2368             if self._options.pileup_input and (self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:')):
2369                 ioJson['pileup']=self._options.pileup_input[4:]
2370             for (o,om) in self.process.outputModules_().items():
2371                 ioJson[o]=om.fileName.value()
2372             ioJson['GT']=self.process.GlobalTag.globaltag.value()
2373             if self.productionFilterSequence:
2374                 ioJson['filter']=self.productionFilterSequence
2375             import json
2376             io.write(json.dumps(ioJson))
2377         return
2378