Applications/python/ConfigBuilder.py

0001 #! /usr/bin/env python3
0002
0003 from __future__ import print_function
0004 __version__ = "$Revision: 1.19 $"
0005 __source__ = "$Source: /local/reps/CMSSW/CMSSW/Configuration/Applications/python/ConfigBuilder.py,v $"
0006
0007 import FWCore.ParameterSet.Config as cms
0008 from FWCore.ParameterSet.Modules import _Module
0009 # The following import is provided for backward compatibility reasons.
0010 # The function used to be defined in this file.
0011 from FWCore.ParameterSet.MassReplace import massReplaceInputTag as MassReplaceInputTag
0012
0013 import hashlib
0014 import sys
0015 import re
0016 import collections
0017 from subprocess import Popen,PIPE
0018 import FWCore.ParameterSet.DictTypes as DictTypes
0019 class Options:
0020     pass
0021
0022 # the canonical defaults
0023 defaultOptions = Options()
0024 defaultOptions.datamix = 'DataOnSim'
0025 defaultOptions.isMC=False
0026 defaultOptions.isData=True
0027 defaultOptions.step=''
0028 defaultOptions.pileup='NoPileUp'
0029 defaultOptions.pileup_input = None
0030 defaultOptions.pileup_dasoption = ''
0031 defaultOptions.geometry = 'SimDB'
0032 defaultOptions.geometryExtendedOptions = ['ExtendedGFlash','Extended','NoCastor']
0033 defaultOptions.magField = ''
0034 defaultOptions.conditions = None
0035 defaultOptions.scenarioOptions=['pp','cosmics','nocoll','HeavyIons']
0036 defaultOptions.harvesting= 'AtRunEnd'
0037 defaultOptions.gflash = False
0038 defaultOptions.number = -1
0039 defaultOptions.number_out = None
0040 defaultOptions.arguments = ""
0041 defaultOptions.name = "NO NAME GIVEN"
0042 defaultOptions.evt_type = ""
0043 defaultOptions.filein = ""
0044 defaultOptions.dasquery=""
0045 defaultOptions.dasoption=""
0046 defaultOptions.secondfilein = ""
0047 defaultOptions.customisation_file = []
0048 defaultOptions.customisation_file_unsch = []
0049 defaultOptions.customise_commands = ""
0050 defaultOptions.inline_custom=False
0051 defaultOptions.particleTable = 'pythiapdt'
0052 defaultOptions.particleTableList = ['pythiapdt','pdt']
0053 defaultOptions.dirin = ''
0054 defaultOptions.dirout = ''
0055 defaultOptions.filetype = 'EDM'
0056 defaultOptions.fileout = 'output.root'
0057 defaultOptions.filtername = ''
0058 defaultOptions.lazy_download = False
0059 defaultOptions.custom_conditions = ''
0060 defaultOptions.hltProcess = ''
0061 defaultOptions.eventcontent = None
0062 defaultOptions.datatier = None
0063 defaultOptions.inlineEventContent = True
0064 defaultOptions.inlineObjets =''
0065 defaultOptions.hideGen=False
0066 from Configuration.StandardSequences.VtxSmeared import VtxSmearedDefaultKey,VtxSmearedHIDefaultKey
0067 defaultOptions.beamspot=None
0068 defaultOptions.outputDefinition =''
0069 defaultOptions.inputCommands = None
0070 defaultOptions.outputCommands = None
0071 defaultOptions.inputEventContent = ''
0072 defaultOptions.dropDescendant = False
0073 defaultOptions.relval = None
0074 defaultOptions.profile = None
0075 defaultOptions.heap_profile = None
0076 defaultOptions.isRepacked = False
0077 defaultOptions.restoreRNDSeeds = False
0078 defaultOptions.donotDropOnInput = ''
0079 defaultOptions.python_filename =''
0080 defaultOptions.io=None
0081 defaultOptions.lumiToProcess=None
0082 defaultOptions.fast=False
0083 defaultOptions.runsAndWeightsForMC = None
0084 defaultOptions.runsScenarioForMC = None
0085 defaultOptions.runsAndWeightsForMCIntegerWeights = None
0086 defaultOptions.runsScenarioForMCIntegerWeights = None
0087 defaultOptions.runUnscheduled = False
0088 defaultOptions.timeoutOutput = False
0089 defaultOptions.nThreads = '1'
0090 defaultOptions.nStreams = '0'
0091 defaultOptions.nConcurrentLumis = '0'
0092 defaultOptions.nConcurrentIOVs = '0'
0093 defaultOptions.accelerators = None
0094
0095 # some helper routines
0096 def dumpPython(process,name):
0097     theObject = getattr(process,name)
0098     if isinstance(theObject,cms.Path) or isinstance(theObject,cms.EndPath) or isinstance(theObject,cms.Sequence):
0099         return "process."+name+" = " + theObject.dumpPython()
0100     elif isinstance(theObject,_Module) or isinstance(theObject,cms.ESProducer):
0101         return "process."+name+" = " + theObject.dumpPython()+"\n"
0102     else:
0103         return "process."+name+" = " + theObject.dumpPython()+"\n"
0104 def filesFromList(fileName,s=None):
0105     import os
0106     import FWCore.ParameterSet.Config as cms
0107     prim=[]
0108     sec=[]
0109     for line in open(fileName,'r'):
0110         if line.count(".root")>=2:
0111             #two files solution...
0112             entries=line.replace("\n","").split()
0113             prim.append(entries[0])
0114             sec.append(entries[1])
0115         elif (line.find(".root")!=-1):
0116             entry=line.replace("\n","")
0117             prim.append(entry)
0118     # remove any duplicates but keep the order
0119     file_seen = set()
0120     prim = [f for f in prim if not (f in file_seen or file_seen.add(f))]
0121     file_seen = set()
0122     sec = [f for f in sec if not (f in file_seen or file_seen.add(f))]
0123     if s:
0124         if not hasattr(s,"fileNames"):
0125             s.fileNames=cms.untracked.vstring(prim)
0126         else:
0127             s.fileNames.extend(prim)
0128         if len(sec)!=0:
0129             if not hasattr(s,"secondaryFileNames"):
0130                 s.secondaryFileNames=cms.untracked.vstring(sec)
0131             else:
0132                 s.secondaryFileNames.extend(sec)
0133     print("found files: ",prim)
0134     if len(prim)==0:
0135         raise Exception("There are not files in input from the file list")
0136     if len(sec)!=0:
0137         print("found parent files:",sec)
0138     return (prim,sec)
0139
0140 def filesFromDASQuery(query,option="",s=None):
0141     import os,time
0142     import FWCore.ParameterSet.Config as cms
0143     prim=[]
0144     sec=[]
0145     print("the query is",query)
0146     eC=5
0147     count=0
0148     while eC!=0 and count<3:
0149         if count!=0:
0150             print('Sleeping, then retrying DAS')
0151             time.sleep(100)
0152         p = Popen('dasgoclient %s --query "%s"'%(option,query), stdout=PIPE,shell=True, universal_newlines=True)
0153         pipe=p.stdout.read()
0154         tupleP = os.waitpid(p.pid, 0)
0155         eC=tupleP[1]
0156         count=count+1
0157     if eC==0:
0158         print("DAS succeeded after",count,"attempts",eC)
0159     else:
0160         print("DAS failed 3 times- I give up")
0161     for line in pipe.split('\n'):
0162         if line.count(".root")>=2:
0163             #two files solution...
0164             entries=line.replace("\n","").split()
0165             prim.append(entries[0])
0166             sec.append(entries[1])
0167         elif (line.find(".root")!=-1):
0168             entry=line.replace("\n","")
0169             prim.append(entry)
0170     # remove any duplicates
0171     prim = sorted(list(set(prim)))
0172     sec = sorted(list(set(sec)))
0173     if s:
0174         if not hasattr(s,"fileNames"):
0175             s.fileNames=cms.untracked.vstring(prim)
0176         else:
0177             s.fileNames.extend(prim)
0178         if len(sec)!=0:
0179             if not hasattr(s,"secondaryFileNames"):
0180                 s.secondaryFileNames=cms.untracked.vstring(sec)
0181             else:
0182                 s.secondaryFileNames.extend(sec)
0183     print("found files: ",prim)
0184     if len(sec)!=0:
0185         print("found parent files:",sec)
0186     return (prim,sec)
0187
0188 def anyOf(listOfKeys,dict,opt=None):
0189     for k in listOfKeys:
0190         if k in dict:
0191             toReturn=dict[k]
0192             dict.pop(k)
0193             return toReturn
0194     if opt!=None:
0195         return opt
0196     else:
0197         raise Exception("any of "+','.join(listOfKeys)+" are mandatory entries of --output options")
0198
0199 class ConfigBuilder(object):
0200     """The main building routines """
0201
0202     def __init__(self, options, process = None, with_output = False, with_input = False ):
0203         """options taken from old cmsDriver and optparse """
0204
0205         options.outfile_name = options.dirout+options.fileout
0206
0207         self._options = options
0208
0209         if self._options.isData and options.isMC:
0210             raise Exception("ERROR: You may specify only --data or --mc, not both")
0211         #if not self._options.conditions:
0212         #        raise Exception("ERROR: No conditions given!\nPlease specify conditions. E.g. via --conditions=IDEAL_30X::All")
0213
0214         # check that MEtoEDMConverter (running in ENDJOB) and DQMIO don't run in the same job
0215         if 'ENDJOB' in self._options.step:
0216             if  (hasattr(self._options,"outputDefinition") and \
0217                 self._options.outputDefinition != '' and \
0218                 any(anyOf(['t','tier','dataTier'],outdic) == 'DQMIO' for outdic in eval(self._options.outputDefinition))) or \
0219                 (hasattr(self._options,"datatier") and \
0220                 self._options.datatier and \
0221                 'DQMIO' in self._options.datatier):
0222                 print("removing ENDJOB from steps since not compatible with DQMIO dataTier")
0223                 self._options.step=self._options.step.replace(',ENDJOB','')
0224
0225
0226
0227         # what steps are provided by this class?
0228         stepList = [re.sub(r'^prepare_', '', methodName) for methodName in ConfigBuilder.__dict__ if methodName.startswith('prepare_')]
0229         self.stepMap={}
0230         self.stepKeys=[]
0231         for step in self._options.step.split(","):
0232             if step=='': continue
0233             stepParts = step.split(":")
0234             stepName = stepParts[0]
0235             if stepName not in stepList and not stepName.startswith('re'):
0236                 raise ValueError("Step {} unknown. Available are {}".format( stepName , sorted(stepList)))
0237             if len(stepParts)==1:
0238                 self.stepMap[stepName]=""
0239             elif len(stepParts)==2:
0240                 self.stepMap[stepName]=stepParts[1].split('+')
0241             elif len(stepParts)==3:
0242                 self.stepMap[stepName]=(stepParts[2].split('+'),stepParts[1])
0243             else:
0244                 raise ValueError(f"Step definition {step} invalid")
0245             self.stepKeys.append(stepName)
0246
0247         #print(f"map of steps is: {self.stepMap}")
0248
0249         self.with_output = with_output
0250         self.process=process
0251
0252         if hasattr(self._options,"no_output_flag") and self._options.no_output_flag:
0253             self.with_output = False
0254         self.with_input = with_input
0255         self.imports = []
0256         self.create_process()
0257         self.define_Configs()
0258         self.schedule = list()
0259         self.scheduleIndexOfFirstHLTPath = None
0260
0261         # we are doing three things here:
0262         # creating a process to catch errors
0263         # building the code to re-create the process
0264
0265         self.additionalCommands = []
0266         # TODO: maybe a list of to be dumped objects would help as well
0267         self.blacklist_paths = []
0268         self.addedObjects = []
0269         self.additionalOutputs = {}
0270
0271         self.productionFilterSequence = None
0272         self.labelsToAssociate=[]
0273         self.nextScheduleIsConditional=False
0274         self.conditionalPaths=[]
0275         self.excludedPaths=[]
0276
0277     def profileOptions(self):
0278         """
0279         addIgProfService
0280         Function to add the igprof profile service so that you can dump in the middle
0281         of the run.
0282         """
0283         profileOpts = self._options.profile.split(':')
0284         profilerStart = 1
0285         profilerInterval = 100
0286         profilerFormat = None
0287         profilerJobFormat = None
0288
0289         if len(profileOpts):
0290             #type, given as first argument is unused here
0291             profileOpts.pop(0)
0292         if len(profileOpts):
0293             startEvent = profileOpts.pop(0)
0294             if not startEvent.isdigit():
0295                 raise Exception("%s is not a number" % startEvent)
0296             profilerStart = int(startEvent)
0297         if len(profileOpts):
0298             eventInterval = profileOpts.pop(0)
0299             if not eventInterval.isdigit():
0300                 raise Exception("%s is not a number" % eventInterval)
0301             profilerInterval = int(eventInterval)
0302         if len(profileOpts):
0303             profilerFormat = profileOpts.pop(0)
0304
0305
0306         if not profilerFormat:
0307             profilerFormat = "%s___%s___%%I.gz" % (
0308                 self._options.evt_type.replace("_cfi", ""),
0309                 hashlib.md5(
0310                     (str(self._options.step) + str(self._options.pileup) + str(self._options.conditions) +
0311                     str(self._options.datatier) + str(self._options.profileTypeLabel)).encode('utf-8')
0312                 ).hexdigest()
0313             )
0314         if not profilerJobFormat and profilerFormat.endswith(".gz"):
0315             profilerJobFormat = profilerFormat.replace(".gz", "_EndOfJob.gz")
0316         elif not profilerJobFormat:
0317             profilerJobFormat = profilerFormat + "_EndOfJob.gz"
0318
0319         return (profilerStart,profilerInterval,profilerFormat,profilerJobFormat)
0320
0321     def heapProfileOptions(self):
0322         """
0323         addJeProfService
0324         Function to add the jemalloc heap  profile service so that you can dump in the middle
0325         of the run.
0326         """
0327         profileOpts = self._options.profile.split(':')
0328         profilerStart = 1
0329         profilerInterval = 100
0330         profilerFormat = None
0331         profilerJobFormat = None
0332
0333         if len(profileOpts):
0334             #type, given as first argument is unused here
0335             profileOpts.pop(0)
0336         if len(profileOpts):
0337             startEvent = profileOpts.pop(0)
0338             if not startEvent.isdigit():
0339                 raise Exception("%s is not a number" % startEvent)
0340             profilerStart = int(startEvent)
0341         if len(profileOpts):
0342             eventInterval = profileOpts.pop(0)
0343             if not eventInterval.isdigit():
0344                 raise Exception("%s is not a number" % eventInterval)
0345             profilerInterval = int(eventInterval)
0346         if len(profileOpts):
0347             profilerFormat = profileOpts.pop(0)
0348
0349
0350         if not profilerFormat:
0351             profilerFormat = "%s___%s___%%I.heap" % (
0352                 self._options.evt_type.replace("_cfi", ""),
0353                 hashlib.md5(
0354                     (str(self._options.step) + str(self._options.pileup) + str(self._options.conditions) +
0355                     str(self._options.datatier) + str(self._options.profileTypeLabel)).encode('utf-8')
0356                 ).hexdigest()
0357             )
0358         if not profilerJobFormat and profilerFormat.endswith(".heap"):
0359             profilerJobFormat = profilerFormat.replace(".heap", "_EndOfJob.heap")
0360         elif not profilerJobFormat:
0361             profilerJobFormat = profilerFormat + "_EndOfJob.heap"
0362
0363         return (profilerStart,profilerInterval,profilerFormat,profilerJobFormat)
0364
0365     def load(self,includeFile):
0366         includeFile = includeFile.replace('/','.')
0367         self.process.load(includeFile)
0368         return sys.modules[includeFile]
0369
0370     def loadAndRemember(self, includeFile):
0371         """helper routine to load am memorize imports"""
0372         # we could make the imports a on-the-fly data method of the process instance itself
0373         # not sure if the latter is a good idea
0374         includeFile = includeFile.replace('/','.')
0375         self.imports.append(includeFile)
0376         self.process.load(includeFile)
0377         return sys.modules[includeFile]
0378
0379     def executeAndRemember(self, command):
0380         """helper routine to remember replace statements"""
0381         self.additionalCommands.append(command)
0382         if not command.strip().startswith("#"):
0383         # substitute: process.foo = process.bar -> self.process.foo = self.process.bar
0384             import re
0385             exec(re.sub(r"([^a-zA-Z_0-9]|^)(process)([^a-zA-Z_0-9])",r"\1self.process\3",command))
0386             #exec(command.replace("process.","self.process."))
0387
0388     def addCommon(self):
0389         if 'HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys():
0390             self.process.options.Rethrow = ['ProductNotFound']
0391             self.process.options.fileMode = 'FULLMERGE'
0392
0393         self.addedObjects.append(("","options"))
0394
0395         if self._options.lazy_download:
0396             self.process.AdaptorConfig = cms.Service("AdaptorConfig",
0397                                                      stats = cms.untracked.bool(True),
0398                                                      enable = cms.untracked.bool(True),
0399                                                      cacheHint = cms.untracked.string("lazy-download"),
0400                                                      readHint = cms.untracked.string("read-ahead-buffered")
0401                                                      )
0402             self.addedObjects.append(("Setup lazy download","AdaptorConfig"))
0403
0404         #self.process.cmsDriverCommand = cms.untracked.PSet( command=cms.untracked.string('cmsDriver.py '+self._options.arguments) )
0405         #self.addedObjects.append(("what cmsDriver command was used","cmsDriverCommand"))
0406
0407         if self._options.profile:
0408             (start, interval, eventFormat, jobFormat)=self.profileOptions()
0409             self.process.IgProfService = cms.Service("IgProfService",
0410                                                      reportFirstEvent            = cms.untracked.int32(start),
0411                                                      reportEventInterval         = cms.untracked.int32(interval),
0412                                                      reportToFileAtPostEvent     = cms.untracked.string("| gzip -c > %s"%(eventFormat)),
0413                                                      reportToFileAtPostEndJob    = cms.untracked.string("| gzip -c > %s"%(jobFormat)))
0414             self.addedObjects.append(("Setup IGProf Service for profiling","IgProfService"))
0415
0416         if self._options.heap_profile:
0417             (start, interval, eventFormat, jobFormat)=self.heapProfileOptions()
0418             self.process.JeProfService = cms.Service("JeProfService",
0419                                                      reportFirstEvent            = cms.untracked.int32(start),
0420                                                      reportEventInterval         = cms.untracked.int32(interval),
0421                                                      reportToFileAtPostEvent     = cms.untracked.string("%s"%(eventFormat)),
0422                                                      reportToFileAtPostEndJob    = cms.untracked.string("%s"%(jobFormat)))
0423             self.addedObjects.append(("Setup JeProf Service for heap profiling","JeProfService"))
0424
0425     def addMaxEvents(self):
0426         """Here we decide how many evts will be processed"""
0427         self.process.maxEvents.input = int(self._options.number)
0428         if self._options.number_out:
0429             self.process.maxEvents.output = int(self._options.number_out)
0430         self.addedObjects.append(("","maxEvents"))
0431
0432     def addSource(self):
0433         """Here the source is built. Priority: file, generator"""
0434         self.addedObjects.append(("Input source","source"))
0435
0436         def filesFromOption(self):
0437             for entry in self._options.filein.split(','):
0438                 print("entry",entry)
0439                 if entry.startswith("filelist:"):
0440                     filesFromList(entry[9:],self.process.source)
0441                 elif entry.startswith("dbs:") or entry.startswith("das:"):
0442                     filesFromDASQuery('file dataset = %s'%(entry[4:]),self._options.dasoption,self.process.source)
0443                 else:
0444                     self.process.source.fileNames.append(self._options.dirin+entry)
0445             if self._options.secondfilein:
0446                 if not hasattr(self.process.source,"secondaryFileNames"):
0447                     raise Exception("--secondfilein not compatible with "+self._options.filetype+"input type")
0448                 for entry in self._options.secondfilein.split(','):
0449                     print("entry",entry)
0450                     if entry.startswith("filelist:"):
0451                         self.process.source.secondaryFileNames.extend((filesFromList(entry[9:]))[0])
0452                     elif entry.startswith("dbs:") or entry.startswith("das:"):
0453                         self.process.source.secondaryFileNames.extend((filesFromDASQuery('file dataset = %s'%(entry[4:]),self._options.dasoption))[0])
0454                     else:
0455                         self.process.source.secondaryFileNames.append(self._options.dirin+entry)
0456
0457         if self._options.filein or self._options.dasquery:
0458             if self._options.filetype == "EDM":
0459                 self.process.source=cms.Source("PoolSource",
0460                                                fileNames = cms.untracked.vstring(),
0461                                                secondaryFileNames= cms.untracked.vstring())
0462                 filesFromOption(self)
0463             elif self._options.filetype == "DAT":
0464                 self.process.source=cms.Source("NewEventStreamFileReader",fileNames = cms.untracked.vstring())
0465                 filesFromOption(self)
0466             elif self._options.filetype == "LHE":
0467                 self.process.source=cms.Source("LHESource", fileNames = cms.untracked.vstring())
0468                 if self._options.filein.startswith("lhe:"):
0469                     #list the article directory automatically
0470                     args=self._options.filein.split(':')
0471                     article=args[1]
0472                     print('LHE input from article ',article)
0473                     location='/store/lhe/'
0474                     import os
0475                     textOfFiles=os.popen('cmsLHEtoEOSManager.py -l '+article)
0476                     for line in textOfFiles:
0477                         for fileName in [x for x in line.split() if '.lhe' in x]:
0478                             self.process.source.fileNames.append(location+article+'/'+fileName)
0479                     #check first if list of LHE files is loaded (not empty)
0480                     if len(line)<2:
0481                         print('Issue to load LHE files, please check and try again.')
0482                         sys.exit(-1)
0483                     #Additional check to protect empty fileNames in process.source
0484                     if len(self.process.source.fileNames)==0:
0485                         print('Issue with empty filename, but can pass line check')
0486                         sys.exit(-1)
0487                     if len(args)>2:
0488                         self.process.source.skipEvents = cms.untracked.uint32(int(args[2]))
0489                 else:
0490                     filesFromOption(self)
0491
0492             elif self._options.filetype == "DQM":
0493                 self.process.source=cms.Source("DQMRootSource",
0494                                                fileNames = cms.untracked.vstring())
0495                 filesFromOption(self)
0496
0497             elif self._options.filetype == "DQMDAQ":
0498                 # FIXME: how to configure it if there are no input files specified?
0499                 self.process.source=cms.Source("DQMStreamerReader")
0500
0501
0502             if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"):
0503                 self.process.source.processingMode = cms.untracked.string("RunsAndLumis")
0504
0505         if self._options.dasquery!='':
0506             self.process.source=cms.Source("PoolSource", fileNames = cms.untracked.vstring(),secondaryFileNames = cms.untracked.vstring())
0507             filesFromDASQuery(self._options.dasquery,self._options.dasoption,self.process.source)
0508
0509             if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"):
0510                 self.process.source.processingMode = cms.untracked.string("RunsAndLumis")
0511
0512         ##drop LHEXMLStringProduct on input to save memory if appropriate
0513         if 'GEN' in self.stepMap.keys() and not self._options.filetype == "LHE":
0514             if self._options.inputCommands:
0515                 self._options.inputCommands+=',drop LHEXMLStringProduct_*_*_*,'
0516             else:
0517                 self._options.inputCommands='keep *, drop LHEXMLStringProduct_*_*_*,'
0518
0519         if self.process.source and self._options.inputCommands and not self._options.filetype == "LHE":
0520             if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring()
0521             for command in self._options.inputCommands.split(','):
0522                 # remove whitespace around the keep/drop statements
0523                 command = command.strip()
0524                 if command=='': continue
0525                 self.process.source.inputCommands.append(command)
0526             if not self._options.dropDescendant:
0527                 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False)
0528
0529         if self._options.lumiToProcess:
0530             import FWCore.PythonUtilities.LumiList as LumiList
0531             self.process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange( LumiList.LumiList(self._options.lumiToProcess).getCMSSWString().split(',') )
0532
0533         if 'GEN' in self.stepMap.keys() or 'LHE' in self.stepMap or (not self._options.filein and hasattr(self._options, "evt_type")):
0534             if self.process.source is None:
0535                 self.process.source=cms.Source("EmptySource")
0536
0537         # modify source in case of run-dependent MC
0538         self.runsAndWeights=None
0539         if self._options.runsAndWeightsForMC or self._options.runsScenarioForMC :
0540             if not self._options.isMC :
0541                 raise Exception("options --runsAndWeightsForMC and --runsScenarioForMC are only valid for MC")
0542             if self._options.runsAndWeightsForMC:
0543                 self.runsAndWeights = eval(self._options.runsAndWeightsForMC)
0544             else:
0545                 from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights
0546                 if isinstance(RunsAndWeights[self._options.runsScenarioForMC], str):
0547                     __import__(RunsAndWeights[self._options.runsScenarioForMC])
0548                     self.runsAndWeights = sys.modules[RunsAndWeights[self._options.runsScenarioForMC]].runProbabilityDistribution
0549                 else:
0550                     self.runsAndWeights = RunsAndWeights[self._options.runsScenarioForMC]
0551
0552         if self.runsAndWeights:
0553             import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun
0554             ThrowAndSetRandomRun.throwAndSetRandomRun(self.process.source,self.runsAndWeights)
0555             self.additionalCommands.append('import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun')
0556             self.additionalCommands.append('ThrowAndSetRandomRun.throwAndSetRandomRun(process.source,%s)'%(self.runsAndWeights))
0557
0558         # modify source in case of run-dependent MC (Run-3 method)
0559         self.runsAndWeightsInt=None
0560         if self._options.runsAndWeightsForMCIntegerWeights or self._options.runsScenarioForMCIntegerWeights:
0561             if not self._options.isMC :
0562                 raise Exception("options --runsAndWeightsForMCIntegerWeights and --runsScenarioForMCIntegerWeights are only valid for MC")
0563             if self._options.runsAndWeightsForMCIntegerWeights:
0564                 self.runsAndWeightsInt = eval(self._options.runsAndWeightsForMCIntegerWeights)
0565             else:
0566                 from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights
0567                 if isinstance(RunsAndWeights[self._options.runsScenarioForMCIntegerWeights], str):
0568                     __import__(RunsAndWeights[self._options.runsScenarioForMCIntegerWeights])
0569                     self.runsAndWeightsInt = sys.modules[RunsAndWeights[self._options.runsScenarioForMCIntegerWeights]].runProbabilityDistribution
0570                 else:
0571                     self.runsAndWeightsInt = RunsAndWeights[self._options.runsScenarioForMCIntegerWeights]
0572
0573         if self.runsAndWeightsInt:
0574             if not self._options.relval:
0575                 raise Exception("--relval option required when using --runsAndWeightsInt")
0576             if 'DATAMIX' in self._options.step:
0577                 from SimGeneral.Configuration.LumiToRun import lumi_to_run
0578                 total_events, events_per_job  = self._options.relval.split(',')
0579                 lumi_to_run_mapping = lumi_to_run(self.runsAndWeightsInt, int(total_events), int(events_per_job))
0580                 self.additionalCommands.append("process.source.firstLuminosityBlockForEachRun = cms.untracked.VLuminosityBlockID(*[cms.LuminosityBlockID(x,y) for x,y in " + str(lumi_to_run_mapping) + "])")
0581
0582         return
0583
0584     def addOutput(self):
0585         """ Add output module to the process """
0586         result=""
0587         if self._options.outputDefinition:
0588             if self._options.datatier:
0589                 print("--datatier & --eventcontent options ignored")
0590
0591             #new output convention with a list of dict
0592             outList = eval(self._options.outputDefinition)
0593             for (id,outDefDict) in enumerate(outList):
0594                 outDefDictStr=outDefDict.__str__()
0595                 if not isinstance(outDefDict,dict):
0596                     raise Exception("--output needs to be passed a list of dict"+self._options.outputDefinition+" is invalid")
0597                 #requires option: tier
0598                 theTier=anyOf(['t','tier','dataTier'],outDefDict)
0599                 #optional option: eventcontent, filtername, selectEvents, moduleLabel, filename
0600                 ## event content
0601                 theStreamType=anyOf(['e','ec','eventContent','streamType'],outDefDict,theTier)
0602                 theFilterName=anyOf(['f','ftN','filterName'],outDefDict,'')
0603                 theSelectEvent=anyOf(['s','sE','selectEvents'],outDefDict,'')
0604                 theModuleLabel=anyOf(['l','mL','moduleLabel'],outDefDict,'')
0605                 theExtraOutputCommands=anyOf(['o','oC','outputCommands'],outDefDict,'')
0606                 # module label has a particular role
0607                 if not theModuleLabel:
0608                     tryNames=[theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+'output',
0609                               theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+'output',
0610                               theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+theSelectEvent.split(',')[0].replace(':','for').replace(' ','')+'output'
0611                               ]
0612                     for name in tryNames:
0613                         if not hasattr(self.process,name):
0614                             theModuleLabel=name
0615                             break
0616                 if not theModuleLabel:
0617                     raise Exception("cannot find a module label for specification: "+outDefDictStr)
0618                 if id==0:
0619                     defaultFileName=self._options.outfile_name
0620                 else:
0621                     defaultFileName=self._options.outfile_name.replace('.root','_in'+theTier+'.root')
0622
0623                 theFileName=self._options.dirout+anyOf(['fn','fileName'],outDefDict,defaultFileName)
0624                 if not theFileName.endswith('.root'):
0625                     theFileName+='.root'
0626
0627                 if len(outDefDict):
0628                     raise Exception("unused keys from --output options: "+','.join(outDefDict.keys()))
0629                 if theStreamType=='DQMIO': theStreamType='DQM'
0630                 if theStreamType=='ALL':
0631                     theEventContent = cms.PSet(outputCommands = cms.untracked.vstring('keep *'))
0632                 else:
0633                     theEventContent = getattr(self.process, theStreamType+"EventContent")
0634
0635
0636                 addAlCaSelects=False
0637                 if theStreamType=='ALCARECO' and not theFilterName:
0638                     theFilterName='StreamALCACombined'
0639                     addAlCaSelects=True
0640
0641                 CppType='PoolOutputModule'
0642                 if self._options.timeoutOutput:
0643                     CppType='TimeoutPoolOutputModule'
0644                 if theStreamType=='DQM' and theTier=='DQMIO': CppType='DQMRootOutputModule'
0645                 output = cms.OutputModule(CppType,
0646                                           theEventContent.clone(),
0647                                           fileName = cms.untracked.string(theFileName),
0648                                           dataset = cms.untracked.PSet(
0649                                              dataTier = cms.untracked.string(theTier),
0650                                              filterName = cms.untracked.string(theFilterName))
0651                                           )
0652                 if not theSelectEvent and hasattr(self.process,'generation_step') and theStreamType!='LHE':
0653                     output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step'))
0654                 if not theSelectEvent and hasattr(self.process,'filtering_step'):
0655                     output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step'))
0656                 if theSelectEvent:
0657                     output.SelectEvents =cms.untracked.PSet(SelectEvents = cms.vstring(theSelectEvent))
0658
0659                 if addAlCaSelects:
0660                     if not hasattr(output,'SelectEvents'):
0661                         output.SelectEvents=cms.untracked.PSet(SelectEvents=cms.vstring())
0662                     for alca in self.AlCaPaths:
0663                         output.SelectEvents.SelectEvents.extend(getattr(self.process,'OutALCARECO'+alca).SelectEvents.SelectEvents)
0664
0665
0666                 if hasattr(self.process,theModuleLabel):
0667                     raise Exception("the current process already has a module "+theModuleLabel+" defined")
0668                 #print "creating output module ",theModuleLabel
0669                 setattr(self.process,theModuleLabel,output)
0670                 outputModule=getattr(self.process,theModuleLabel)
0671                 setattr(self.process,theModuleLabel+'_step',cms.EndPath(outputModule))
0672                 path=getattr(self.process,theModuleLabel+'_step')
0673                 self.schedule.append(path)
0674
0675                 if not self._options.inlineEventContent and hasattr(self.process,theStreamType+"EventContent"):
0676                     def doNotInlineEventContent(instance,label = "cms.untracked.vstring(process."+theStreamType+"EventContent.outputCommands)"):
0677                         return label
0678                     outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
0679                 if theExtraOutputCommands:
0680                     if not isinstance(theExtraOutputCommands,list):
0681                         raise Exception("extra ouput command in --option must be a list of strings")
0682                     if hasattr(self.process,theStreamType+"EventContent"):
0683                         self.executeAndRemember('process.%s.outputCommands.extend(%s)'%(theModuleLabel,theExtraOutputCommands))
0684                     else:
0685                         outputModule.outputCommands.extend(theExtraOutputCommands)
0686
0687                 result+="\nprocess."+theModuleLabel+" = "+outputModule.dumpPython()
0688
0689             ##ends the --output options model
0690             return result
0691
0692         streamTypes=self._options.eventcontent.split(',')
0693         tiers=self._options.datatier.split(',')
0694         if not self._options.outputDefinition and len(streamTypes)!=len(tiers):
0695             raise Exception("number of event content arguments does not match number of datatier arguments")
0696
0697         # if the only step is alca we don't need to put in an output
0698         if self._options.step.split(',')[0].split(':')[0] == 'ALCA':
0699             return "\n"
0700
0701         for i,(streamType,tier) in enumerate(zip(streamTypes,tiers)):
0702             if streamType=='': continue
0703             if streamType == 'ALCARECO' and not 'ALCAPRODUCER' in self._options.step: continue
0704             if streamType=='DQMIO': streamType='DQM'
0705             eventContent=streamType
0706             ## override streamType to eventContent in case NANOEDM
0707             if streamType == "NANOEDMAOD" :
0708                 eventContent = "NANOAOD"
0709             elif streamType == "NANOEDMAODSIM" :
0710                 eventContent = "NANOAODSIM"
0711             theEventContent = getattr(self.process, eventContent+"EventContent")
0712             if i==0:
0713                 theFileName=self._options.outfile_name
0714                 theFilterName=self._options.filtername
0715             else:
0716                 theFileName=self._options.outfile_name.replace('.root','_in'+streamType+'.root')
0717                 theFilterName=self._options.filtername
0718             CppType='PoolOutputModule'
0719             if self._options.timeoutOutput:
0720                 CppType='TimeoutPoolOutputModule'
0721             if streamType=='DQM' and tier=='DQMIO': CppType='DQMRootOutputModule'
0722             if "NANOAOD" in streamType : CppType='NanoAODOutputModule'
0723             output = cms.OutputModule(CppType,
0724                                       theEventContent,
0725                                       fileName = cms.untracked.string(theFileName),
0726                                       dataset = cms.untracked.PSet(dataTier = cms.untracked.string(tier),
0727                                                                    filterName = cms.untracked.string(theFilterName)
0728                                                                    )
0729                                       )
0730             if hasattr(self.process,"generation_step") and streamType!='LHE':
0731                 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step'))
0732             if hasattr(self.process,"filtering_step"):
0733                 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step'))
0734
0735             if streamType=='ALCARECO':
0736                 output.dataset.filterName = cms.untracked.string('StreamALCACombined')
0737
0738             if "MINIAOD" in streamType:
0739                 from PhysicsTools.PatAlgos.slimming.miniAOD_tools import miniAOD_customizeOutput
0740                 miniAOD_customizeOutput(output)
0741
0742             outputModuleName=streamType+'output'
0743             setattr(self.process,outputModuleName,output)
0744             outputModule=getattr(self.process,outputModuleName)
0745             setattr(self.process,outputModuleName+'_step',cms.EndPath(outputModule))
0746             path=getattr(self.process,outputModuleName+'_step')
0747             self.schedule.append(path)
0748
0749             if self._options.outputCommands and streamType!='DQM':
0750                 for evct in self._options.outputCommands.split(','):
0751                     if not evct: continue
0752                     self.executeAndRemember("process.%s.outputCommands.append('%s')"%(outputModuleName,evct.strip()))
0753
0754             if not self._options.inlineEventContent:
0755                 tmpstreamType=streamType
0756                 if "NANOEDM" in tmpstreamType :
0757                     tmpstreamType=tmpstreamType.replace("NANOEDM","NANO")
0758                 def doNotInlineEventContent(instance,label = "process."+tmpstreamType+"EventContent.outputCommands"):
0759                     return label
0760                 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
0761
0762             result+="\nprocess."+outputModuleName+" = "+outputModule.dumpPython()
0763
0764         return result
0765
0766     def addStandardSequences(self):
0767         """
0768         Add selected standard sequences to the process
0769         """
0770         # load the pile up file
0771         if self._options.pileup:
0772             pileupSpec=self._options.pileup.split(',')[0]
0773
0774             # Does the requested pile-up scenario exist?
0775             from Configuration.StandardSequences.Mixing import Mixing,defineMixing
0776             if not pileupSpec in Mixing and '.' not in pileupSpec and 'file:' not in pileupSpec:
0777                 message = pileupSpec+' is not a know mixing scenario:\n available are: '+'\n'.join(Mixing.keys())
0778                 raise Exception(message)
0779
0780             # Put mixing parameters in a dictionary
0781             if '.' in pileupSpec:
0782                 mixingDict={'file':pileupSpec}
0783             elif pileupSpec.startswith('file:'):
0784                 mixingDict={'file':pileupSpec[5:]}
0785             else:
0786                 import copy
0787                 mixingDict=copy.copy(Mixing[pileupSpec])
0788             if len(self._options.pileup.split(','))>1:
0789                 mixingDict.update(eval(self._options.pileup[self._options.pileup.find(',')+1:]))
0790
0791             # Load the pu cfg file corresponding to the requested pu scenario
0792             if 'file:' in pileupSpec:
0793                 #the file is local
0794                 self.process.load(mixingDict['file'])
0795                 print("inlining mixing module configuration")
0796                 self._options.inlineObjets+=',mix'
0797             else:
0798                 self.loadAndRemember(mixingDict['file'])
0799
0800             mixingDict.pop('file')
0801             if not "DATAMIX" in self.stepMap.keys(): # when DATAMIX is present, pileup_input refers to pre-mixed GEN-RAW
0802                 if self._options.pileup_input:
0803                     if self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:'):
0804                         mixingDict['F']=filesFromDASQuery('file dataset = %s'%(self._options.pileup_input[4:],),self._options.pileup_dasoption)[0]
0805                     elif self._options.pileup_input.startswith("filelist:"):
0806                         mixingDict['F']=(filesFromList(self._options.pileup_input[9:]))[0]
0807                     else:
0808                         mixingDict['F']=self._options.pileup_input.split(',')
0809                 specialization=defineMixing(mixingDict)
0810                 for command in specialization:
0811                     self.executeAndRemember(command)
0812                 if len(mixingDict)!=0:
0813                     raise Exception('unused mixing specification: '+mixingDict.keys().__str__())
0814
0815
0816         # load the geometry file
0817         try:
0818             if len(self.stepMap):
0819                 self.loadAndRemember(self.GeometryCFF)
0820                 if ('SIM' in self.stepMap or 'reSIM' in self.stepMap) and not self._options.fast:
0821                     self.loadAndRemember(self.SimGeometryCFF)
0822                     if self.geometryDBLabel:
0823                         self.executeAndRemember('if hasattr(process, "XMLFromDBSource"): process.XMLFromDBSource.label="%s"'%(self.geometryDBLabel))
0824                         self.executeAndRemember('if hasattr(process, "DDDetectorESProducerFromDB"): process.DDDetectorESProducerFromDB.label="%s"'%(self.geometryDBLabel))
0825
0826         except ImportError:
0827             print("Geometry option",self._options.geometry,"unknown.")
0828             raise
0829
0830         if len(self.stepMap):
0831             self.loadAndRemember(self.magFieldCFF)
0832
0833         for stepName in self.stepKeys:
0834             stepSpec = self.stepMap[stepName]
0835             print("Step:", stepName,"Spec:",stepSpec)
0836             if stepName.startswith('re'):
0837                 ##add the corresponding input content
0838                 if stepName[2:] not in self._options.donotDropOnInput:
0839                     self._options.inputEventContent='%s,%s'%(stepName.upper(),self._options.inputEventContent)
0840                 stepName=stepName[2:]
0841             if stepSpec=="":
0842                 getattr(self,"prepare_"+stepName)(stepSpec = getattr(self,stepName+"DefaultSeq"))
0843             elif isinstance(stepSpec, list):
0844                 getattr(self,"prepare_"+stepName)(stepSpec = '+'.join(stepSpec))
0845             elif isinstance(stepSpec, tuple):
0846                 getattr(self,"prepare_"+stepName)(stepSpec = ','.join([stepSpec[1],'+'.join(stepSpec[0])]))
0847             else:
0848                 raise ValueError("Invalid step definition")
0849
0850         if self._options.restoreRNDSeeds!=False:
0851             #it is either True, or a process name
0852             if self._options.restoreRNDSeeds==True:
0853                 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateLabel=cms.untracked.string("randomEngineStateProducer")')
0854             else:
0855                 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateTag=cms.untracked.InputTag("randomEngineStateProducer","","%s")'%(self._options.restoreRNDSeeds))
0856             if self._options.inputEventContent or self._options.inputCommands:
0857                 if self._options.inputCommands:
0858                     self._options.inputCommands+='keep *_randomEngineStateProducer_*_*,'
0859                 else:
0860                     self._options.inputCommands='keep *_randomEngineStateProducer_*_*,'
0861
0862
0863     def completeInputCommand(self):
0864         if self._options.inputEventContent:
0865             import copy
0866             def dropSecondDropStar(iec):
0867                 #drop occurence of 'drop *' in the list
0868                 count=0
0869                 for item in iec:
0870                     if item=='drop *':
0871                         if count!=0:
0872                             iec.remove(item)
0873                         count+=1
0874
0875             ## allow comma separated input eventcontent
0876             if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring()
0877             for evct in self._options.inputEventContent.split(','):
0878                 if evct=='': continue
0879                 theEventContent = getattr(self.process, evct+"EventContent")
0880                 if hasattr(theEventContent,'outputCommands'):
0881                     self.process.source.inputCommands.extend(copy.copy(theEventContent.outputCommands))
0882                 if hasattr(theEventContent,'inputCommands'):
0883                     self.process.source.inputCommands.extend(copy.copy(theEventContent.inputCommands))
0884
0885             dropSecondDropStar(self.process.source.inputCommands)
0886
0887             if not self._options.dropDescendant:
0888                 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False)
0889
0890
0891         return
0892
0893     def addConditions(self):
0894         """Add conditions to the process"""
0895         if not self._options.conditions: return
0896
0897         if 'FrontierConditions_GlobalTag' in self._options.conditions:
0898             print('using FrontierConditions_GlobalTag in --conditions is not necessary anymore and will be deprecated soon. please update your command line')
0899             self._options.conditions = self._options.conditions.replace("FrontierConditions_GlobalTag,",'')
0900
0901         self.loadAndRemember(self.ConditionsDefaultCFF)
0902         from Configuration.AlCa.GlobalTag import GlobalTag
0903         self.process.GlobalTag = GlobalTag(self.process.GlobalTag, self._options.conditions, self._options.custom_conditions)
0904         self.additionalCommands.append('from Configuration.AlCa.GlobalTag import GlobalTag')
0905         self.additionalCommands.append('process.GlobalTag = GlobalTag(process.GlobalTag, %s, %s)' % (repr(self._options.conditions), repr(self._options.custom_conditions)))
0906
0907
0908     def addCustomise(self,unsch=0):
0909         """Include the customise code """
0910
0911         custOpt=[]
0912         if unsch==0:
0913             for c in self._options.customisation_file:
0914                 custOpt.extend(c.split(","))
0915         else:
0916             for c in self._options.customisation_file_unsch:
0917                 custOpt.extend(c.split(","))
0918
0919         custMap=DictTypes.SortedKeysDict()
0920         for opt in custOpt:
0921             if opt=='': continue
0922             if opt.count('.')>1:
0923                 raise Exception("more than . in the specification:"+opt)
0924             fileName=opt.split('.')[0]
0925             if opt.count('.')==0: rest='customise'
0926             else:
0927                 rest=opt.split('.')[1]
0928                 if rest=='py': rest='customise' #catch the case of --customise file.py
0929
0930             if fileName in custMap:
0931                 custMap[fileName].extend(rest.split('+'))
0932             else:
0933                 custMap[fileName]=rest.split('+')
0934
0935         if len(custMap)==0:
0936             final_snippet='\n'
0937         else:
0938             final_snippet='\n# customisation of the process.\n'
0939
0940         allFcn=[]
0941         for opt in custMap:
0942             allFcn.extend(custMap[opt])
0943         for fcn in allFcn:
0944             if allFcn.count(fcn)!=1:
0945                 raise Exception("cannot specify twice "+fcn+" as a customisation method")
0946
0947         for f in custMap:
0948             # let python search for that package and do syntax checking at the same time
0949             packageName = f.replace(".py","").replace("/",".")
0950             __import__(packageName)
0951             package = sys.modules[packageName]
0952
0953             # now ask the package for its definition and pick .py instead of .pyc
0954             customiseFile = re.sub(r'\.pyc$', '.py', package.__file__)
0955
0956             final_snippet+='\n# Automatic addition of the customisation function from '+packageName+'\n'
0957             if self._options.inline_custom:
0958                 for line in file(customiseFile,'r'):
0959                     if "import FWCore.ParameterSet.Config" in line:
0960                         continue
0961                     final_snippet += line
0962             else:
0963                 final_snippet += 'from %s import %s \n'%(packageName,','.join(custMap[f]))
0964             for fcn in custMap[f]:
0965                 print("customising the process with",fcn,"from",f)
0966                 if not hasattr(package,fcn):
0967                     #bound to fail at run time
0968                     raise Exception("config "+f+" has no function "+fcn)
0969                 #execute the command
0970                 self.process=getattr(package,fcn)(self.process)
0971                 #and print it in the configuration
0972                 final_snippet += "\n#call to customisation function "+fcn+" imported from "+packageName
0973                 final_snippet += "\nprocess = %s(process)\n"%(fcn,)
0974
0975         if len(custMap)!=0:
0976             final_snippet += '\n# End of customisation functions\n'
0977
0978         ### now for a useful command
0979         return final_snippet
0980
0981     def addCustomiseCmdLine(self):
0982         final_snippet='\n# Customisation from command line\n'
0983         if self._options.customise_commands:
0984             import string
0985             for com in self._options.customise_commands.split('\\n'):
0986                 com=com.lstrip()
0987                 self.executeAndRemember(com)
0988                 final_snippet +='\n'+com
0989
0990         return final_snippet
0991
0992     #----------------------------------------------------------------------------
0993     # here the methods to define the python includes for each step or
0994     # conditions
0995     #----------------------------------------------------------------------------
0996     def define_Configs(self):
0997         if len(self.stepMap):
0998             self.loadAndRemember('Configuration/StandardSequences/Services_cff')
0999         if self._options.particleTable not in defaultOptions.particleTableList:
1000             print('Invalid particle table provided. Options are:')
1001             print(defaultOptions.particleTable)
1002             sys.exit(-1)
1003         else:
1004             if len(self.stepMap):
1005                 self.loadAndRemember('SimGeneral.HepPDTESSource.'+self._options.particleTable+'_cfi')
1006
1007         self.loadAndRemember('FWCore/MessageService/MessageLogger_cfi')
1008
1009         self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreams_cff"
1010         self.GENDefaultCFF="Configuration/StandardSequences/Generator_cff"
1011         self.SIMDefaultCFF="Configuration/StandardSequences/Sim_cff"
1012         self.DIGIDefaultCFF="Configuration/StandardSequences/Digi_cff"
1013         self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRaw_cff"
1014         self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1Emulator_cff'
1015         self.L1MENUDefaultCFF="Configuration/StandardSequences/L1TriggerDefaultMenu_cff"
1016         self.HLTDefaultCFF="Configuration/StandardSequences/HLTtable_cff"
1017         self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_Data_cff"
1018         if self._options.isRepacked: self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_DataMapper_cff"
1019         self.L1RecoDefaultCFF="Configuration/StandardSequences/L1Reco_cff"
1020         self.L1TrackTriggerDefaultCFF="Configuration/StandardSequences/L1TrackTrigger_cff"
1021         self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_Data_cff"
1022         self.RECOSIMDefaultCFF="Configuration/StandardSequences/RecoSim_cff"
1023         self.PATDefaultCFF="Configuration/StandardSequences/PAT_cff"
1024         self.NANODefaultCFF="PhysicsTools/NanoAOD/nano_cff"
1025         self.NANOGENDefaultCFF="PhysicsTools/NanoAOD/nanogen_cff"
1026         self.SKIMDefaultCFF="Configuration/StandardSequences/Skims_cff"
1027         self.POSTRECODefaultCFF="Configuration/StandardSequences/PostRecoGenerator_cff"
1028         self.VALIDATIONDefaultCFF="Configuration/StandardSequences/Validation_cff"
1029         self.L1HwValDefaultCFF = "Configuration/StandardSequences/L1HwVal_cff"
1030         self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOffline_cff"
1031         self.HARVESTINGDefaultCFF="Configuration/StandardSequences/Harvesting_cff"
1032         self.ALCAHARVESTDefaultCFF="Configuration/StandardSequences/AlCaHarvesting_cff"
1033         self.ENDJOBDefaultCFF="Configuration/StandardSequences/EndOfProcess_cff"
1034         self.ConditionsDefaultCFF = "Configuration/StandardSequences/FrontierConditions_GlobalTag_cff"
1035         self.CFWRITERDefaultCFF = "Configuration/StandardSequences/CrossingFrameWriter_cff"
1036         self.REPACKDefaultCFF="Configuration/StandardSequences/DigiToRaw_Repack_cff"
1037
1038         if "DATAMIX" in self.stepMap.keys():
1039             self.DATAMIXDefaultCFF="Configuration/StandardSequences/DataMixer"+self._options.datamix+"_cff"
1040             self.DIGIDefaultCFF="Configuration/StandardSequences/DigiDM_cff"
1041             self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRawDM_cff"
1042             self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1EmulatorDM_cff'
1043
1044         self.ALCADefaultSeq=None
1045         self.LHEDefaultSeq='externalLHEProducer'
1046         self.GENDefaultSeq='pgen'
1047         self.SIMDefaultSeq='psim'
1048         self.DIGIDefaultSeq='pdigi'
1049         self.DATAMIXDefaultSeq=None
1050         self.DIGI2RAWDefaultSeq='DigiToRaw'
1051         self.HLTDefaultSeq='GRun'
1052         self.L1DefaultSeq=None
1053         self.L1REPACKDefaultSeq='GT'
1054         self.HARVESTINGDefaultSeq=None
1055         self.ALCAHARVESTDefaultSeq=None
1056         self.CFWRITERDefaultSeq=None
1057         self.RAW2DIGIDefaultSeq='RawToDigi'
1058         self.L1RecoDefaultSeq='L1Reco'
1059         self.L1TrackTriggerDefaultSeq='L1TrackTrigger'
1060         if self._options.fast or ('RAW2DIGI' in self.stepMap and 'RECO' in self.stepMap):
1061             self.RECODefaultSeq='reconstruction'
1062         else:
1063             self.RECODefaultSeq='reconstruction_fromRECO'
1064         self.RECOSIMDefaultSeq='recosim'
1065         self.POSTRECODefaultSeq=None
1066         self.L1HwValDefaultSeq='L1HwVal'
1067         self.DQMDefaultSeq='DQMOffline'
1068         self.VALIDATIONDefaultSeq=''
1069         self.ENDJOBDefaultSeq='endOfProcess'
1070         self.REPACKDefaultSeq='DigiToRawRepack'
1071         self.PATDefaultSeq='miniAOD'
1072         self.PATGENDefaultSeq='miniGEN'
1073         #TODO: Check based of file input
1074         self.NANOGENDefaultSeq='nanogenSequence'
1075         self.NANODefaultSeq='nanoSequence'
1076
1077         self.EVTCONTDefaultCFF="Configuration/EventContent/EventContent_cff"
1078
1079         if not self._options.beamspot:
1080             self._options.beamspot=VtxSmearedDefaultKey
1081
1082         # if its MC then change the raw2digi
1083         if self._options.isMC==True:
1084             self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_cff"
1085             self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_cff"
1086             self.PATDefaultCFF="Configuration/StandardSequences/PATMC_cff"
1087             self.PATGENDefaultCFF="Configuration/StandardSequences/PATGEN_cff"
1088             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineMC_cff"
1089             self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreamsMC_cff"
1090             self.NANODefaultSeq='nanoSequenceMC'
1091         else:
1092             self._options.beamspot = None
1093
1094         #patch for gen, due to backward incompatibility
1095         if 'reGEN' in self.stepMap:
1096             self.GENDefaultSeq='fixGenInfo'
1097
1098         if self._options.scenario=='cosmics':
1099             self._options.pileup='Cosmics'
1100             self.DIGIDefaultCFF="Configuration/StandardSequences/DigiCosmics_cff"
1101             self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionCosmics_cff"
1102             self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsCosmics_cff"
1103             self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentCosmics_cff"
1104             self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationCosmics_cff"
1105             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmics_cff"
1106             if self._options.isMC==True:
1107                 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmicsMC_cff"
1108             self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingCosmics_cff"
1109             self.RECODefaultSeq='reconstructionCosmics'
1110             self.DQMDefaultSeq='DQMOfflineCosmics'
1111
1112         if self._options.scenario=='HeavyIons':
1113             if not self._options.beamspot:
1114                 self._options.beamspot=VtxSmearedHIDefaultKey
1115             self.HLTDefaultSeq = 'HIon'
1116             self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationHeavyIons_cff"
1117             self.VALIDATIONDefaultSeq=''
1118             self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentHeavyIons_cff"
1119             self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionHeavyIons_cff"
1120             self.RECODefaultSeq='reconstructionHeavyIons'
1121             self.ALCADefaultCFF = "Configuration/StandardSequences/AlCaRecoStreamsHeavyIons_cff"
1122             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIons_cff"
1123             self.DQMDefaultSeq='DQMOfflineHeavyIons'
1124             self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsHeavyIons_cff"
1125             self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingHeavyIons_cff"
1126             if self._options.isMC==True:
1127                 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIonsMC_cff"
1128
1129
1130         self.RAW2RECODefaultSeq=','.join([self.RAW2DIGIDefaultSeq,self.RECODefaultSeq])
1131
1132         self.USERDefaultSeq='user'
1133         self.USERDefaultCFF=None
1134
1135         # the magnetic field
1136         self.magFieldCFF = 'Configuration/StandardSequences/MagneticField_'+self._options.magField.replace('.','')+'_cff'
1137         self.magFieldCFF = self.magFieldCFF.replace("__",'_')
1138
1139         # the geometry
1140         self.GeometryCFF='Configuration/StandardSequences/GeometryRecoDB_cff'
1141         self.geometryDBLabel=None
1142         simGeometry=''
1143         if self._options.fast:
1144             if 'start' in self._options.conditions.lower():
1145                 self.GeometryCFF='FastSimulation/Configuration/Geometries_START_cff'
1146             else:
1147                 self.GeometryCFF='FastSimulation/Configuration/Geometries_MC_cff'
1148         else:
1149             def inGeometryKeys(opt):
1150                 from Configuration.StandardSequences.GeometryConf import GeometryConf
1151                 if opt in GeometryConf:
1152                     return GeometryConf[opt]
1153                 else:
1154                     return opt
1155
1156             geoms=self._options.geometry.split(',')
1157             if len(geoms)==1: geoms=inGeometryKeys(geoms[0]).split(',')
1158             if len(geoms)==2:
1159                 #may specify the reco geometry
1160                 if '/' in geoms[1] or '_cff' in geoms[1]:
1161                     self.GeometryCFF=geoms[1]
1162                 else:
1163                     self.GeometryCFF='Configuration/Geometry/Geometry'+geoms[1]+'_cff'
1164
1165             if (geoms[0].startswith('DB:')):
1166                 self.SimGeometryCFF='Configuration/StandardSequences/GeometrySimDB_cff'
1167                 self.geometryDBLabel=geoms[0][3:]
1168                 print("with DB:")
1169             else:
1170                 if '/' in geoms[0] or '_cff' in geoms[0]:
1171                     self.SimGeometryCFF=geoms[0]
1172                 else:
1173                     simGeometry=geoms[0]
1174                     if self._options.gflash==True:
1175                         self.SimGeometryCFF='Configuration/Geometry/Geometry'+geoms[0]+'GFlash_cff'
1176                     else:
1177                         self.SimGeometryCFF='Configuration/Geometry/Geometry'+geoms[0]+'_cff'
1178
1179         # synchronize the geometry configuration and the FullSimulation sequence to be used
1180         if simGeometry not in defaultOptions.geometryExtendedOptions:
1181             self.SIMDefaultCFF="Configuration/StandardSequences/SimIdeal_cff"
1182
1183         if self._options.scenario=='nocoll' or self._options.scenario=='cosmics':
1184             self.SIMDefaultCFF="Configuration/StandardSequences/SimNOBEAM_cff"
1185             self._options.beamspot='NoSmear'
1186
1187         # fastsim requires some changes to the default cff files and sequences
1188         if self._options.fast:
1189             self.SIMDefaultCFF = 'FastSimulation.Configuration.SimIdeal_cff'
1190             self.RECODefaultCFF= 'FastSimulation.Configuration.Reconstruction_AftMix_cff'
1191             self.RECOBEFMIXDefaultCFF = 'FastSimulation.Configuration.Reconstruction_BefMix_cff'
1192             self.RECOBEFMIXDefaultSeq = 'reconstruction_befmix'
1193             self.NANODefaultSeq = 'nanoSequenceFS'
1194             self.DQMOFFLINEDefaultCFF="DQMOffline.Configuration.DQMOfflineFS_cff"
1195
1196         # Mixing
1197         if self._options.pileup=='default':
1198             from Configuration.StandardSequences.Mixing import MixingDefaultKey
1199             self._options.pileup=MixingDefaultKey
1200
1201
1202         #not driven by a default cff anymore
1203         if self._options.isData:
1204             self._options.pileup=None
1205
1206
1207         self.REDIGIDefaultSeq=self.DIGIDefaultSeq
1208
1209     # for alca, skims, etc
1210     def addExtraStream(self, name, stream, workflow='full'):
1211             # define output module and go from there
1212         output = cms.OutputModule("PoolOutputModule")
1213         if stream.selectEvents.parameters_().__len__()!=0:
1214             output.SelectEvents = stream.selectEvents
1215         else:
1216             output.SelectEvents = cms.untracked.PSet()
1217             output.SelectEvents.SelectEvents=cms.vstring()
1218             if isinstance(stream.paths,tuple):
1219                 for path in stream.paths:
1220                     output.SelectEvents.SelectEvents.append(path.label())
1221             else:
1222                 output.SelectEvents.SelectEvents.append(stream.paths.label())
1223
1224
1225
1226         if isinstance(stream.content,str):
1227             evtPset=getattr(self.process,stream.content)
1228             for p in evtPset.parameters_():
1229                 setattr(output,p,getattr(evtPset,p))
1230             if not self._options.inlineEventContent:
1231                 def doNotInlineEventContent(instance,label = "process."+stream.content+".outputCommands"):
1232                     return label
1233                 output.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
1234         else:
1235             output.outputCommands = stream.content
1236
1237
1238         output.fileName = cms.untracked.string(self._options.dirout+stream.name+'.root')
1239
1240         output.dataset  = cms.untracked.PSet( dataTier = stream.dataTier,
1241                                               filterName = cms.untracked.string(stream.name))
1242
1243         if self._options.filtername:
1244             output.dataset.filterName= cms.untracked.string(self._options.filtername+"_"+stream.name)
1245
1246         #add an automatic flushing to limit memory consumption
1247         output.eventAutoFlushCompressedSize=cms.untracked.int32(5*1024*1024)
1248
1249         if workflow in ("producers,full"):
1250             if isinstance(stream.paths,tuple):
1251                 for path in stream.paths:
1252                     self.schedule.append(path)
1253             else:
1254                 self.schedule.append(stream.paths)
1255
1256
1257         # in case of relvals we don't want to have additional outputs
1258         if (not self._options.relval) and workflow in ("full","output"):
1259             self.additionalOutputs[name] = output
1260             setattr(self.process,name,output)
1261
1262         if workflow == 'output':
1263             # adjust the select events to the proper trigger results from previous process
1264             filterList = output.SelectEvents.SelectEvents
1265             for i, filter in enumerate(filterList):
1266                 filterList[i] = filter+":"+self._options.triggerResultsProcess
1267
1268         return output
1269
1270     #----------------------------------------------------------------------------
1271     # here the methods to create the steps. Of course we are doing magic here ;)
1272     # prepare_STEPNAME modifies self.process and what else's needed.
1273     #----------------------------------------------------------------------------
1274
1275     def loadDefaultOrSpecifiedCFF(self, stepSpec, defaultCFF, defaultSEQ=''):
1276         _dotsplit = stepSpec.split('.')
1277         if ( len(_dotsplit)==1 ):
1278             if '/' in _dotsplit[0]:
1279                 _sequence = defaultSEQ if defaultSEQ else stepSpec
1280                 _cff = _dotsplit[0]
1281             else:
1282                 _sequence = stepSpec
1283                 _cff = defaultCFF
1284         elif ( len(_dotsplit)==2 ):
1285             _cff,_sequence  = _dotsplit
1286         else:
1287             print("sub sequence configuration must be of the form dir/subdir/cff.a+b+c or cff.a")
1288             print(stepSpec,"not recognized")
1289             raise
1290         l=self.loadAndRemember(_cff)
1291         return l,_sequence,_cff
1292
1293     def scheduleSequence(self,seq,prefix,what='Path'):
1294         if '*' in seq:
1295             #create only one path with all sequences in it
1296             for i,s in enumerate(seq.split('*')):
1297                 if i==0:
1298                     setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, s) ))
1299                 else:
1300                     p=getattr(self.process,prefix)
1301                     tmp = getattr(self.process, s)
1302                     if isinstance(tmp, cms.Task):
1303                         p.associate(tmp)
1304                     else:
1305                         p+=tmp
1306             self.schedule.append(getattr(self.process,prefix))
1307             return
1308         else:
1309             #create as many path as many sequences
1310             if not '+' in seq:
1311                 if self.nextScheduleIsConditional:
1312                     self.conditionalPaths.append(prefix)
1313                 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, seq) ))
1314                 self.schedule.append(getattr(self.process,prefix))
1315             else:
1316                 for i,s in enumerate(seq.split('+')):
1317                     sn=prefix+'%d'%(i)
1318                     setattr(self.process,sn,getattr(cms,what)( getattr(self.process, s) ))
1319                     self.schedule.append(getattr(self.process,sn))
1320             return
1321
1322     def scheduleSequenceAtEnd(self,seq,prefix):
1323         self.scheduleSequence(seq,prefix,what='EndPath')
1324         return
1325
1326     def prepare_ALCAPRODUCER(self, stepSpec = None):
1327         self.prepare_ALCA(stepSpec, workflow = "producers")
1328
1329     def prepare_ALCAOUTPUT(self, stepSpec = None):
1330         self.prepare_ALCA(stepSpec, workflow = "output")
1331
1332     def prepare_ALCA(self, stepSpec = None, workflow = 'full'):
1333         """ Enrich the process with alca streams """
1334         alcaConfig,sequence,_=self.loadDefaultOrSpecifiedCFF(stepSpec,self.ALCADefaultCFF)
1335
1336         MAXLEN=31 #the alca producer name should be shorter than 31 chars as per https://cms-talk.web.cern.ch/t/alcaprompt-datasets-not-loaded-in-dbs/11146/2
1337         # decide which ALCA paths to use
1338         alcaList = sequence.split("+")
1339         for alca in alcaList:
1340             if (len(alca)>MAXLEN):
1341                 raise Exception("The following alca "+str(alca)+" name (with length "+str(len(alca))+" chars) cannot be accepted because it exceeds the DBS constraints on the length of the name of the ALCARECOs producers ("+str(MAXLEN)+")!")
1342
1343         maxLevel=0
1344         from Configuration.AlCa.autoAlca import autoAlca, AlCaNoConcurrentLumis
1345         # support @X from autoAlca.py, and recursion support: i.e T0:@Mu+@EG+...
1346         self.expandMapping(alcaList,autoAlca)
1347         self.AlCaPaths=[]
1348         for name in alcaConfig.__dict__:
1349             alcastream = getattr(alcaConfig,name)
1350             shortName = name.replace('ALCARECOStream','')
1351             if shortName in alcaList and isinstance(alcastream,cms.FilteredStream):
1352                 if shortName in AlCaNoConcurrentLumis:
1353                     print("Setting numberOfConcurrentLuminosityBlocks=1 because of AlCa sequence {}".format(shortName))
1354                     self._options.nConcurrentLumis = "1"
1355                     self._options.nConcurrentIOVs = "1"
1356                 output = self.addExtraStream(name,alcastream, workflow = workflow)
1357                 self.executeAndRemember('process.ALCARECOEventContent.outputCommands.extend(process.OutALCARECO'+shortName+'_noDrop.outputCommands)')
1358                 self.AlCaPaths.append(shortName)
1359                 if 'DQM' in alcaList:
1360                     if not self._options.inlineEventContent and hasattr(self.process,name):
1361                         self.executeAndRemember('process.' + name + '.outputCommands.append("keep *_MEtoEDMConverter_*_*")')
1362                     else:
1363                         output.outputCommands.append("keep *_MEtoEDMConverter_*_*")
1364
1365                 #rename the HLT process name in the alca modules
1366                 if self._options.hltProcess or 'HLT' in self.stepMap:
1367                     if isinstance(alcastream.paths,tuple):
1368                         for path in alcastream.paths:
1369                             self.renameHLTprocessInSequence(path.label())
1370                     else:
1371                         self.renameHLTprocessInSequence(alcastream.paths.label())
1372
1373                 for i in range(alcaList.count(shortName)):
1374                     alcaList.remove(shortName)
1375
1376             # DQM needs a special handling
1377             elif name == 'pathALCARECODQM' and 'DQM' in alcaList:
1378                 path = getattr(alcaConfig,name)
1379                 self.schedule.append(path)
1380                 alcaList.remove('DQM')
1381
1382             if isinstance(alcastream,cms.Path):
1383                 #black list the alca path so that they do not appear in the cfg
1384                 self.blacklist_paths.append(alcastream)
1385
1386
1387         if len(alcaList) != 0:
1388             available=[]
1389             for name in alcaConfig.__dict__:
1390                 alcastream = getattr(alcaConfig,name)
1391                 if isinstance(alcastream,cms.FilteredStream):
1392                     available.append(name.replace('ALCARECOStream',''))
1393             print("The following alcas could not be found "+str(alcaList))
1394             print("available ",available)
1395             #print "verify your configuration, ignoring for now"
1396             raise Exception("The following alcas could not be found "+str(alcaList))
1397
1398     def prepare_LHE(self, stepSpec = None):
1399             #load the fragment
1400             ##make it loadable
1401         loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','').replace('/','.')
1402         print("Loading lhe fragment from",loadFragment)
1403         __import__(loadFragment)
1404         self.process.load(loadFragment)
1405         ##inline the modules
1406         self._options.inlineObjets+=','+stepSpec
1407
1408         getattr(self.process,stepSpec).nEvents = int(self._options.number)
1409
1410         #schedule it
1411         self.process.lhe_step = cms.Path( getattr( self.process,stepSpec)  )
1412         self.excludedPaths.append("lhe_step")
1413         self.schedule.append( self.process.lhe_step )
1414
1415     def prepare_GEN(self, stepSpec = None):
1416         """ load the fragment of generator configuration """
1417         loadFailure=False
1418         #remove trailing .py
1419         #support old style .cfi by changing into something.cfi into something_cfi
1420         #remove python/ from the name
1421         loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','')
1422         #standard location of fragments
1423         if not '/' in loadFragment:
1424             loadFragment='Configuration.Generator.'+loadFragment
1425         else:
1426             loadFragment=loadFragment.replace('/','.')
1427         try:
1428             print("Loading generator fragment from",loadFragment)
1429             __import__(loadFragment)
1430         except:
1431             loadFailure=True
1432             #if self.process.source and self.process.source.type_()=='EmptySource':
1433             if not (self._options.filein or self._options.dasquery):
1434                 raise Exception("Neither gen fragment of input files provided: this is an inconsistent GEN step configuration")
1435
1436         if not loadFailure:
1437             from Configuration.Generator.concurrentLumisDisable import noConcurrentLumiGenerators
1438
1439             generatorModule=sys.modules[loadFragment]
1440             genModules=generatorModule.__dict__
1441             #remove lhe producer module since this should have been
1442             #imported instead in the LHE step
1443             if self.LHEDefaultSeq in genModules:
1444                 del genModules[self.LHEDefaultSeq]
1445
1446             if self._options.hideGen:
1447                 self.loadAndRemember(loadFragment)
1448             else:
1449                 self.process.load(loadFragment)
1450                 # expose the objects from that fragment to the configuration
1451                 import FWCore.ParameterSet.Modules as cmstypes
1452                 for name in genModules:
1453                     theObject = getattr(generatorModule,name)
1454                     if isinstance(theObject, cmstypes._Module):
1455                         self._options.inlineObjets=name+','+self._options.inlineObjets
1456                         if theObject.type_() in noConcurrentLumiGenerators:
1457                             print("Setting numberOfConcurrentLuminosityBlocks=1 because of generator {}".format(theObject.type_()))
1458                             self._options.nConcurrentLumis = "1"
1459                             self._options.nConcurrentIOVs = "1"
1460                     elif isinstance(theObject, cms.Sequence) or isinstance(theObject, cmstypes.ESProducer):
1461                         self._options.inlineObjets+=','+name
1462
1463             if stepSpec == self.GENDefaultSeq or stepSpec == 'pgen_genonly' or stepSpec == 'pgen_smear':
1464                 if 'ProductionFilterSequence' in genModules and ('generator' in genModules):
1465                     self.productionFilterSequence = 'ProductionFilterSequence'
1466                 elif 'generator' in genModules:
1467                     self.productionFilterSequence = 'generator'
1468
1469         """ Enrich the schedule with the rest of the generation step """
1470         _,_genSeqName,_=self.loadDefaultOrSpecifiedCFF(stepSpec,self.GENDefaultCFF)
1471
1472         if True:
1473             try:
1474                 from Configuration.StandardSequences.VtxSmeared import VtxSmeared
1475                 cffToBeLoaded=VtxSmeared[self._options.beamspot]
1476                 self.loadAndRemember(cffToBeLoaded)
1477             except ImportError:
1478                 raise Exception("VertexSmearing type or beamspot "+self._options.beamspot+" unknown.")
1479
1480             if self._options.scenario == 'HeavyIons':
1481                 if self._options.pileup=='HiMixGEN':
1482                     self.loadAndRemember("Configuration/StandardSequences/GeneratorMix_cff")
1483                 elif self._options.pileup=='HiMixEmbGEN':
1484                     self.loadAndRemember("Configuration/StandardSequences/GeneratorEmbMix_cff")
1485                 else:
1486                     self.loadAndRemember("Configuration/StandardSequences/GeneratorHI_cff")
1487
1488         self.process.generation_step = cms.Path( getattr(self.process,_genSeqName) )
1489         self.schedule.append(self.process.generation_step)
1490
1491         #register to the genstepfilter the name of the path (static right now, but might evolve)
1492         self.executeAndRemember('process.genstepfilter.triggerConditions=cms.vstring("generation_step")')
1493
1494         if 'reGEN' in self.stepMap:
1495             #stop here
1496             return
1497
1498         """ Enrich the schedule with the summary of the filter step """
1499         #the gen filter in the endpath
1500         self.loadAndRemember("GeneratorInterface/Core/genFilterSummary_cff")
1501         self.scheduleSequenceAtEnd('genFilterSummary','genfiltersummary_step')
1502         return
1503
1504     def prepare_SIM(self, stepSpec = None):
1505         """ Enrich the schedule with the simulation step"""
1506         _,_simSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.SIMDefaultCFF)
1507         if not self._options.fast:
1508             if self._options.gflash==True:
1509                 self.loadAndRemember("Configuration/StandardSequences/GFlashSIM_cff")
1510
1511             if self._options.magField=='0T':
1512                 self.executeAndRemember("process.g4SimHits.UseMagneticField = cms.bool(False)")
1513         else:
1514             if self._options.magField=='0T':
1515                 self.executeAndRemember("process.fastSimProducer.detectorDefinition.magneticFieldZ = cms.untracked.double(0.)")
1516
1517         self.scheduleSequence(_simSeq,'simulation_step')
1518         return
1519
1520     def prepare_DIGI(self, stepSpec = None):
1521         """ Enrich the schedule with the digitisation step"""
1522         _,_digiSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.DIGIDefaultCFF)
1523
1524         if self._options.gflash==True:
1525             self.loadAndRemember("Configuration/StandardSequences/GFlashDIGI_cff")
1526
1527         if _digiSeq == 'pdigi_valid' or _digiSeq == 'pdigi_hi':
1528             self.executeAndRemember("process.mix.digitizers = cms.PSet(process.theDigitizersValid)")
1529
1530         if _digiSeq != 'pdigi_nogen' and _digiSeq != 'pdigi_valid_nogen' and _digiSeq != 'pdigi_hi_nogen' and not self.process.source.type_()=='EmptySource' and not self._options.filetype == "LHE":
1531             if self._options.inputEventContent=='':
1532                 self._options.inputEventContent='REGEN'
1533             else:
1534                 self._options.inputEventContent=self._options.inputEventContent+',REGEN'
1535
1536
1537         self.scheduleSequence(_digiSeq,'digitisation_step')
1538         return
1539
1540     def prepare_CFWRITER(self, stepSpec = None):
1541         """ Enrich the schedule with the crossing frame writer step"""
1542         self.loadAndRemember(self.CFWRITERDefaultCFF)
1543         self.scheduleSequence('pcfw','cfwriter_step')
1544         return
1545
1546     def prepare_DATAMIX(self, stepSpec = None):
1547         """ Enrich the schedule with the digitisation step"""
1548         self.loadAndRemember(self.DATAMIXDefaultCFF)
1549         self.scheduleSequence('pdatamix','datamixing_step')
1550
1551         if self._options.pileup_input:
1552             theFiles=''
1553             if self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:'):
1554                 theFiles=filesFromDASQuery('file dataset = %s'%(self._options.pileup_input[4:],),self._options.pileup_dasoption)[0]
1555             elif self._options.pileup_input.startswith("filelist:"):
1556                 theFiles= (filesFromList(self._options.pileup_input[9:]))[0]
1557             else:
1558                 theFiles=self._options.pileup_input.split(',')
1559             #print theFiles
1560             self.executeAndRemember( "process.mixData.input.fileNames = cms.untracked.vstring(%s)"%(  theFiles ) )
1561
1562         return
1563
1564     def prepare_DIGI2RAW(self, stepSpec = None):
1565         _,_digi2rawSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.DIGI2RAWDefaultCFF)
1566         self.scheduleSequence(_digi2rawSeq,'digi2raw_step')
1567         return
1568
1569     def prepare_REPACK(self, stepSpec = None):
1570         _,_repackSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.REPACKDefaultCFF)
1571         self.scheduleSequence(_repackSeq,'digi2repack_step')
1572         return
1573
1574     def prepare_L1(self, stepSpec = None):
1575         """ Enrich the schedule with the L1 simulation step"""
1576         assert(stepSpec == None)
1577         self.loadAndRemember(self.L1EMDefaultCFF)
1578         self.scheduleSequence('SimL1Emulator','L1simulation_step')
1579         return
1580
1581     def prepare_L1REPACK(self, stepSpec = None):
1582         """ Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection"""
1583         supported = ['GT','GT1','GT2','GCTGT','Full','FullSimTP','FullMC','Full2015Data','uGT','CalouGT']
1584         if stepSpec in supported:
1585             self.loadAndRemember('Configuration/StandardSequences/SimL1EmulatorRepack_%s_cff'% stepSpec)
1586             if self._options.scenario == 'HeavyIons':
1587                 self.renameInputTagsInSequence("SimL1Emulator","rawDataCollector","rawDataRepacker")
1588             self.scheduleSequence('SimL1Emulator','L1RePack_step')
1589         else:
1590             print("L1REPACK with '",stepSpec,"' is not supported! Supported choices are: ",supported)
1591             raise Exception('unsupported feature')
1592
1593     def prepare_HLT(self, stepSpec = None):
1594         """ Enrich the schedule with the HLT simulation step"""
1595         if not stepSpec:
1596             print("no specification of the hlt menu has been given, should never happen")
1597             raise  Exception('no HLT specifications provided')
1598
1599         if '@' in stepSpec:
1600             # case where HLT:@something was provided
1601             from Configuration.HLT.autoHLT import autoHLT
1602             key = stepSpec[1:]
1603             if key in autoHLT:
1604                 stepSpec = autoHLT[key]
1605             else:
1606                 raise ValueError('no HLT mapping key "%s" found in autoHLT' % key)
1607
1608         if ',' in stepSpec:
1609             #case where HLT:something:something was provided
1610             self.executeAndRemember('import HLTrigger.Configuration.Utilities')
1611             optionsForHLT = {}
1612             if self._options.scenario == 'HeavyIons':
1613                 optionsForHLT['type'] = 'HIon'
1614             else:
1615                 optionsForHLT['type'] = 'GRun'
1616             optionsForHLTConfig = ', '.join('%s=%s' % (key, repr(val)) for (key, val) in optionsForHLT.items())
1617             if stepSpec == 'run,fromSource':
1618                 if hasattr(self.process.source,'firstRun'):
1619                     self.executeAndRemember('process.loadHltConfiguration("run:%%d"%%(process.source.firstRun.value()),%s)'%(optionsForHLTConfig))
1620                 elif hasattr(self.process.source,'setRunNumber'):
1621                     self.executeAndRemember('process.loadHltConfiguration("run:%%d"%%(process.source.setRunNumber.value()),%s)'%(optionsForHLTConfig))
1622                 else:
1623                     raise Exception(f'Cannot replace menu to load {stepSpec}')
1624             else:
1625                 self.executeAndRemember('process.loadHltConfiguration("%s",%s)'%(stepSpec.replace(',',':'),optionsForHLTConfig))
1626         else:
1627             self.loadAndRemember('HLTrigger/Configuration/HLT_%s_cff' % stepSpec)
1628
1629         if self._options.isMC:
1630             self._options.customisation_file.append("HLTrigger/Configuration/customizeHLTforMC.customizeHLTforMC")
1631
1632         if self._options.name != 'HLT':
1633             self.additionalCommands.append('from HLTrigger.Configuration.CustomConfigs import ProcessName')
1634             self.additionalCommands.append('process = ProcessName(process)')
1635             self.additionalCommands.append('')
1636             from HLTrigger.Configuration.CustomConfigs import ProcessName
1637             self.process = ProcessName(self.process)
1638
1639         if self.process.schedule == None:
1640             raise Exception('the HLT step did not attach a valid schedule to the process')
1641
1642         self.scheduleIndexOfFirstHLTPath = len(self.schedule)
1643         [self.blacklist_paths.append(path) for path in self.process.schedule if isinstance(path,(cms.Path,cms.EndPath))]
1644
1645         # this is a fake, to be removed with fastim migration and HLT menu dump
1646         if self._options.fast:
1647             if not hasattr(self.process,'HLTEndSequence'):
1648                 self.executeAndRemember("process.HLTEndSequence = cms.Sequence( process.dummyModule )")
1649
1650
1651     def prepare_RAW2RECO(self, stepSpec = None):
1652         if ','in stepSpec:
1653             seqReco,seqDigi=stepSpec.spli(',')
1654         else:
1655             print(f"RAW2RECO requires two specifications {stepSpec} insufficient")
1656
1657         self.prepare_RAW2DIGI(seqDigi)
1658         self.prepare_RECO(seqReco)
1659         return
1660
1661     def prepare_RAW2DIGI(self, stepSpec = "RawToDigi"):
1662         _,_raw2digiSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.RAW2DIGIDefaultCFF)
1663         self.scheduleSequence(_raw2digiSeq,'raw2digi_step')
1664         return
1665
1666     def prepare_PATFILTER(self, stepSpec = None):
1667         self.loadAndRemember("PhysicsTools/PatAlgos/slimming/metFilterPaths_cff")
1668         from PhysicsTools.PatAlgos.slimming.metFilterPaths_cff import allMetFilterPaths
1669         for filt in allMetFilterPaths:
1670             self.schedule.append(getattr(self.process,'Flag_'+filt))
1671
1672     def prepare_L1HwVal(self, stepSpec = 'L1HwVal'):
1673         ''' Enrich the schedule with L1 HW validation '''
1674         self.loadDefaultOrSpecifiedCFF(stepSpec,self.L1HwValDefaultCFF)
1675         print('\n\n\n DEPRECATED this has no action \n\n\n')
1676         return
1677
1678     def prepare_L1Reco(self, stepSpec = "L1Reco"):
1679         ''' Enrich the schedule with L1 reconstruction '''
1680         _,_l1recoSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.L1RecoDefaultCFF)
1681         self.scheduleSequence(_l1recoSeq,'L1Reco_step')
1682         return
1683
1684     def prepare_L1TrackTrigger(self, stepSpec = "L1TrackTrigger"):
1685         ''' Enrich the schedule with L1 reconstruction '''
1686         _,_l1tracktriggerSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.L1TrackTriggerDefaultCFF)
1687         self.scheduleSequence(_l1tracktriggerSeq,'L1TrackTrigger_step')
1688         return
1689
1690     def prepare_FILTER(self, stepSpec = None):
1691         ''' Enrich the schedule with a user defined filter sequence '''
1692         ## load the relevant part
1693         filterConfig,filterSeq = stepSpec.split('.')
1694         filterConfig=self.load(filterConfig)
1695         ## print it in the configuration
1696         class PrintAllModules(object):
1697             def __init__(self):
1698                 self.inliner=''
1699                 pass
1700             def enter(self,visitee):
1701                 try:
1702                     label=visitee.label()
1703                     ##needs to be in reverse order
1704                     self.inliner=label+','+self.inliner
1705                 except:
1706                     pass
1707             def leave(self,v): pass
1708
1709         expander=PrintAllModules()
1710         getattr(self.process,filterSeq).visit( expander )
1711         self._options.inlineObjets+=','+expander.inliner
1712         self._options.inlineObjets+=','+filterSeq
1713
1714         ## put the filtering path in the schedule
1715         self.scheduleSequence(filterSeq,'filtering_step')
1716         self.nextScheduleIsConditional=True
1717         ## put it before all the other paths
1718         self.productionFilterSequence = filterSeq
1719
1720         return
1721
1722     def prepare_RECO(self, stepSpec = "reconstruction"):
1723         ''' Enrich the schedule with reconstruction '''
1724         _,_recoSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.RECODefaultCFF)
1725         self.scheduleSequence(_recoSeq,'reconstruction_step')
1726         return
1727
1728     def prepare_RECOSIM(self, stepSpec = "recosim"):
1729         ''' Enrich the schedule with reconstruction '''
1730         _,_recosimSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.RECOSIMDefaultCFF)
1731         self.scheduleSequence(_recosimSeq,'recosim_step')
1732         return
1733
1734     def prepare_RECOBEFMIX(self, stepSpec = "reconstruction"):
1735         ''' Enrich the schedule with the part of reconstruction that is done before mixing in FastSim'''
1736         if not self._options.fast:
1737             print("ERROR: this step is only implemented for FastSim")
1738             sys.exit()
1739         _,_recobefmixSeq,_ = self.loadDefaultOrSpecifiedCFF(self.RECOBEFMIXDefaultSeq,self.RECOBEFMIXDefaultCFF)
1740         self.scheduleSequence(_recobefmixSeq,'reconstruction_befmix_step')
1741         return
1742
1743     def prepare_PAT(self, stepSpec = "miniAOD"):
1744         ''' Enrich the schedule with PAT '''
1745         self.prepare_PATFILTER(self)
1746         self.loadDefaultOrSpecifiedCFF(stepSpec,self.PATDefaultCFF)
1747         self.labelsToAssociate.append('patTask')
1748         if self._options.isData:
1749             self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllData")
1750         else:
1751             if self._options.fast:
1752                 self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllMCFastSim")
1753             else:
1754                 self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllMC")
1755
1756         if self._options.hltProcess:
1757             if len(self._options.customise_commands) > 1:
1758                 self._options.customise_commands = self._options.customise_commands + " \n"
1759             self._options.customise_commands = self._options.customise_commands + "process.patTrigger.processName = \""+self._options.hltProcess+"\"\n"
1760             self._options.customise_commands = self._options.customise_commands + "process.slimmedPatTrigger.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1761             self._options.customise_commands = self._options.customise_commands + "process.patMuons.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1762
1763 #            self.renameHLTprocessInSequence(sequence)
1764
1765         return
1766
1767     def prepare_PATGEN(self, stepSpec = "miniGEN"):
1768         ''' Enrich the schedule with PATGEN '''
1769         self.loadDefaultOrSpecifiedCFF(stepSpec,self.PATGENDefaultCFF) #this is unscheduled
1770         self.labelsToAssociate.append('patGENTask')
1771         if self._options.isData:
1772             raise Exception("PATGEN step can only run on MC")
1773         return
1774
1775     def prepare_NANO(self, stepSpec = '' ):
1776         print(f"in prepare_nano {stepSpec}")
1777         ''' Enrich the schedule with NANO '''
1778         _,_nanoSeq,_nanoCff = self.loadDefaultOrSpecifiedCFF(stepSpec,self.NANODefaultCFF,self.NANODefaultSeq)
1779         self.scheduleSequence(_nanoSeq,'nanoAOD_step')
1780         custom = "nanoAOD_customizeCommon"
1781         self._options.customisation_file.insert(0,'.'.join([_nanoCff,custom]))
1782         if self._options.hltProcess:
1783             if len(self._options.customise_commands) > 1:
1784                 self._options.customise_commands = self._options.customise_commands + " \n"
1785             self._options.customise_commands = self._options.customise_commands + "process.unpackedPatTrigger.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1786
1787     def prepare_NANOGEN(self, stepSpec = "nanoAOD"):
1788         ''' Enrich the schedule with NANOGEN '''
1789         # TODO: Need to modify this based on the input file type
1790         fromGen = any([x in self.stepMap for x in ['LHE', 'GEN', 'AOD']])
1791         _,_nanogenSeq,_nanogenCff = self.loadDefaultOrSpecifiedCFF(stepSpec,self.NANOGENDefaultCFF)
1792         self.scheduleSequence(_nanogenSeq,'nanoAOD_step')
1793         custom = "customizeNanoGEN" if fromGen else "customizeNanoGENFromMini"
1794         if self._options.runUnscheduled:
1795             self._options.customisation_file_unsch.insert(0, '.'.join([_nanogenCff, custom]))
1796         else:
1797             self._options.customisation_file.insert(0, '.'.join([_nanogenCff, custom]))
1798
1799     def prepare_SKIM(self, stepSpec = "all"):
1800         ''' Enrich the schedule with skimming fragments'''
1801         skimConfig,sequence,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.SKIMDefaultCFF)
1802
1803         stdHLTProcName = 'HLT'
1804         newHLTProcName = self._options.hltProcess
1805         customiseForReHLT = (newHLTProcName or (stdHLTProcName in self.stepMap)) and (newHLTProcName != stdHLTProcName)
1806         if customiseForReHLT:
1807             print("replacing %s process name - step SKIM:%s will use '%s'" % (stdHLTProcName, sequence, newHLTProcName))
1808
1809         ## support @Mu+DiJet+@Electron configuration via autoSkim.py
1810         from Configuration.Skimming.autoSkim import autoSkim
1811         skimlist = sequence.split('+')
1812         self.expandMapping(skimlist,autoSkim)
1813
1814         #print("dictionary for skims:", skimConfig.__dict__)
1815         for skim in skimConfig.__dict__:
1816             skimstream = getattr(skimConfig, skim)
1817
1818             # blacklist AlCa paths so that they do not appear in the cfg
1819             if isinstance(skimstream, cms.Path):
1820                 self.blacklist_paths.append(skimstream)
1821             # if enabled, apply "hltProcess" renaming to Sequences
1822             elif isinstance(skimstream, cms.Sequence):
1823                 if customiseForReHLT:
1824                     self.renameHLTprocessInSequence(skim, proc = newHLTProcName, HLTprocess = stdHLTProcName, verbosityLevel = 0)
1825
1826             if not isinstance(skimstream, cms.FilteredStream):
1827                 continue
1828
1829             shortname = skim.replace('SKIMStream','')
1830             if (sequence=="all"):
1831                 self.addExtraStream(skim,skimstream)
1832             elif (shortname in skimlist):
1833                 self.addExtraStream(skim,skimstream)
1834                 #add a DQM eventcontent for this guy
1835                 if self._options.datatier=='DQM':
1836                     self.process.load(self.EVTCONTDefaultCFF)
1837                     skimstreamDQM = cms.FilteredStream(
1838                             responsible = skimstream.responsible,
1839                             name = skimstream.name+'DQM',
1840                             paths = skimstream.paths,
1841                             selectEvents = skimstream.selectEvents,
1842                             content = self._options.datatier+'EventContent',
1843                             dataTier = cms.untracked.string(self._options.datatier)
1844                             )
1845                     self.addExtraStream(skim+'DQM',skimstreamDQM)
1846                 for i in range(skimlist.count(shortname)):
1847                     skimlist.remove(shortname)
1848
1849         if (skimlist.__len__()!=0 and sequence!="all"):
1850             print('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
1851             raise Exception('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
1852
1853
1854     def prepare_USER(self, stepSpec = None):
1855         ''' Enrich the schedule with a user defined sequence '''
1856         _,_userSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.USERDefaultCFF)
1857         self.scheduleSequence(_userSeq,'user_step')
1858         return
1859
1860     def prepare_POSTRECO(self, stepSpec = None):
1861         """ Enrich the schedule with the postreco step """
1862         self.loadAndRemember(self.POSTRECODefaultCFF)
1863         self.scheduleSequence('postreco_generator','postreco_step')
1864         return
1865
1866
1867     def prepare_VALIDATION(self, stepSpec = 'validation'):
1868         print(f"{stepSpec} in preparing validation")
1869         _,sequence,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.VALIDATIONDefaultCFF)
1870         from Validation.Configuration.autoValidation import autoValidation
1871         #in case VALIDATION:something:somethingelse -> something,somethingelse
1872         if sequence.find(',')!=-1:
1873             prevalSeqName=sequence.split(',')[0].split('+')
1874             valSeqName=sequence.split(',')[1].split('+')
1875             self.expandMapping(prevalSeqName,autoValidation,index=0)
1876             self.expandMapping(valSeqName,autoValidation,index=1)
1877         else:
1878             if '@' in sequence:
1879                 prevalSeqName=sequence.split('+')
1880                 valSeqName=sequence.split('+')
1881                 self.expandMapping(prevalSeqName,autoValidation,index=0)
1882                 self.expandMapping(valSeqName,autoValidation,index=1)
1883             else:
1884                 postfix=''
1885                 if sequence:
1886                     postfix='_'+sequence
1887                 prevalSeqName=['prevalidation'+postfix]
1888                 valSeqName=['validation'+postfix]
1889                 if not hasattr(self.process,valSeqName[0]):
1890                     prevalSeqName=['']
1891                     valSeqName=[sequence]
1892
1893         def NFI(index):
1894             ##name from index, required to keep backward compatibility
1895             if index==0:
1896                 return ''
1897             else:
1898                 return '%s'%index
1899
1900
1901         #rename the HLT process in validation steps
1902         if ('HLT' in self.stepMap and not self._options.fast) or self._options.hltProcess:
1903             for s in valSeqName+prevalSeqName:
1904                 if s:
1905                     self.renameHLTprocessInSequence(s)
1906         for (i,s) in enumerate(prevalSeqName):
1907             if s:
1908                 setattr(self.process,'prevalidation_step%s'%NFI(i),  cms.Path( getattr(self.process, s)) )
1909                 self.schedule.append(getattr(self.process,'prevalidation_step%s'%NFI(i)))
1910
1911         for (i,s) in enumerate(valSeqName):
1912             setattr(self.process,'validation_step%s'%NFI(i), cms.EndPath( getattr(self.process, s)))
1913             self.schedule.append(getattr(self.process,'validation_step%s'%NFI(i)))
1914
1915         #needed in case the miniAODValidation sequence is run starting from AODSIM
1916         if 'PAT' in self.stepMap and not 'RECO' in self.stepMap:
1917             return
1918
1919         if not 'DIGI' in self.stepMap and not self._options.fast and not any(map( lambda s : s.startswith('genvalid'), valSeqName)):
1920             if self._options.restoreRNDSeeds==False and not self._options.restoreRNDSeeds==True:
1921                 self._options.restoreRNDSeeds=True
1922
1923         if not 'DIGI' in self.stepMap and not self._options.isData and not self._options.fast:
1924             self.executeAndRemember("process.mix.playback = True")
1925             self.executeAndRemember("process.mix.digitizers = cms.PSet()")
1926             self.executeAndRemember("for a in process.aliases: delattr(process, a)")
1927             self._options.customisation_file.append("SimGeneral/MixingModule/fullMixCustomize_cff.setCrossingFrameOn")
1928
1929         if hasattr(self.process,"genstepfilter") and len(self.process.genstepfilter.triggerConditions):
1930             #will get in the schedule, smoothly
1931             for (i,s) in enumerate(valSeqName):
1932                 getattr(self.process,'validation_step%s'%NFI(i)).insert(0, self.process.genstepfilter)
1933
1934         return
1935
1936
1937     class MassSearchReplaceProcessNameVisitor(object):
1938         """Visitor that travels within a cms.Sequence, looks for a parameter and replace its value
1939         It will climb down within PSets, VPSets and VInputTags to find its target"""
1940         def __init__(self, paramSearch, paramReplace, verbose=False, whitelist=()):
1941             self._paramReplace = paramReplace
1942             self._paramSearch = paramSearch
1943             self._verbose = verbose
1944             self._whitelist = whitelist
1945
1946         def doIt(self, pset, base):
1947             if isinstance(pset, cms._Parameterizable):
1948                 for name in pset.parameters_().keys():
1949                     # skip whitelisted parameters
1950                     if name in self._whitelist:
1951                         continue
1952                     # if I use pset.parameters_().items() I get copies of the parameter values
1953                     # so I can't modify the nested pset
1954                     value = getattr(pset, name)
1955                     valueType = type(value)
1956                     if valueType in [cms.PSet, cms.untracked.PSet, cms.EDProducer]:
1957                         self.doIt(value,base+"."+name)
1958                     elif valueType in [cms.VPSet, cms.untracked.VPSet]:
1959                         for (i,ps) in enumerate(value): self.doIt(ps, "%s.%s[%d]"%(base,name,i) )
1960                     elif valueType in [cms.string, cms.untracked.string]:
1961                         if value.value() == self._paramSearch:
1962                             if self._verbose: print("set string process name %s.%s %s ==> %s"% (base, name, value, self._paramReplace))
1963                             setattr(pset, name,self._paramReplace)
1964                     elif valueType in [cms.VInputTag, cms.untracked.VInputTag]:
1965                         for (i,n) in enumerate(value):
1966                             if not isinstance(n, cms.InputTag):
1967                                 n=cms.InputTag(n)
1968                             if n.processName == self._paramSearch:
1969                                 # VInputTag can be declared as a list of strings, so ensure that n is formatted correctly
1970                                 if self._verbose:print("set process name %s.%s[%d] %s ==> %s " % (base, name, i, n, self._paramReplace))
1971                                 setattr(n,"processName",self._paramReplace)
1972                                 value[i]=n
1973                     elif valueType in [cms.vstring, cms.untracked.vstring]:
1974                         for (i,n) in enumerate(value):
1975                             if n==self._paramSearch:
1976                                 getattr(pset,name)[i]=self._paramReplace
1977                     elif valueType in [cms.InputTag, cms.untracked.InputTag]:
1978                         if value.processName == self._paramSearch:
1979                             if self._verbose: print("set process name %s.%s %s ==> %s " % (base, name, value, self._paramReplace))
1980                             setattr(getattr(pset, name),"processName",self._paramReplace)
1981
1982         def enter(self,visitee):
1983             label = ''
1984             try:
1985                 label = visitee.label()
1986             except AttributeError:
1987                 label = '<Module not in a Process>'
1988             except:
1989                 label = 'other execption'
1990             self.doIt(visitee, label)
1991
1992         def leave(self,visitee):
1993             pass
1994
1995     #visit a sequence to repalce all input tags
1996     def renameInputTagsInSequence(self,sequence,oldT="rawDataCollector",newT="rawDataRepacker"):
1997         print("Replacing all InputTag %s => %s"%(oldT,newT))
1998         from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag
1999         massSearchReplaceAnyInputTag(getattr(self.process,sequence),oldT,newT)
2000         loadMe='from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag'
2001         if not loadMe in self.additionalCommands:
2002             self.additionalCommands.append(loadMe)
2003         self.additionalCommands.append('massSearchReplaceAnyInputTag(process.%s,"%s","%s",False,True)'%(sequence,oldT,newT))
2004
2005     #change the process name used to address HLT results in any sequence
2006     def renameHLTprocessInSequence(self, sequence, proc=None, HLTprocess='HLT', verbosityLevel=1):
2007         if proc == None:
2008             proc = self._options.hltProcess if self._options.hltProcess else self.process.name_()
2009         if proc == HLTprocess:
2010             return
2011         # look up all module in sequence
2012         if verbosityLevel > 0:
2013             print("replacing %s process name - sequence %s will use '%s'" % (HLTprocess, sequence, proc))
2014         verboseVisit = (verbosityLevel > 1)
2015         getattr(self.process,sequence).visit(
2016             ConfigBuilder.MassSearchReplaceProcessNameVisitor(HLTprocess, proc, whitelist = ("subSystemFolder",), verbose = verboseVisit))
2017         if 'from Configuration.Applications.ConfigBuilder import ConfigBuilder' not in self.additionalCommands:
2018             self.additionalCommands.append('from Configuration.Applications.ConfigBuilder import ConfigBuilder')
2019         self.additionalCommands.append(
2020             'process.%s.visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor("%s", "%s", whitelist = ("subSystemFolder",), verbose = %s))'
2021             % (sequence, HLTprocess, proc, verboseVisit))
2022
2023     def expandMapping(self,seqList,mapping,index=None):
2024         maxLevel=30
2025         level=0
2026         while '@' in repr(seqList) and level<maxLevel:
2027             level+=1
2028             for specifiedCommand in seqList:
2029                 if specifiedCommand.startswith('@'):
2030                     location=specifiedCommand[1:]
2031                     if not location in mapping:
2032                         raise Exception("Impossible to map "+location+" from "+repr(mapping))
2033                     mappedTo=mapping[location]
2034                     if index!=None:
2035                         mappedTo=mappedTo[index]
2036                     seqList.remove(specifiedCommand)
2037                     seqList.extend(mappedTo.split('+'))
2038                     break;
2039         if level==maxLevel:
2040             raise Exception("Could not fully expand "+repr(seqList)+" from "+repr(mapping))
2041
2042     def prepare_DQM(self, stepSpec = 'DQMOffline'):
2043         # this one needs replacement
2044
2045         # any 'DQM' job should use DQMStore in non-legacy mode (but not HARVESTING)
2046         self.loadAndRemember("DQMServices/Core/DQMStoreNonLegacy_cff")
2047         _,_dqmSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.DQMOFFLINEDefaultCFF)
2048         sequenceList=_dqmSeq.split('+')
2049         postSequenceList=_dqmSeq.split('+')
2050         from DQMOffline.Configuration.autoDQM import autoDQM
2051         self.expandMapping(sequenceList,autoDQM,index=0)
2052         self.expandMapping(postSequenceList,autoDQM,index=1)
2053
2054         if len(set(sequenceList))!=len(sequenceList):
2055             sequenceList=list(set(sequenceList))
2056             print("Duplicate entries for DQM:, using",sequenceList)
2057
2058         pathName='dqmoffline_step'
2059         for (i,_sequence) in enumerate(sequenceList):
2060             if (i!=0):
2061                 pathName='dqmoffline_%d_step'%(i)
2062
2063             if 'HLT' in self.stepMap.keys() or self._options.hltProcess:
2064                 self.renameHLTprocessInSequence(_sequence)
2065
2066             setattr(self.process,pathName, cms.EndPath( getattr(self.process,_sequence ) ) )
2067             self.schedule.append(getattr(self.process,pathName))
2068
2069             if hasattr(self.process,"genstepfilter") and len(self.process.genstepfilter.triggerConditions):
2070                 #will get in the schedule, smoothly
2071                 getattr(self.process,pathName).insert(0,self.process.genstepfilter)
2072
2073
2074         pathName='dqmofflineOnPAT_step'
2075         for (i,_sequence) in enumerate(postSequenceList):
2076             #Fix needed to avoid duplication of sequences not defined in autoDQM or without a PostDQM
2077             if (sequenceList[i]==postSequenceList[i]):
2078                       continue
2079             if (i!=0):
2080                 pathName='dqmofflineOnPAT_%d_step'%(i)
2081
2082             setattr(self.process,pathName, cms.EndPath( getattr(self.process, _sequence ) ) )
2083             self.schedule.append(getattr(self.process,pathName))
2084
2085     def prepare_HARVESTING(self, stepSpec = None):
2086         """ Enrich the process with harvesting step """
2087         self.DQMSaverCFF='Configuration/StandardSequences/DQMSaver'+self._options.harvesting+'_cff'
2088         self.loadAndRemember(self.DQMSaverCFF)
2089
2090         harvestingConfig,sequence,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.HARVESTINGDefaultCFF)
2091
2092         # decide which HARVESTING paths to use
2093         harvestingList = sequence.split("+")
2094         from DQMOffline.Configuration.autoDQM import autoDQM
2095         from Validation.Configuration.autoValidation import autoValidation
2096         import copy
2097         combined_mapping = copy.deepcopy( autoDQM )
2098         combined_mapping.update( autoValidation )
2099         self.expandMapping(harvestingList,combined_mapping,index=-1)
2100
2101         if len(set(harvestingList))!=len(harvestingList):
2102             harvestingList=list(set(harvestingList))
2103             print("Duplicate entries for HARVESTING, using",harvestingList)
2104
2105         for name in harvestingList:
2106             if not name in harvestingConfig.__dict__:
2107                 print(name,"is not a possible harvesting type. Available are",harvestingConfig.__dict__.keys())
2108                 # trigger hard error, like for other sequence types
2109                 getattr(self.process, name)
2110                 continue
2111             harvestingstream = getattr(harvestingConfig,name)
2112             if isinstance(harvestingstream,cms.Path):
2113                 self.schedule.append(harvestingstream)
2114                 self.blacklist_paths.append(harvestingstream)
2115             if isinstance(harvestingstream,cms.Sequence):
2116                 setattr(self.process,name+"_step",cms.Path(harvestingstream))
2117                 self.schedule.append(getattr(self.process,name+"_step"))
2118
2119         # # NOTE: the "hltProcess" option currently does nothing in the HARVEST step
2120         # if self._options.hltProcess or ('HLT' in self.stepMap):
2121         #     pass
2122
2123         self.scheduleSequence('DQMSaver','dqmsave_step')
2124         return
2125
2126     def prepare_ALCAHARVEST(self, stepSpec = None):
2127         """ Enrich the process with AlCaHarvesting step """
2128         harvestingConfig = self.loadAndRemember(self.ALCAHARVESTDefaultCFF)
2129         sequence=stepSpec.split(".")[-1]
2130
2131         # decide which AlcaHARVESTING paths to use
2132         harvestingList = sequence.split("+")
2133
2134
2135
2136         from Configuration.AlCa.autoPCL import autoPCL
2137         self.expandMapping(harvestingList,autoPCL)
2138
2139         for name in harvestingConfig.__dict__:
2140             harvestingstream = getattr(harvestingConfig,name)
2141             if name in harvestingList and isinstance(harvestingstream,cms.Path):
2142                 self.schedule.append(harvestingstream)
2143                 if isinstance(getattr(harvestingConfig,"ALCAHARVEST" + name + "_dbOutput"), cms.VPSet) and \
2144                    isinstance(getattr(harvestingConfig,"ALCAHARVEST" + name + "_metadata"), cms.VPSet):
2145                     self.executeAndRemember("process.PoolDBOutputService.toPut.extend(process.ALCAHARVEST" + name + "_dbOutput)")
2146                     self.executeAndRemember("process.pclMetadataWriter.recordsToMap.extend(process.ALCAHARVEST" + name + "_metadata)")
2147                 else:
2148                     self.executeAndRemember("process.PoolDBOutputService.toPut.append(process.ALCAHARVEST" + name + "_dbOutput)")
2149                     self.executeAndRemember("process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVEST" + name + "_metadata)")
2150                 harvestingList.remove(name)
2151         # append the common part at the end of the sequence
2152         lastStep = getattr(harvestingConfig,"ALCAHARVESTDQMSaveAndMetadataWriter")
2153         self.schedule.append(lastStep)
2154
2155         if len(harvestingList) != 0 and 'dummyHarvesting' not in harvestingList :
2156             print("The following harvesting could not be found : ", harvestingList)
2157             raise Exception("The following harvesting could not be found : "+str(harvestingList))
2158
2159
2160
2161     def prepare_ENDJOB(self, stepSpec = 'endOfProcess'):
2162         _,_endjobSeq,_=self.loadDefaultOrSpecifiedCFF(stepSpec,self.ENDJOBDefaultCFF)
2163         self.scheduleSequenceAtEnd(_endjobSeq,'endjob_step')
2164         return
2165
2166     def finalizeFastSimHLT(self):
2167         self.process.reconstruction = cms.Path(self.process.reconstructionWithFamos)
2168         self.schedule.append(self.process.reconstruction)
2169
2170
2171     def build_production_info(self, evt_type, evtnumber):
2172         """ Add useful info for the production. """
2173         self.process.configurationMetadata=cms.untracked.PSet\
2174                                             (version=cms.untracked.string("$Revision: 1.19 $"),
2175                                              name=cms.untracked.string("Applications"),
2176                                              annotation=cms.untracked.string(evt_type+ " nevts:"+str(evtnumber))
2177                                              )
2178
2179         self.addedObjects.append(("Production Info","configurationMetadata"))
2180
2181
2182     def create_process(self):
2183         self.pythonCfgCode =  "# Auto generated configuration file\n"
2184         self.pythonCfgCode += "# using: \n# "+__version__[1:-1]+"\n# "+__source__[1:-1]+'\n'
2185         self.pythonCfgCode += "# with command line options: "+self._options.arguments+'\n'
2186         self.pythonCfgCode += "import FWCore.ParameterSet.Config as cms\n\n"
2187
2188         # now set up the modifies
2189         modifiers=[]
2190         modifierStrings=[]
2191         modifierImports=[]
2192
2193         if hasattr(self._options,"era") and self._options.era :
2194         # Multiple eras can be specified in a comma seperated list
2195             from Configuration.StandardSequences.Eras import eras
2196             for requestedEra in self._options.era.split(",") :
2197                 modifierStrings.append(requestedEra)
2198                 modifierImports.append(eras.pythonCfgLines[requestedEra])
2199                 modifiers.append(getattr(eras,requestedEra))
2200
2201
2202         if hasattr(self._options,"procModifiers") and self._options.procModifiers:
2203             import importlib
2204             thingsImported=[]
2205             for c in self._options.procModifiers:
2206                 thingsImported.extend(c.split(","))
2207             for pm in thingsImported:
2208                 modifierStrings.append(pm)
2209                 modifierImports.append('from Configuration.ProcessModifiers.'+pm+'_cff import '+pm)
2210                 modifiers.append(getattr(importlib.import_module('Configuration.ProcessModifiers.'+pm+'_cff'),pm))
2211
2212         self.pythonCfgCode += '\n'.join(modifierImports)+'\n\n'
2213         self.pythonCfgCode += "process = cms.Process('"+self._options.name+"'" # Start of the line, finished after the loop
2214
2215
2216         if len(modifierStrings)>0:
2217             self.pythonCfgCode+= ','+','.join(modifierStrings)
2218         self.pythonCfgCode+=')\n\n'
2219
2220         #yes, the cfg code gets out of sync here if a process is passed in. That could be fixed in the future
2221         #assuming there is some way for the fwk to get the list of modifiers (and their stringified name)
2222         if self.process == None:
2223             if len(modifiers)>0:
2224                 self.process = cms.Process(self._options.name,*modifiers)
2225             else:
2226                 self.process = cms.Process(self._options.name)
2227
2228
2229
2230
2231     def prepare(self, doChecking = False):
2232         """ Prepare the configuration string and add missing pieces."""
2233
2234         self.loadAndRemember(self.EVTCONTDefaultCFF)  #load the event contents regardless
2235         self.addMaxEvents()
2236         if self.with_input:
2237             self.addSource()
2238         self.addStandardSequences()
2239         ##adding standard sequences might change the inputEventContent option and therefore needs to be finalized after
2240         self.completeInputCommand()
2241         self.addConditions()
2242
2243
2244         outputModuleCfgCode=""
2245         if not 'HARVESTING' in self.stepMap.keys() and not 'ALCAHARVEST' in self.stepMap.keys() and not 'ALCAOUTPUT' in self.stepMap.keys() and self.with_output:
2246             outputModuleCfgCode=self.addOutput()
2247
2248         self.addCommon()
2249
2250         self.pythonCfgCode += "# import of standard configurations\n"
2251         for module in self.imports:
2252             self.pythonCfgCode += ("process.load('"+module+"')\n")
2253
2254         # production info
2255         if not hasattr(self.process,"configurationMetadata"):
2256             self.build_production_info(self._options.evt_type, self._options.number)
2257         else:
2258             #the PSet was added via a load
2259             self.addedObjects.append(("Production Info","configurationMetadata"))
2260
2261         self.pythonCfgCode +="\n"
2262         for comment,object in self.addedObjects:
2263             if comment!="":
2264                 self.pythonCfgCode += "\n# "+comment+"\n"
2265             self.pythonCfgCode += dumpPython(self.process,object)
2266
2267         # dump the output definition
2268         self.pythonCfgCode += "\n# Output definition\n"
2269         self.pythonCfgCode += outputModuleCfgCode
2270
2271         # dump all additional outputs (e.g. alca or skim streams)
2272         self.pythonCfgCode += "\n# Additional output definition\n"
2273         #I do not understand why the keys are not normally ordered.
2274         nl=sorted(self.additionalOutputs.keys())
2275         for name in nl:
2276             output = self.additionalOutputs[name]
2277             self.pythonCfgCode += "process.%s = %s" %(name, output.dumpPython())
2278             tmpOut = cms.EndPath(output)
2279             setattr(self.process,name+'OutPath',tmpOut)
2280             self.schedule.append(tmpOut)
2281
2282         # dump all additional commands
2283         self.pythonCfgCode += "\n# Other statements\n"
2284         for command in self.additionalCommands:
2285             self.pythonCfgCode += command + "\n"
2286
2287         #comma separated list of objects that deserve to be inlined in the configuration (typically from a modified config deep down)
2288         for object in self._options.inlineObjets.split(','):
2289             if not object:
2290                 continue
2291             if not hasattr(self.process,object):
2292                 print('cannot inline -'+object+'- : not known')
2293             else:
2294                 self.pythonCfgCode +='\n'
2295                 self.pythonCfgCode +=dumpPython(self.process,object)
2296
2297         if self._options.pileup=='HiMixEmbGEN':
2298             self.pythonCfgCode += "\nprocess.generator.embeddingMode=cms.int32(1)\n"
2299
2300         # dump all paths
2301         self.pythonCfgCode += "\n# Path and EndPath definitions\n"
2302         for path in self.process.paths:
2303             if getattr(self.process,path) not in self.blacklist_paths:
2304                 self.pythonCfgCode += dumpPython(self.process,path)
2305
2306         for endpath in self.process.endpaths:
2307             if getattr(self.process,endpath) not in self.blacklist_paths:
2308                 self.pythonCfgCode += dumpPython(self.process,endpath)
2309
2310         # dump the schedule
2311         self.pythonCfgCode += "\n# Schedule definition\n"
2312
2313         # handling of the schedule
2314         pathNames = ['process.'+p.label_() for p in self.schedule]
2315         if self.process.schedule == None:
2316             self.process.schedule = cms.Schedule()
2317             for item in self.schedule:
2318                 self.process.schedule.append(item)
2319             result = 'process.schedule = cms.Schedule('+','.join(pathNames)+')\n'
2320         else:
2321             if not isinstance(self.scheduleIndexOfFirstHLTPath, int):
2322                 raise Exception('the schedule was imported from a cff in HLTrigger.Configuration, but the final index of the first HLT path is undefined')
2323
2324             for index, item in enumerate(self.schedule):
2325                 if index < self.scheduleIndexOfFirstHLTPath:
2326                     self.process.schedule.insert(index, item)
2327                 else:
2328                     self.process.schedule.append(item)
2329
2330             result = "# process.schedule imported from cff in HLTrigger.Configuration\n"
2331             for index, item in enumerate(pathNames[:self.scheduleIndexOfFirstHLTPath]):
2332                 result += 'process.schedule.insert('+str(index)+', '+item+')\n'
2333             if self.scheduleIndexOfFirstHLTPath < len(pathNames):
2334                 result += 'process.schedule.extend(['+','.join(pathNames[self.scheduleIndexOfFirstHLTPath:])+'])\n'
2335
2336         self.pythonCfgCode += result
2337
2338         for labelToAssociate in self.labelsToAssociate:
2339             self.process.schedule.associate(getattr(self.process, labelToAssociate))
2340             self.pythonCfgCode += 'process.schedule.associate(process.' + labelToAssociate + ')\n'
2341
2342         from PhysicsTools.PatAlgos.tools.helpers import associatePatAlgosToolsTask
2343         associatePatAlgosToolsTask(self.process)
2344         self.pythonCfgCode+="from PhysicsTools.PatAlgos.tools.helpers import associatePatAlgosToolsTask\n"
2345         self.pythonCfgCode+="associatePatAlgosToolsTask(process)\n"
2346
2347         overrideThreads = (self._options.nThreads != "1")
2348         overrideConcurrentLumis = (self._options.nConcurrentLumis != defaultOptions.nConcurrentLumis)
2349         overrideConcurrentIOVs = (self._options.nConcurrentIOVs != defaultOptions.nConcurrentIOVs)
2350
2351         if overrideThreads or overrideConcurrentLumis or overrideConcurrentIOVs:
2352             self.pythonCfgCode +="\n"
2353             self.pythonCfgCode +="#Setup FWK for multithreaded\n"
2354             if overrideThreads:
2355                 self.pythonCfgCode +="process.options.numberOfThreads = "+self._options.nThreads+"\n"
2356                 self.pythonCfgCode +="process.options.numberOfStreams = "+self._options.nStreams+"\n"
2357                 self.process.options.numberOfThreads = int(self._options.nThreads)
2358                 self.process.options.numberOfStreams = int(self._options.nStreams)
2359             if overrideConcurrentLumis:
2360                 self.pythonCfgCode +="process.options.numberOfConcurrentLuminosityBlocks = "+self._options.nConcurrentLumis+"\n"
2361                 self.process.options.numberOfConcurrentLuminosityBlocks = int(self._options.nConcurrentLumis)
2362             if overrideConcurrentIOVs:
2363                 self.pythonCfgCode +="process.options.eventSetup.numberOfConcurrentIOVs = "+self._options.nConcurrentIOVs+"\n"
2364                 self.process.options.eventSetup.numberOfConcurrentIOVs = int(self._options.nConcurrentIOVs)
2365
2366         if self._options.accelerators is not None:
2367             accelerators = self._options.accelerators.split(',')
2368             self.pythonCfgCode += "\n"
2369             self.pythonCfgCode += "# Enable only these accelerator backends\n"
2370             self.pythonCfgCode += "process.load('Configuration.StandardSequences.Accelerators_cff')\n"
2371             self.pythonCfgCode += "process.options.accelerators = ['" + "', '".join(accelerators) + "']\n"
2372             self.process.load('Configuration.StandardSequences.Accelerators_cff')
2373             self.process.options.accelerators = accelerators
2374
2375         #repacked version
2376         if self._options.isRepacked:
2377             self.pythonCfgCode +="\n"
2378             self.pythonCfgCode +="from Configuration.Applications.ConfigBuilder import MassReplaceInputTag\n"
2379             self.pythonCfgCode +="MassReplaceInputTag(process, new=\"rawDataMapperByLabel\", old=\"rawDataCollector\")\n"
2380             MassReplaceInputTag(self.process, new="rawDataMapperByLabel", old="rawDataCollector")
2381
2382         # special treatment in case of production filter sequence 2/2
2383         if self.productionFilterSequence and not (self._options.pileup=='HiMixEmbGEN'):
2384             self.pythonCfgCode +='# filter all path with the production filter sequence\n'
2385             self.pythonCfgCode +='for path in process.paths:\n'
2386             if len(self.conditionalPaths):
2387                 self.pythonCfgCode +='\tif not path in %s: continue\n'%str(self.conditionalPaths)
2388             if len(self.excludedPaths):
2389                 self.pythonCfgCode +='\tif path in %s: continue\n'%str(self.excludedPaths)
2390             self.pythonCfgCode +='\tgetattr(process,path).insert(0, process.%s)\n'%(self.productionFilterSequence,)
2391             pfs = getattr(self.process,self.productionFilterSequence)
2392             for path in self.process.paths:
2393                 if not path in self.conditionalPaths: continue
2394                 if path in self.excludedPaths: continue
2395                 getattr(self.process,path).insert(0, pfs)
2396
2397
2398         # dump customise fragment
2399         self.pythonCfgCode += self.addCustomise()
2400
2401         if self._options.runUnscheduled:
2402             print("--runUnscheduled is deprecated and not necessary anymore, and will be removed soon. Please update your command line.")
2403         # Keep the "unscheduled customise functions" separate for now,
2404         # there are customize functions given by users (in our unit
2405         # tests) that need to be run before the "unscheduled customise
2406         # functions"
2407         self.pythonCfgCode += self.addCustomise(1)
2408
2409         self.pythonCfgCode += self.addCustomiseCmdLine()
2410
2411         if hasattr(self.process,"logErrorHarvester"):
2412             #configure logErrorHarvester to wait for same EDProducers to finish as the OutputModules
2413             self.pythonCfgCode +="\n#Have logErrorHarvester wait for the same EDProducers to finish as those providing data for the OutputModule\n"
2414             self.pythonCfgCode +="from FWCore.Modules.logErrorHarvester_cff import customiseLogErrorHarvesterUsingOutputCommands\n"
2415             self.pythonCfgCode +="process = customiseLogErrorHarvesterUsingOutputCommands(process)\n"
2416             from FWCore.Modules.logErrorHarvester_cff import customiseLogErrorHarvesterUsingOutputCommands
2417             self.process = customiseLogErrorHarvesterUsingOutputCommands(self.process)
2418
2419         # Temporary hack to put the early delete customization after
2420         # everything else
2421         #
2422         # FIXME: remove when no longer needed
2423         self.pythonCfgCode += "\n# Add early deletion of temporary data products to reduce peak memory need\n"
2424         self.pythonCfgCode += "from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete\n"
2425         self.pythonCfgCode += "process = customiseEarlyDelete(process)\n"
2426         self.pythonCfgCode += "# End adding early deletion\n"
2427         from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete
2428         self.process = customiseEarlyDelete(self.process)
2429
2430         imports = cms.specialImportRegistry.getSpecialImports()
2431         if len(imports) > 0:
2432             #need to inject this at the top
2433             index = self.pythonCfgCode.find("import FWCore.ParameterSet.Config")
2434             #now find the end of line
2435             index = self.pythonCfgCode.find("\n",index)
2436             self.pythonCfgCode = self.pythonCfgCode[:index]+ "\n" + "\n".join(imports)+"\n" +self.pythonCfgCode[index:]
2437
2438
2439         # make the .io file
2440
2441         if self._options.io:
2442             #io=open(self._options.python_filename.replace('.py','.io'),'w')
2443             if not self._options.io.endswith('.io'): self._option.io+='.io'
2444             io=open(self._options.io,'w')
2445             ioJson={}
2446             if hasattr(self.process.source,"fileNames"):
2447                 if len(self.process.source.fileNames.value()):
2448                     ioJson['primary']=self.process.source.fileNames.value()
2449             if hasattr(self.process.source,"secondaryFileNames"):
2450                 if len(self.process.source.secondaryFileNames.value()):
2451                     ioJson['secondary']=self.process.source.secondaryFileNames.value()
2452             if self._options.pileup_input and (self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:')):
2453                 ioJson['pileup']=self._options.pileup_input[4:]
2454             for (o,om) in self.process.outputModules_().items():
2455                 ioJson[o]=om.fileName.value()
2456             ioJson['GT']=self.process.GlobalTag.globaltag.value()
2457             if self.productionFilterSequence:
2458                 ioJson['filter']=self.productionFilterSequence
2459             import json
2460             io.write(json.dumps(ioJson))
2461         return
2462