Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-04-06 12:03:31

0001 #! /usr/bin/env python3
0002 
0003 __version__ = "$Revision: 1.19 $"
0004 __source__ = "$Source: /local/reps/CMSSW/CMSSW/Configuration/Applications/python/ConfigBuilder.py,v $"
0005 
0006 import FWCore.ParameterSet.Config as cms
0007 from FWCore.ParameterSet.Modules import _Module
0008 # The following import is provided for backward compatibility reasons.
0009 # The function used to be defined in this file.
0010 from FWCore.ParameterSet.MassReplace import massReplaceInputTag as MassReplaceInputTag
0011 
0012 import hashlib
0013 import sys
0014 import re
0015 import collections
0016 from subprocess import Popen,PIPE
0017 import FWCore.ParameterSet.DictTypes as DictTypes
0018 from FWCore.ParameterSet.OrderedSet import OrderedSet
0019 class Options:
0020     pass
0021 
0022 # the canonical defaults
0023 defaultOptions = Options()
0024 defaultOptions.datamix = 'DataOnSim'
0025 defaultOptions.isMC=False
0026 defaultOptions.isData=True
0027 defaultOptions.step=''
0028 defaultOptions.pileup='NoPileUp'
0029 defaultOptions.pileup_input = None
0030 defaultOptions.pileup_dasoption = ''
0031 defaultOptions.geometry = 'SimDB'
0032 defaultOptions.geometryExtendedOptions = ['ExtendedGFlash','Extended','NoCastor']
0033 defaultOptions.magField = ''
0034 defaultOptions.conditions = None
0035 defaultOptions.scenarioOptions=['pp','cosmics','nocoll','HeavyIons']
0036 defaultOptions.harvesting= 'AtRunEnd'
0037 defaultOptions.gflash = False
0038 defaultOptions.number = -1
0039 defaultOptions.number_out = None
0040 defaultOptions.arguments = ""
0041 defaultOptions.name = "NO NAME GIVEN"
0042 defaultOptions.evt_type = ""
0043 defaultOptions.filein = ""
0044 defaultOptions.dasquery=""
0045 defaultOptions.dasoption=""
0046 defaultOptions.secondfilein = ""
0047 defaultOptions.customisation_file = []
0048 defaultOptions.customisation_file_unsch = []
0049 defaultOptions.customise_commands = ""
0050 defaultOptions.inline_custom=False
0051 defaultOptions.particleTable = 'pythiapdt'
0052 defaultOptions.particleTableList = ['pythiapdt','pdt']
0053 defaultOptions.dirin = ''
0054 defaultOptions.dirout = ''
0055 defaultOptions.filetype = 'EDM'
0056 defaultOptions.fileout = 'output.root'
0057 defaultOptions.filtername = ''
0058 defaultOptions.lazy_download = False
0059 defaultOptions.custom_conditions = ''
0060 defaultOptions.hltProcess = ''
0061 defaultOptions.eventcontent = None
0062 defaultOptions.datatier = None
0063 defaultOptions.inlineEventContent = True
0064 defaultOptions.inlineObjects =''
0065 defaultOptions.hideGen=False
0066 from Configuration.StandardSequences.VtxSmeared import VtxSmearedDefaultKey,VtxSmearedHIDefaultKey
0067 defaultOptions.beamspot=None
0068 defaultOptions.outputDefinition =''
0069 defaultOptions.inputCommands = None
0070 defaultOptions.outputCommands = None
0071 defaultOptions.inputEventContent = ''
0072 defaultOptions.dropDescendant = False
0073 defaultOptions.relval = None
0074 defaultOptions.prefix = None
0075 defaultOptions.profile = None
0076 defaultOptions.heap_profile = None
0077 defaultOptions.maxmem_profile = None
0078 defaultOptions.isRepacked = False
0079 defaultOptions.restoreRNDSeeds = False
0080 defaultOptions.donotDropOnInput = ''
0081 defaultOptions.python_filename =''
0082 defaultOptions.io=None
0083 defaultOptions.lumiToProcess=None
0084 defaultOptions.fast=False
0085 defaultOptions.runsAndWeightsForMC = None
0086 defaultOptions.runsScenarioForMC = None
0087 defaultOptions.runsAndWeightsForMCIntegerWeights = None
0088 defaultOptions.runsScenarioForMCIntegerWeights = None
0089 defaultOptions.runUnscheduled = False
0090 defaultOptions.timeoutOutput = False
0091 defaultOptions.nThreads = 1
0092 defaultOptions.nStreams = 0
0093 defaultOptions.nConcurrentLumis = 0
0094 defaultOptions.nConcurrentIOVs = 0
0095 defaultOptions.accelerators = None
0096 
0097 # some helper routines
0098 def dumpPython(process,name):
0099     theObject = getattr(process,name)
0100     if isinstance(theObject,cms.Path) or isinstance(theObject,cms.EndPath) or isinstance(theObject,cms.Sequence):
0101         return "process."+name+" = " + theObject.dumpPython()
0102     elif isinstance(theObject,_Module) or isinstance(theObject,cms.ESProducer):
0103         return "process."+name+" = " + theObject.dumpPython()+"\n"
0104     else:
0105         return "process."+name+" = " + theObject.dumpPython()+"\n"
0106 def filesFromList(fileName,s=None):
0107     import os
0108     import FWCore.ParameterSet.Config as cms
0109     prim=[]
0110     sec=[]
0111     for line in open(fileName,'r'):
0112         if line.count(".root")>=2:
0113             #two files solution...
0114             entries=line.replace("\n","").split()
0115             prim.append(entries[0])
0116             sec.append(entries[1])
0117         elif (line.find(".root")!=-1):
0118             entry=line.replace("\n","")
0119             prim.append(entry)
0120     # remove any duplicates but keep the order
0121     file_seen = set()
0122     prim = [f for f in prim if not (f in file_seen or file_seen.add(f))]
0123     file_seen = set()
0124     sec = [f for f in sec if not (f in file_seen or file_seen.add(f))]
0125     if s:
0126         if not hasattr(s,"fileNames"):
0127             s.fileNames=cms.untracked.vstring(prim)
0128         else:
0129             s.fileNames.extend(prim)
0130         if len(sec)!=0:
0131             if not hasattr(s,"secondaryFileNames"):
0132                 s.secondaryFileNames=cms.untracked.vstring(sec)
0133             else:
0134                 s.secondaryFileNames.extend(sec)
0135     print("found files: ",prim)
0136     if len(prim)==0:
0137         raise Exception("There are not files in input from the file list")
0138     if len(sec)!=0:
0139         print("found parent files:",sec)
0140     return (prim,sec)
0141 
0142 def filesFromDASQuery(query,option="",s=None):
0143     import os,time
0144     import FWCore.ParameterSet.Config as cms
0145     prim=[]
0146     sec=[]
0147     print("the query is",query)
0148     eC=5
0149     count=0
0150     while eC!=0 and count<3:
0151         if count!=0:
0152             print('Sleeping, then retrying DAS')
0153             time.sleep(100)
0154         p = Popen('dasgoclient %s --query "%s"'%(option,query), stdout=PIPE,shell=True, universal_newlines=True)
0155         pipe=p.stdout.read()
0156         tupleP = os.waitpid(p.pid, 0)
0157         eC=tupleP[1]
0158         count=count+1
0159     if eC==0:
0160         print("DAS succeeded after",count,"attempts",eC)
0161     else:
0162         print("DAS failed 3 times- I give up")
0163     for line in pipe.split('\n'):
0164         if line.count(".root")>=2:
0165             #two files solution...
0166             entries=line.replace("\n","").split()
0167             prim.append(entries[0])
0168             sec.append(entries[1])
0169         elif (line.find(".root")!=-1):
0170             entry=line.replace("\n","")
0171             prim.append(entry)
0172     # remove any duplicates
0173     prim = sorted(list(set(prim)))
0174     sec = sorted(list(set(sec)))
0175     if s:
0176         if not hasattr(s,"fileNames"):
0177             s.fileNames=cms.untracked.vstring(prim)
0178         else:
0179             s.fileNames.extend(prim)
0180         if len(sec)!=0:
0181             if not hasattr(s,"secondaryFileNames"):
0182                 s.secondaryFileNames=cms.untracked.vstring(sec)
0183             else:
0184                 s.secondaryFileNames.extend(sec)
0185     print("found files: ",prim)
0186     if len(sec)!=0:
0187         print("found parent files:",sec)
0188     return (prim,sec)
0189 
0190 def anyOf(listOfKeys,dict,opt=None):
0191     for k in listOfKeys:
0192         if k in dict:
0193             toReturn=dict[k]
0194             dict.pop(k)
0195             return toReturn
0196     if opt!=None:
0197         return opt
0198     else:
0199         raise Exception("any of "+','.join(listOfKeys)+" are mandatory entries of --output options")
0200 
0201 class ConfigBuilder(object):
0202     """The main building routines """
0203 
0204     def __init__(self, options, process = None, with_output = False, with_input = False ):
0205         """options taken from old cmsDriver and optparse """
0206 
0207         options.outfile_name = options.dirout+options.fileout
0208 
0209         self._options = options
0210 
0211         if self._options.isData and options.isMC:
0212             raise Exception("ERROR: You may specify only --data or --mc, not both")
0213         #if not self._options.conditions:
0214         #        raise Exception("ERROR: No conditions given!\nPlease specify conditions. E.g. via --conditions=IDEAL_30X::All")
0215 
0216         # check that MEtoEDMConverter (running in ENDJOB) and DQMIO don't run in the same job
0217         if 'ENDJOB' in self._options.step:
0218             if  (hasattr(self._options,"outputDefinition") and \
0219                 self._options.outputDefinition != '' and \
0220                 any(anyOf(['t','tier','dataTier'],outdic) == 'DQMIO' for outdic in eval(self._options.outputDefinition))) or \
0221                 (hasattr(self._options,"datatier") and \
0222                 self._options.datatier and \
0223                 'DQMIO' in self._options.datatier):
0224                 print("removing ENDJOB from steps since not compatible with DQMIO dataTier")
0225                 self._options.step=self._options.step.replace(',ENDJOB','')
0226 
0227 
0228 
0229         # what steps are provided by this class?
0230         stepList = [re.sub(r'^prepare_', '', methodName) for methodName in ConfigBuilder.__dict__ if methodName.startswith('prepare_')]
0231         self.stepMap={}
0232         self.stepKeys=[]
0233         for step in self._options.step.split(","):
0234             if step=='': continue
0235             stepParts = step.split(":")
0236             stepName = stepParts[0]
0237             if stepName not in stepList and not stepName.startswith('re'):
0238                 raise ValueError("Step {} unknown. Available are {}".format( stepName , sorted(stepList)))
0239             if len(stepParts)==1:
0240                 self.stepMap[stepName]=""
0241             elif len(stepParts)==2:
0242                 self.stepMap[stepName]=stepParts[1].split('+')
0243             elif len(stepParts)==3:
0244                 self.stepMap[stepName]=(stepParts[2].split('+'),stepParts[1])
0245             else:
0246                 raise ValueError(f"Step definition {step} invalid")
0247             self.stepKeys.append(stepName)
0248 
0249         #print(f"map of steps is: {self.stepMap}")
0250 
0251         self.with_output = with_output
0252         self.process=process
0253 
0254         if hasattr(self._options,"no_output_flag") and self._options.no_output_flag:
0255             self.with_output = False
0256         self.with_input = with_input
0257         self.imports = []
0258         self.create_process()
0259         self.define_Configs()
0260         self.schedule = list()
0261         self.scheduleIndexOfFirstHLTPath = None
0262 
0263         # we are doing three things here:
0264         # creating a process to catch errors
0265         # building the code to re-create the process
0266 
0267         self.additionalCommands = []
0268         # TODO: maybe a list of to be dumped objects would help as well
0269         self.blacklist_paths = []
0270         self.addedObjects = []
0271         self.additionalOutputs = {}
0272 
0273         self.productionFilterSequence = None
0274         self.labelsToAssociate=[]
0275         self.nextScheduleIsConditional=False
0276         self.conditionalPaths=[]
0277         self.excludedPaths=[]
0278 
0279     def profileOptions(self):
0280         """
0281         addIgProfService
0282         Function to add the igprof profile service so that you can dump in the middle
0283         of the run.
0284         """
0285         profileOpts = self._options.profile.split(':')
0286         profilerStart = 1
0287         profilerInterval = 100
0288         profilerFormat = None
0289         profilerJobFormat = None
0290 
0291         if len(profileOpts):
0292             #type, given as first argument is unused here
0293             profileOpts.pop(0)
0294         if len(profileOpts):
0295             startEvent = profileOpts.pop(0)
0296             if not startEvent.isdigit():
0297                 raise Exception("%s is not a number" % startEvent)
0298             profilerStart = int(startEvent)
0299         if len(profileOpts):
0300             eventInterval = profileOpts.pop(0)
0301             if not eventInterval.isdigit():
0302                 raise Exception("%s is not a number" % eventInterval)
0303             profilerInterval = int(eventInterval)
0304         if len(profileOpts):
0305             profilerFormat = profileOpts.pop(0)
0306 
0307 
0308         if not profilerFormat:
0309             profilerFormat = "%s___%s___%%I.gz" % (
0310                 self._options.evt_type.replace("_cfi", ""),
0311                 hashlib.md5(
0312                     (str(self._options.step) + str(self._options.pileup) + str(self._options.conditions) +
0313                     str(self._options.datatier) + str(self._options.profileTypeLabel)).encode('utf-8')
0314                 ).hexdigest()
0315             )
0316         if not profilerJobFormat and profilerFormat.endswith(".gz"):
0317             profilerJobFormat = profilerFormat.replace(".gz", "_EndOfJob.gz")
0318         elif not profilerJobFormat:
0319             profilerJobFormat = profilerFormat + "_EndOfJob.gz"
0320 
0321         return (profilerStart,profilerInterval,profilerFormat,profilerJobFormat)
0322 
0323     def heapProfileOptions(self):
0324         """
0325         addJeProfService
0326         Function to add the jemalloc heap  profile service so that you can dump in the middle
0327         of the run.
0328         """
0329         profileOpts = []
0330         profilerStart = 1
0331         profilerInterval = 100
0332         profilerFormat = "jeprof_%s.heap"
0333         profilerJobFormat = None
0334 
0335 
0336         if not profilerJobFormat and profilerFormat.endswith(".heap"):
0337             profilerJobFormat = profilerFormat.replace(".heap", "_EndOfJob.heap")
0338         elif not profilerJobFormat:
0339             profilerJobFormat = profilerFormat + "_EndOfJob.heap"
0340 
0341         return (profilerStart,profilerInterval,profilerFormat,profilerJobFormat)
0342 
0343     def load(self,includeFile):
0344         includeFile = includeFile.replace('/','.')
0345         self.process.load(includeFile)
0346         return sys.modules[includeFile]
0347 
0348     def loadAndRemember(self, includeFile):
0349         """helper routine to load am memorize imports"""
0350         # we could make the imports a on-the-fly data method of the process instance itself
0351         # not sure if the latter is a good idea
0352         includeFile = includeFile.replace('/','.')
0353         self.imports.append(includeFile)
0354         self.process.load(includeFile)
0355         return sys.modules[includeFile]
0356 
0357     def executeAndRemember(self, command):
0358         """helper routine to remember replace statements"""
0359         self.additionalCommands.append(command)
0360         if not command.strip().startswith("#"):
0361         # substitute: process.foo = process.bar -> self.process.foo = self.process.bar
0362             import re
0363             exec(re.sub(r"([^a-zA-Z_0-9]|^)(process)([^a-zA-Z_0-9])",r"\1self.process\3",command))
0364             #exec(command.replace("process.","self.process."))
0365 
0366     def addCommon(self):
0367         if 'HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys():
0368             self.process.options.Rethrow = ['ProductNotFound']
0369             self.process.options.fileMode = 'FULLMERGE'
0370 
0371         self.addedObjects.append(("","options"))
0372 
0373         if self._options.lazy_download:
0374             self.process.AdaptorConfig = cms.Service("AdaptorConfig",
0375                                                      stats = cms.untracked.bool(True),
0376                                                      enable = cms.untracked.bool(True),
0377                                                      cacheHint = cms.untracked.string("lazy-download"),
0378                                                      readHint = cms.untracked.string("read-ahead-buffered")
0379                                                      )
0380             self.addedObjects.append(("Setup lazy download","AdaptorConfig"))
0381 
0382         #self.process.cmsDriverCommand = cms.untracked.PSet( command=cms.untracked.string('cmsDriver.py '+self._options.arguments) )
0383         #self.addedObjects.append(("what cmsDriver command was used","cmsDriverCommand"))
0384 
0385         if self._options.profile:
0386             (start, interval, eventFormat, jobFormat)=self.profileOptions()
0387             self.process.IgProfService = cms.Service("IgProfService",
0388                                                      reportFirstEvent            = cms.untracked.int32(start),
0389                                                      reportEventInterval         = cms.untracked.int32(interval),
0390                                                      reportToFileAtPostEvent     = cms.untracked.string("| gzip -c > %s"%(eventFormat)),
0391                                                      reportToFileAtPostEndJob    = cms.untracked.string("| gzip -c > %s"%(jobFormat)))
0392             self.addedObjects.append(("Setup IGProf Service for profiling","IgProfService"))
0393 
0394         if self._options.heap_profile:
0395             (start, interval, eventFormat, jobFormat)=self.heapProfileOptions()
0396             self.process.JeProfService = cms.Service("JeProfService",
0397                                                      reportFirstEvent            = cms.untracked.int32(start),
0398                                                      reportEventInterval         = cms.untracked.int32(interval),
0399                                                      reportToFileAtPostEvent     = cms.untracked.string("%s"%(eventFormat)),
0400                                                      reportToFileAtPostEndJob    = cms.untracked.string("%s"%(jobFormat)))
0401             self.addedObjects.append(("Setup JeProf Service for heap profiling","JeProfService"))
0402 
0403     def addMaxEvents(self):
0404         """Here we decide how many evts will be processed"""
0405         self.process.maxEvents.input = self._options.number
0406         if self._options.number_out:
0407             self.process.maxEvents.output = self._options.number_out
0408         self.addedObjects.append(("","maxEvents"))
0409 
0410     def addSource(self):
0411         """Here the source is built. Priority: file, generator"""
0412         self.addedObjects.append(("Input source","source"))
0413 
0414         def filesFromOption(self):
0415             for entry in self._options.filein.split(','):
0416                 print("entry",entry)
0417                 if entry.startswith("filelist:"):
0418                     filesFromList(entry[9:],self.process.source)
0419                 elif entry.startswith("dbs:") or entry.startswith("das:"):
0420                     filesFromDASQuery('file dataset = %s'%(entry[4:]),self._options.dasoption,self.process.source)
0421                 else:
0422                     self.process.source.fileNames.append(self._options.dirin+entry)
0423             if self._options.secondfilein:
0424                 if not hasattr(self.process.source,"secondaryFileNames"):
0425                     raise Exception("--secondfilein not compatible with "+self._options.filetype+"input type")
0426                 for entry in self._options.secondfilein.split(','):
0427                     print("entry",entry)
0428                     if entry.startswith("filelist:"):
0429                         self.process.source.secondaryFileNames.extend((filesFromList(entry[9:]))[0])
0430                     elif entry.startswith("dbs:") or entry.startswith("das:"):
0431                         self.process.source.secondaryFileNames.extend((filesFromDASQuery('file dataset = %s'%(entry[4:]),self._options.dasoption))[0])
0432                     else:
0433                         self.process.source.secondaryFileNames.append(self._options.dirin+entry)
0434 
0435         if self._options.filein or self._options.dasquery:
0436             if self._options.filetype == "EDM":
0437                 self.process.source=cms.Source("PoolSource",
0438                                                fileNames = cms.untracked.vstring(),
0439                                                secondaryFileNames= cms.untracked.vstring())
0440                 filesFromOption(self)
0441             elif self._options.filetype == "DAT":
0442                 self.process.source=cms.Source("NewEventStreamFileReader",fileNames = cms.untracked.vstring())
0443                 filesFromOption(self)
0444             elif self._options.filetype == "LHE":
0445                 self.process.source=cms.Source("LHESource", fileNames = cms.untracked.vstring())
0446                 if self._options.filein.startswith("lhe:"):
0447                     #list the article directory automatically
0448                     args=self._options.filein.split(':')
0449                     article=args[1]
0450                     print('LHE input from article ',article)
0451                     location='/store/lhe/'
0452                     import os
0453                     textOfFiles=os.popen('cmsLHEtoEOSManager.py -l '+article)
0454                     for line in textOfFiles:
0455                         for fileName in [x for x in line.split() if '.lhe' in x]:
0456                             self.process.source.fileNames.append(location+article+'/'+fileName)
0457                     #check first if list of LHE files is loaded (not empty)
0458                     if len(line)<2:
0459                         print('Issue to load LHE files, please check and try again.')
0460                         sys.exit(-1)
0461                     #Additional check to protect empty fileNames in process.source
0462                     if len(self.process.source.fileNames)==0:
0463                         print('Issue with empty filename, but can pass line check')
0464                         sys.exit(-1)
0465                     if len(args)>2:
0466                         self.process.source.skipEvents = cms.untracked.uint32(int(args[2]))
0467                 else:
0468                     filesFromOption(self)
0469 
0470             elif self._options.filetype == "DQM":
0471                 self.process.source=cms.Source("DQMRootSource",
0472                                                fileNames = cms.untracked.vstring())
0473                 filesFromOption(self)
0474 
0475             elif self._options.filetype == "DQMDAQ":
0476                 # FIXME: how to configure it if there are no input files specified?
0477                 self.process.source=cms.Source("DQMStreamerReader")
0478 
0479 
0480             if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"):
0481                 self.process.source.processingMode = cms.untracked.string("RunsAndLumis")
0482 
0483         if self._options.dasquery!='':
0484             self.process.source=cms.Source("PoolSource", fileNames = cms.untracked.vstring(),secondaryFileNames = cms.untracked.vstring())
0485             filesFromDASQuery(self._options.dasquery,self._options.dasoption,self.process.source)
0486 
0487             if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"):
0488                 self.process.source.processingMode = cms.untracked.string("RunsAndLumis")
0489 
0490         ##drop LHEXMLStringProduct on input to save memory if appropriate
0491         if 'GEN' in self.stepMap.keys() and not self._options.filetype == "LHE":
0492             if self._options.inputCommands:
0493                 self._options.inputCommands+=',drop LHEXMLStringProduct_*_*_*,'
0494             else:
0495                 self._options.inputCommands='keep *, drop LHEXMLStringProduct_*_*_*,'
0496 
0497         if self.process.source and self._options.inputCommands and not self._options.filetype == "LHE":
0498             if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring()
0499             for command in self._options.inputCommands.split(','):
0500                 # remove whitespace around the keep/drop statements
0501                 command = command.strip()
0502                 if command=='': continue
0503                 self.process.source.inputCommands.append(command)
0504             if not self._options.dropDescendant:
0505                 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False)
0506 
0507         if self._options.lumiToProcess:
0508             import FWCore.PythonUtilities.LumiList as LumiList
0509             self.process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange( LumiList.LumiList(self._options.lumiToProcess).getCMSSWString().split(',') )
0510 
0511         if 'GEN' in self.stepMap.keys() or 'LHE' in self.stepMap or (not self._options.filein and hasattr(self._options, "evt_type")):
0512             if self.process.source is None:
0513                 self.process.source=cms.Source("EmptySource")
0514 
0515         # modify source in case of run-dependent MC
0516         self.runsAndWeights=None
0517         if self._options.runsAndWeightsForMC or self._options.runsScenarioForMC :
0518             if not self._options.isMC :
0519                 raise Exception("options --runsAndWeightsForMC and --runsScenarioForMC are only valid for MC")
0520             if self._options.runsAndWeightsForMC:
0521                 self.runsAndWeights = eval(self._options.runsAndWeightsForMC)
0522             else:
0523                 from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights
0524                 if isinstance(RunsAndWeights[self._options.runsScenarioForMC], str):
0525                     __import__(RunsAndWeights[self._options.runsScenarioForMC])
0526                     self.runsAndWeights = sys.modules[RunsAndWeights[self._options.runsScenarioForMC]].runProbabilityDistribution
0527                 else:
0528                     self.runsAndWeights = RunsAndWeights[self._options.runsScenarioForMC]
0529 
0530         if self.runsAndWeights:
0531             import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun
0532             ThrowAndSetRandomRun.throwAndSetRandomRun(self.process.source,self.runsAndWeights)
0533             self.additionalCommands.append('import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun')
0534             self.additionalCommands.append('ThrowAndSetRandomRun.throwAndSetRandomRun(process.source,%s)'%(self.runsAndWeights))
0535 
0536         # modify source in case of run-dependent MC (Run-3 method)
0537         self.runsAndWeightsInt=None
0538         if self._options.runsAndWeightsForMCIntegerWeights or self._options.runsScenarioForMCIntegerWeights:
0539             if not self._options.isMC :
0540                 raise Exception("options --runsAndWeightsForMCIntegerWeights and --runsScenarioForMCIntegerWeights are only valid for MC")
0541             if self._options.runsAndWeightsForMCIntegerWeights:
0542                 self.runsAndWeightsInt = eval(self._options.runsAndWeightsForMCIntegerWeights)
0543             else:
0544                 from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights
0545                 if isinstance(RunsAndWeights[self._options.runsScenarioForMCIntegerWeights], str):
0546                     __import__(RunsAndWeights[self._options.runsScenarioForMCIntegerWeights])
0547                     self.runsAndWeightsInt = sys.modules[RunsAndWeights[self._options.runsScenarioForMCIntegerWeights]].runProbabilityDistribution
0548                 else:
0549                     self.runsAndWeightsInt = RunsAndWeights[self._options.runsScenarioForMCIntegerWeights]
0550 
0551         if self.runsAndWeightsInt:
0552             if not self._options.relval:
0553                 raise Exception("--relval option required when using --runsAndWeightsInt")
0554             if 'DATAMIX' in self._options.step:
0555                 from SimGeneral.Configuration.LumiToRun import lumi_to_run
0556                 total_events, events_per_job  = self._options.relval.split(',')
0557                 lumi_to_run_mapping = lumi_to_run(self.runsAndWeightsInt, int(total_events), int(events_per_job))
0558                 self.additionalCommands.append("process.source.firstLuminosityBlockForEachRun = cms.untracked.VLuminosityBlockID(*[cms.LuminosityBlockID(x,y) for x,y in " + str(lumi_to_run_mapping) + "])")
0559 
0560         return
0561 
0562     def addOutput(self):
0563         """ Add output module to the process """
0564         result=""
0565         if self._options.outputDefinition:
0566             if self._options.datatier:
0567                 print("--datatier & --eventcontent options ignored")
0568 
0569             #new output convention with a list of dict
0570             outList = eval(self._options.outputDefinition)
0571             for (id,outDefDict) in enumerate(outList):
0572                 outDefDictStr=outDefDict.__str__()
0573                 if not isinstance(outDefDict,dict):
0574                     raise Exception("--output needs to be passed a list of dict"+self._options.outputDefinition+" is invalid")
0575                 #requires option: tier
0576                 theTier=anyOf(['t','tier','dataTier'],outDefDict)
0577                 #optional option: eventcontent, filtername, selectEvents, moduleLabel, filename
0578                 ## event content
0579                 theStreamType=anyOf(['e','ec','eventContent','streamType'],outDefDict,theTier)
0580                 theFilterName=anyOf(['f','ftN','filterName'],outDefDict,'')
0581                 theSelectEvent=anyOf(['s','sE','selectEvents'],outDefDict,'')
0582                 theModuleLabel=anyOf(['l','mL','moduleLabel'],outDefDict,'')
0583                 theExtraOutputCommands=anyOf(['o','oC','outputCommands'],outDefDict,'')
0584                 # module label has a particular role
0585                 if not theModuleLabel:
0586                     tryNames=[theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+'output',
0587                               theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+'output',
0588                               theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+theSelectEvent.split(',')[0].replace(':','for').replace(' ','')+'output'
0589                               ]
0590                     for name in tryNames:
0591                         if not hasattr(self.process,name):
0592                             theModuleLabel=name
0593                             break
0594                 if not theModuleLabel:
0595                     raise Exception("cannot find a module label for specification: "+outDefDictStr)
0596                 if id==0:
0597                     defaultFileName=self._options.outfile_name
0598                 else:
0599                     defaultFileName=self._options.outfile_name.replace('.root','_in'+theTier+'.root')
0600 
0601                 theFileName=self._options.dirout+anyOf(['fn','fileName'],outDefDict,defaultFileName)
0602                 if not theFileName.endswith('.root'):
0603                     theFileName+='.root'
0604 
0605                 if len(outDefDict):
0606                     raise Exception("unused keys from --output options: "+','.join(outDefDict.keys()))
0607                 if theStreamType=='DQMIO': theStreamType='DQM'
0608                 if theStreamType=='ALL':
0609                     theEventContent = cms.PSet(outputCommands = cms.untracked.vstring('keep *'))
0610                 else:
0611                     theEventContent = getattr(self.process, theStreamType+"EventContent")
0612 
0613 
0614                 addAlCaSelects=False
0615                 if theStreamType=='ALCARECO' and not theFilterName:
0616                     theFilterName='StreamALCACombined'
0617                     addAlCaSelects=True
0618 
0619                 CppType='PoolOutputModule'
0620                 if self._options.timeoutOutput:
0621                     CppType='TimeoutPoolOutputModule'
0622                 if theStreamType=='DQM' and theTier=='DQMIO': CppType='DQMRootOutputModule'
0623                 output = cms.OutputModule(CppType,
0624                                           theEventContent.clone(),
0625                                           fileName = cms.untracked.string(theFileName),
0626                                           dataset = cms.untracked.PSet(
0627                                              dataTier = cms.untracked.string(theTier),
0628                                              filterName = cms.untracked.string(theFilterName))
0629                                           )
0630                 if not theSelectEvent and hasattr(self.process,'generation_step') and theStreamType!='LHE':
0631                     output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step'))
0632                 if not theSelectEvent and hasattr(self.process,'filtering_step'):
0633                     output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step'))
0634                 if theSelectEvent:
0635                     output.SelectEvents =cms.untracked.PSet(SelectEvents = cms.vstring(theSelectEvent))
0636 
0637                 if addAlCaSelects:
0638                     if not hasattr(output,'SelectEvents'):
0639                         output.SelectEvents=cms.untracked.PSet(SelectEvents=cms.vstring())
0640                     for alca in self.AlCaPaths:
0641                         output.SelectEvents.SelectEvents.extend(getattr(self.process,'OutALCARECO'+alca).SelectEvents.SelectEvents)
0642 
0643 
0644                 if hasattr(self.process,theModuleLabel):
0645                     raise Exception("the current process already has a module "+theModuleLabel+" defined")
0646                 #print "creating output module ",theModuleLabel
0647                 setattr(self.process,theModuleLabel,output)
0648                 outputModule=getattr(self.process,theModuleLabel)
0649                 setattr(self.process,theModuleLabel+'_step',cms.EndPath(outputModule))
0650                 path=getattr(self.process,theModuleLabel+'_step')
0651                 self.schedule.append(path)
0652 
0653                 if not self._options.inlineEventContent and hasattr(self.process,theStreamType+"EventContent"):
0654                     def doNotInlineEventContent(instance,label = "cms.untracked.vstring(process."+theStreamType+"EventContent.outputCommands)"):
0655                         return label
0656                     outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
0657                 if theExtraOutputCommands:
0658                     if not isinstance(theExtraOutputCommands,list):
0659                         raise Exception("extra ouput command in --option must be a list of strings")
0660                     if hasattr(self.process,theStreamType+"EventContent"):
0661                         self.executeAndRemember('process.%s.outputCommands.extend(%s)'%(theModuleLabel,theExtraOutputCommands))
0662                     else:
0663                         outputModule.outputCommands.extend(theExtraOutputCommands)
0664 
0665                 result+="\nprocess."+theModuleLabel+" = "+outputModule.dumpPython()
0666 
0667             ##ends the --output options model
0668             return result
0669 
0670         streamTypes=self._options.eventcontent.split(',')
0671         tiers=self._options.datatier.split(',')
0672         if not self._options.outputDefinition and len(streamTypes)!=len(tiers):
0673             raise Exception("number of event content arguments does not match number of datatier arguments")
0674 
0675         # if the only step is alca we don't need to put in an output
0676         if self._options.step.split(',')[0].split(':')[0] == 'ALCA':
0677             return "\n"
0678 
0679         for i,(streamType,tier) in enumerate(zip(streamTypes,tiers)):
0680             if streamType=='': continue
0681             if streamType == 'ALCARECO' and not 'ALCAPRODUCER' in self._options.step: continue
0682             if streamType=='DQMIO': streamType='DQM'
0683             eventContent=streamType
0684             ## override streamType to eventContent in case NANOEDM
0685             if streamType == "NANOEDMAOD" :
0686                 eventContent = "NANOAOD"
0687             elif streamType == "NANOEDMAODSIM" :
0688                 eventContent = "NANOAODSIM"
0689             theEventContent = getattr(self.process, eventContent+"EventContent")
0690             if i==0:
0691                 theFileName=self._options.outfile_name
0692                 theFilterName=self._options.filtername
0693             else:
0694                 theFileName=self._options.outfile_name.replace('.root','_in'+streamType+'.root')
0695                 theFilterName=self._options.filtername
0696             CppType='PoolOutputModule'
0697             if self._options.timeoutOutput:
0698                 CppType='TimeoutPoolOutputModule'
0699             if streamType=='DQM' and tier=='DQMIO': CppType='DQMRootOutputModule'
0700             if "NANOAOD" in streamType : CppType='NanoAODOutputModule'
0701             output = cms.OutputModule(CppType,
0702                                       theEventContent,
0703                                       fileName = cms.untracked.string(theFileName),
0704                                       dataset = cms.untracked.PSet(dataTier = cms.untracked.string(tier),
0705                                                                    filterName = cms.untracked.string(theFilterName)
0706                                                                    )
0707                                       )
0708             if hasattr(self.process,"generation_step") and streamType!='LHE':
0709                 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step'))
0710             if hasattr(self.process,"filtering_step"):
0711                 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step'))
0712 
0713             if streamType=='ALCARECO':
0714                 output.dataset.filterName = cms.untracked.string('StreamALCACombined')
0715 
0716             if "MINIAOD" in streamType:
0717                 from PhysicsTools.PatAlgos.slimming.miniAOD_tools import miniAOD_customizeOutput
0718                 miniAOD_customizeOutput(output)
0719 
0720             outputModuleName=streamType+'output'
0721             setattr(self.process,outputModuleName,output)
0722             outputModule=getattr(self.process,outputModuleName)
0723             setattr(self.process,outputModuleName+'_step',cms.EndPath(outputModule))
0724             path=getattr(self.process,outputModuleName+'_step')
0725             self.schedule.append(path)
0726 
0727             if self._options.outputCommands and streamType!='DQM':
0728                 for evct in self._options.outputCommands.split(','):
0729                     if not evct: continue
0730                     self.executeAndRemember("process.%s.outputCommands.append('%s')"%(outputModuleName,evct.strip()))
0731 
0732             if not self._options.inlineEventContent:
0733                 tmpstreamType=streamType
0734                 if "NANOEDM" in tmpstreamType :
0735                     tmpstreamType=tmpstreamType.replace("NANOEDM","NANO")
0736                 def doNotInlineEventContent(instance,label = "process."+tmpstreamType+"EventContent.outputCommands"):
0737                     return label
0738                 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
0739 
0740             result+="\nprocess."+outputModuleName+" = "+outputModule.dumpPython()
0741 
0742         return result
0743 
0744     def addStandardSequences(self):
0745         """
0746         Add selected standard sequences to the process
0747         """
0748         # load the pile up file
0749         if self._options.pileup:
0750             pileupSpec=self._options.pileup.split(',')[0]
0751 
0752             #make sure there is a set of pileup files specified when needed
0753             pileups_without_input=[defaultOptions.pileup,"Cosmics","default","HiMixNoPU",None]
0754             if self._options.pileup not in pileups_without_input and self._options.pileup_input==None:
0755                 message = "Pileup scenerio requires input files. Please add an appropriate --pileup_input option"
0756                 raise Exception(message)
0757 
0758             # Does the requested pile-up scenario exist?
0759             from Configuration.StandardSequences.Mixing import Mixing,defineMixing
0760             if not pileupSpec in Mixing and '.' not in pileupSpec and 'file:' not in pileupSpec:
0761                 message = pileupSpec+' is not a know mixing scenario:\n available are: '+'\n'.join(Mixing.keys())
0762                 raise Exception(message)
0763 
0764             # Put mixing parameters in a dictionary
0765             if '.' in pileupSpec:
0766                 mixingDict={'file':pileupSpec}
0767             elif pileupSpec.startswith('file:'):
0768                 mixingDict={'file':pileupSpec[5:]}
0769             else:
0770                 import copy
0771                 mixingDict=copy.copy(Mixing[pileupSpec])
0772             if len(self._options.pileup.split(','))>1:
0773                 mixingDict.update(eval(self._options.pileup[self._options.pileup.find(',')+1:]))
0774 
0775             # Load the pu cfg file corresponding to the requested pu scenario
0776             if 'file:' in pileupSpec:
0777                 #the file is local
0778                 self.process.load(mixingDict['file'])
0779                 print("inlining mixing module configuration")
0780                 self._options.inlineObjects+=',mix'
0781             else:
0782                 self.loadAndRemember(mixingDict['file'])
0783 
0784             mixingDict.pop('file')
0785             if not "DATAMIX" in self.stepMap.keys(): # when DATAMIX is present, pileup_input refers to pre-mixed GEN-RAW
0786                 if self._options.pileup_input:
0787                     if self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:'):
0788                         mixingDict['F']=filesFromDASQuery('file dataset = %s'%(self._options.pileup_input[4:],),self._options.pileup_dasoption)[0]
0789                     elif self._options.pileup_input.startswith("filelist:"):
0790                         mixingDict['F']=(filesFromList(self._options.pileup_input[9:]))[0]
0791                     else:
0792                         mixingDict['F']=self._options.pileup_input.split(',')
0793                 specialization=defineMixing(mixingDict)
0794                 for command in specialization:
0795                     self.executeAndRemember(command)
0796                 if len(mixingDict)!=0:
0797                     raise Exception('unused mixing specification: '+mixingDict.keys().__str__())
0798 
0799 
0800         # load the geometry file
0801         try:
0802             if len(self.stepMap):
0803                 self.loadAndRemember(self.GeometryCFF)
0804                 if ('SIM' in self.stepMap or 'reSIM' in self.stepMap) and not self._options.fast:
0805                     self.loadAndRemember(self.SimGeometryCFF)
0806                     if self.geometryDBLabel:
0807                         self.executeAndRemember('if hasattr(process, "XMLFromDBSource"): process.XMLFromDBSource.label="%s"'%(self.geometryDBLabel))
0808                         self.executeAndRemember('if hasattr(process, "DDDetectorESProducerFromDB"): process.DDDetectorESProducerFromDB.label="%s"'%(self.geometryDBLabel))
0809 
0810         except ImportError:
0811             print("Geometry option",self._options.geometry,"unknown.")
0812             raise
0813 
0814         if len(self.stepMap):
0815             self.loadAndRemember(self.magFieldCFF)
0816 
0817         for stepName in self.stepKeys:
0818             stepSpec = self.stepMap[stepName]
0819             print("Step:", stepName,"Spec:",stepSpec)
0820             if stepName.startswith('re'):
0821                 ##add the corresponding input content
0822                 if stepName[2:] not in self._options.donotDropOnInput:
0823                     self._options.inputEventContent='%s,%s'%(stepName.upper(),self._options.inputEventContent)
0824                 stepName=stepName[2:]
0825             if stepSpec=="":
0826                 getattr(self,"prepare_"+stepName)(stepSpec = getattr(self,stepName+"DefaultSeq"))
0827             elif isinstance(stepSpec, list):
0828                 getattr(self,"prepare_"+stepName)(stepSpec = '+'.join(stepSpec))
0829             elif isinstance(stepSpec, tuple):
0830                 getattr(self,"prepare_"+stepName)(stepSpec = ','.join([stepSpec[1],'+'.join(stepSpec[0])]))
0831             else:
0832                 raise ValueError("Invalid step definition")
0833 
0834         if self._options.restoreRNDSeeds!=False:
0835             #it is either True, or a process name
0836             if self._options.restoreRNDSeeds==True:
0837                 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateLabel=cms.untracked.string("randomEngineStateProducer")')
0838             else:
0839                 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateTag=cms.untracked.InputTag("randomEngineStateProducer","","%s")'%(self._options.restoreRNDSeeds))
0840             if self._options.inputEventContent or self._options.inputCommands:
0841                 if self._options.inputCommands:
0842                     self._options.inputCommands+='keep *_randomEngineStateProducer_*_*,'
0843                 else:
0844                     self._options.inputCommands='keep *_randomEngineStateProducer_*_*,'
0845 
0846 
0847     def completeInputCommand(self):
0848         if self._options.inputEventContent:
0849             import copy
0850             def dropSecondDropStar(iec):
0851                 #drop occurence of 'drop *' in the list
0852                 count=0
0853                 for item in iec:
0854                     if item=='drop *':
0855                         if count!=0:
0856                             iec.remove(item)
0857                         count+=1
0858 
0859             ## allow comma separated input eventcontent
0860             if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring()
0861             for evct in self._options.inputEventContent.split(','):
0862                 if evct=='': continue
0863                 theEventContent = getattr(self.process, evct+"EventContent")
0864                 if hasattr(theEventContent,'outputCommands'):
0865                     self.process.source.inputCommands.extend(copy.copy(theEventContent.outputCommands))
0866                 if hasattr(theEventContent,'inputCommands'):
0867                     self.process.source.inputCommands.extend(copy.copy(theEventContent.inputCommands))
0868 
0869             dropSecondDropStar(self.process.source.inputCommands)
0870 
0871             if not self._options.dropDescendant:
0872                 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False)
0873 
0874 
0875         return
0876 
0877     def addConditions(self):
0878         """Add conditions to the process"""
0879         if not self._options.conditions: return
0880 
0881         if 'FrontierConditions_GlobalTag' in self._options.conditions:
0882             print('using FrontierConditions_GlobalTag in --conditions is not necessary anymore and will be deprecated soon. please update your command line')
0883             self._options.conditions = self._options.conditions.replace("FrontierConditions_GlobalTag,",'')
0884 
0885         self.loadAndRemember(self.ConditionsDefaultCFF)
0886         from Configuration.AlCa.GlobalTag import GlobalTag
0887         self.process.GlobalTag = GlobalTag(self.process.GlobalTag, self._options.conditions, self._options.custom_conditions)
0888         self.additionalCommands.append('from Configuration.AlCa.GlobalTag import GlobalTag')
0889         self.additionalCommands.append('process.GlobalTag = GlobalTag(process.GlobalTag, %s, %s)' % (repr(self._options.conditions), repr(self._options.custom_conditions)))
0890 
0891 
0892     def addCustomise(self,unsch=0):
0893         """Include the customise code """
0894 
0895         custOpt=[]
0896         if unsch==0:
0897             for c in self._options.customisation_file:
0898                 custOpt.extend(c.split(","))
0899         else:
0900             for c in self._options.customisation_file_unsch:
0901                 custOpt.extend(c.split(","))
0902 
0903         custMap=DictTypes.SortedKeysDict()
0904         for opt in custOpt:
0905             if opt=='': continue
0906             if opt.count('.')>1:
0907                 raise Exception("more than . in the specification:"+opt)
0908             fileName=opt.split('.')[0]
0909             if opt.count('.')==0: rest='customise'
0910             else:
0911                 rest=opt.split('.')[1]
0912                 if rest=='py': rest='customise' #catch the case of --customise file.py
0913 
0914             if fileName in custMap:
0915                 custMap[fileName].extend(rest.split('+'))
0916             else:
0917                 custMap[fileName]=rest.split('+')
0918 
0919         if len(custMap)==0:
0920             final_snippet='\n'
0921         else:
0922             final_snippet='\n# customisation of the process.\n'
0923 
0924         allFcn=[]
0925         for opt in custMap:
0926             allFcn.extend(custMap[opt])
0927         for fcn in allFcn:
0928             if allFcn.count(fcn)!=1:
0929                 raise Exception("cannot specify twice "+fcn+" as a customisation method")
0930 
0931         for f in custMap:
0932             # let python search for that package and do syntax checking at the same time
0933             packageName = f.replace(".py","").replace("/",".")
0934             __import__(packageName)
0935             package = sys.modules[packageName]
0936 
0937             # now ask the package for its definition and pick .py instead of .pyc
0938             customiseFile = re.sub(r'\.pyc$', '.py', package.__file__)
0939 
0940             final_snippet+='\n# Automatic addition of the customisation function from '+packageName+'\n'
0941             if self._options.inline_custom:
0942                 for line in file(customiseFile,'r'):
0943                     if "import FWCore.ParameterSet.Config" in line:
0944                         continue
0945                     final_snippet += line
0946             else:
0947                 final_snippet += 'from %s import %s \n'%(packageName,','.join(custMap[f]))
0948             for fcn in custMap[f]:
0949                 print("customising the process with",fcn,"from",f)
0950                 if not hasattr(package,fcn):
0951                     #bound to fail at run time
0952                     raise Exception("config "+f+" has no function "+fcn)
0953                 #execute the command
0954                 self.process=getattr(package,fcn)(self.process)
0955                 #and print it in the configuration
0956                 final_snippet += "\n#call to customisation function "+fcn+" imported from "+packageName
0957                 final_snippet += "\nprocess = %s(process)\n"%(fcn,)
0958 
0959         if len(custMap)!=0:
0960             final_snippet += '\n# End of customisation functions\n'
0961 
0962         ### now for a useful command
0963         return final_snippet
0964 
0965     def addCustomiseCmdLine(self):
0966         final_snippet='\n# Customisation from command line\n'
0967         if self._options.customise_commands:
0968             import string
0969             for com in self._options.customise_commands.split('\\n'):
0970                 com=com.lstrip()
0971                 self.executeAndRemember(com)
0972                 final_snippet +='\n'+com
0973 
0974         return final_snippet
0975 
0976     #----------------------------------------------------------------------------
0977     # here the methods to define the python includes for each step or
0978     # conditions
0979     #----------------------------------------------------------------------------
0980     def define_Configs(self):
0981         if len(self.stepMap):
0982             self.loadAndRemember('Configuration/StandardSequences/Services_cff')
0983         if self._options.particleTable not in defaultOptions.particleTableList:
0984             print('Invalid particle table provided. Options are:')
0985             print(defaultOptions.particleTable)
0986             sys.exit(-1)
0987         else:
0988             if len(self.stepMap):
0989                 self.loadAndRemember('SimGeneral.HepPDTESSource.'+self._options.particleTable+'_cfi')
0990 
0991         self.loadAndRemember('FWCore/MessageService/MessageLogger_cfi')
0992 
0993         self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreams_cff"
0994         self.GENDefaultCFF="Configuration/StandardSequences/Generator_cff"
0995         self.SIMDefaultCFF="Configuration/StandardSequences/Sim_cff"
0996         self.DIGIDefaultCFF="Configuration/StandardSequences/Digi_cff"
0997         self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRaw_cff"
0998         self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1Emulator_cff'
0999         self.L1P2GTDefaultCFF = 'Configuration/StandardSequences/SimPhase2L1GlobalTriggerEmulator_cff'
1000         self.L1MENUDefaultCFF="Configuration/StandardSequences/L1TriggerDefaultMenu_cff"
1001         self.HLTDefaultCFF="Configuration/StandardSequences/HLTtable_cff"
1002         self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_Data_cff"
1003         if self._options.isRepacked: self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_DataMapper_cff"
1004         self.L1RecoDefaultCFF="Configuration/StandardSequences/L1Reco_cff"
1005         self.L1TrackTriggerDefaultCFF="Configuration/StandardSequences/L1TrackTrigger_cff"
1006         self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_Data_cff"
1007         self.RECOSIMDefaultCFF="Configuration/StandardSequences/RecoSim_cff"
1008         self.PATDefaultCFF="Configuration/StandardSequences/PAT_cff"
1009         self.NANODefaultCFF="PhysicsTools/NanoAOD/nano_cff"
1010         self.NANOGENDefaultCFF="PhysicsTools/NanoAOD/nanogen_cff"
1011         self.SKIMDefaultCFF="Configuration/StandardSequences/Skims_cff"
1012         self.POSTRECODefaultCFF="Configuration/StandardSequences/PostRecoGenerator_cff"
1013         self.VALIDATIONDefaultCFF="Configuration/StandardSequences/Validation_cff"
1014         self.L1HwValDefaultCFF = "Configuration/StandardSequences/L1HwVal_cff"
1015         self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOffline_cff"
1016         self.HARVESTINGDefaultCFF="Configuration/StandardSequences/Harvesting_cff"
1017         self.ALCAHARVESTDefaultCFF="Configuration/StandardSequences/AlCaHarvesting_cff"
1018         self.ENDJOBDefaultCFF="Configuration/StandardSequences/EndOfProcess_cff"
1019         self.ConditionsDefaultCFF = "Configuration/StandardSequences/FrontierConditions_GlobalTag_cff"
1020         self.CFWRITERDefaultCFF = "Configuration/StandardSequences/CrossingFrameWriter_cff"
1021         self.REPACKDefaultCFF="Configuration/StandardSequences/DigiToRaw_Repack_cff"
1022 
1023         if "DATAMIX" in self.stepMap.keys():
1024             self.DATAMIXDefaultCFF="Configuration/StandardSequences/DataMixer"+self._options.datamix+"_cff"
1025             self.DIGIDefaultCFF="Configuration/StandardSequences/DigiDM_cff"
1026             self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRawDM_cff"
1027             self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1EmulatorDM_cff'
1028 
1029         self.ALCADefaultSeq=None
1030         self.LHEDefaultSeq='externalLHEProducer'
1031         self.GENDefaultSeq='pgen'
1032         self.SIMDefaultSeq='psim'
1033         self.DIGIDefaultSeq='pdigi'
1034         self.DATAMIXDefaultSeq=None
1035         self.DIGI2RAWDefaultSeq='DigiToRaw'
1036         self.HLTDefaultSeq='GRun'
1037         self.L1DefaultSeq=None
1038         self.L1P2GTDefaultSeq=None
1039         self.L1REPACKDefaultSeq='GT'
1040         self.HARVESTINGDefaultSeq=None
1041         self.ALCAHARVESTDefaultSeq=None
1042         self.CFWRITERDefaultSeq=None
1043         self.RAW2DIGIDefaultSeq='RawToDigi'
1044         self.L1RecoDefaultSeq='L1Reco'
1045         self.L1TrackTriggerDefaultSeq='L1TrackTrigger'
1046         if self._options.fast or ('RAW2DIGI' in self.stepMap and 'RECO' in self.stepMap):
1047             self.RECODefaultSeq='reconstruction'
1048         else:
1049             self.RECODefaultSeq='reconstruction_fromRECO'
1050         self.RECOSIMDefaultSeq='recosim'
1051         self.POSTRECODefaultSeq=None
1052         self.L1HwValDefaultSeq='L1HwVal'
1053         self.DQMDefaultSeq='DQMOffline'
1054         self.VALIDATIONDefaultSeq=''
1055         self.ENDJOBDefaultSeq='endOfProcess'
1056         self.REPACKDefaultSeq='DigiToRawRepack'
1057         self.PATDefaultSeq='miniAOD'
1058         self.PATGENDefaultSeq='miniGEN'
1059         #TODO: Check based of file input
1060         self.NANOGENDefaultSeq='nanogenSequence'
1061         self.NANODefaultSeq='nanoSequence'
1062         self.NANODefaultCustom='nanoAOD_customizeCommon'
1063 
1064         self.EVTCONTDefaultCFF="Configuration/EventContent/EventContent_cff"
1065 
1066         if not self._options.beamspot:
1067             self._options.beamspot=VtxSmearedDefaultKey
1068 
1069         # if its MC then change the raw2digi
1070         if self._options.isMC==True:
1071             self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_cff"
1072             self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_cff"
1073             self.PATDefaultCFF="Configuration/StandardSequences/PATMC_cff"
1074             self.PATGENDefaultCFF="Configuration/StandardSequences/PATGEN_cff"
1075             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineMC_cff"
1076             self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreamsMC_cff"
1077             self.NANODefaultSeq='nanoSequenceMC'
1078         else:
1079             self._options.beamspot = None
1080 
1081         #patch for gen, due to backward incompatibility
1082         if 'reGEN' in self.stepMap:
1083             self.GENDefaultSeq='fixGenInfo'
1084 
1085         if self._options.scenario=='cosmics':
1086             self._options.pileup='Cosmics'
1087             self.DIGIDefaultCFF="Configuration/StandardSequences/DigiCosmics_cff"
1088             self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionCosmics_cff"
1089             self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsCosmics_cff"
1090             self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentCosmics_cff"
1091             self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationCosmics_cff"
1092             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmics_cff"
1093             if self._options.isMC==True:
1094                 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmicsMC_cff"
1095             self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingCosmics_cff"
1096             self.RECODefaultSeq='reconstructionCosmics'
1097             self.DQMDefaultSeq='DQMOfflineCosmics'
1098 
1099         if self._options.scenario=='HeavyIons':
1100             if not self._options.beamspot:
1101                 self._options.beamspot=VtxSmearedHIDefaultKey
1102             self.HLTDefaultSeq = 'HIon'
1103             self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationHeavyIons_cff"
1104             self.VALIDATIONDefaultSeq=''
1105             self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentHeavyIons_cff"
1106             self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_cff"
1107             self.RECODefaultSeq='reconstruction'
1108             self.ALCADefaultCFF = "Configuration/StandardSequences/AlCaRecoStreamsHeavyIons_cff"
1109             self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIons_cff"
1110             self.DQMDefaultSeq='DQMOfflineHeavyIons'
1111             self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsHeavyIons_cff"
1112             self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingHeavyIons_cff"
1113             if self._options.isMC==True:
1114                 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIonsMC_cff"
1115 
1116 
1117         self.RAW2RECODefaultSeq=','.join([self.RAW2DIGIDefaultSeq,self.RECODefaultSeq])
1118 
1119         self.USERDefaultSeq='user'
1120         self.USERDefaultCFF=None
1121 
1122         # the magnetic field
1123         self.magFieldCFF = 'Configuration/StandardSequences/MagneticField_'+self._options.magField.replace('.','')+'_cff'
1124         self.magFieldCFF = self.magFieldCFF.replace("__",'_')
1125 
1126         # the geometry
1127         self.GeometryCFF='Configuration/StandardSequences/GeometryRecoDB_cff'
1128         self.geometryDBLabel=None
1129         simGeometry=''
1130         if self._options.fast:
1131             if 'start' in self._options.conditions.lower():
1132                 self.GeometryCFF='FastSimulation/Configuration/Geometries_START_cff'
1133             else:
1134                 self.GeometryCFF='FastSimulation/Configuration/Geometries_MC_cff'
1135         else:
1136             def inGeometryKeys(opt):
1137                 from Configuration.StandardSequences.GeometryConf import GeometryConf
1138                 if opt in GeometryConf:
1139                     return GeometryConf[opt]
1140                 else:
1141                     return opt
1142 
1143             geoms=self._options.geometry.split(',')
1144             if len(geoms)==1: geoms=inGeometryKeys(geoms[0]).split(',')
1145             if len(geoms)==2:
1146                 #may specify the reco geometry
1147                 if '/' in geoms[1] or '_cff' in geoms[1]:
1148                     self.GeometryCFF=geoms[1]
1149                 else:
1150                     self.GeometryCFF='Configuration/Geometry/Geometry'+geoms[1]+'_cff'
1151 
1152             if (geoms[0].startswith('DB:')):
1153                 self.SimGeometryCFF='Configuration/StandardSequences/GeometrySimDB_cff'
1154                 self.geometryDBLabel=geoms[0][3:]
1155                 print("with DB:")
1156             else:
1157                 if '/' in geoms[0] or '_cff' in geoms[0]:
1158                     self.SimGeometryCFF=geoms[0]
1159                 else:
1160                     simGeometry=geoms[0]
1161                     if self._options.gflash==True:
1162                         self.SimGeometryCFF='Configuration/Geometry/Geometry'+geoms[0]+'GFlash_cff'
1163                     else:
1164                         self.SimGeometryCFF='Configuration/Geometry/Geometry'+geoms[0]+'_cff'
1165 
1166         # synchronize the geometry configuration and the FullSimulation sequence to be used
1167         if simGeometry not in defaultOptions.geometryExtendedOptions:
1168             self.SIMDefaultCFF="Configuration/StandardSequences/SimIdeal_cff"
1169 
1170         if self._options.scenario=='nocoll' or self._options.scenario=='cosmics':
1171             self.SIMDefaultCFF="Configuration/StandardSequences/SimNOBEAM_cff"
1172             self._options.beamspot='NoSmear'
1173 
1174         # fastsim requires some changes to the default cff files and sequences
1175         if self._options.fast:
1176             self.SIMDefaultCFF = 'FastSimulation.Configuration.SimIdeal_cff'
1177             self.RECODefaultCFF= 'FastSimulation.Configuration.Reconstruction_AftMix_cff'
1178             self.RECOBEFMIXDefaultCFF = 'FastSimulation.Configuration.Reconstruction_BefMix_cff'
1179             self.RECOBEFMIXDefaultSeq = 'reconstruction_befmix'
1180             self.NANODefaultSeq = 'nanoSequenceFS'
1181             self.DQMOFFLINEDefaultCFF="DQMOffline.Configuration.DQMOfflineFS_cff"
1182 
1183         # Mixing
1184         if self._options.pileup=='default':
1185             from Configuration.StandardSequences.Mixing import MixingDefaultKey
1186             self._options.pileup=MixingDefaultKey
1187 
1188 
1189         #not driven by a default cff anymore
1190         if self._options.isData:
1191             self._options.pileup=None
1192 
1193 
1194         self.REDIGIDefaultSeq=self.DIGIDefaultSeq
1195 
1196     # for alca, skims, etc
1197     def addExtraStream(self, name, stream, workflow='full'):
1198             # define output module and go from there
1199         output = cms.OutputModule("PoolOutputModule")
1200         if stream.selectEvents.parameters_().__len__()!=0:
1201             output.SelectEvents = stream.selectEvents
1202         else:
1203             output.SelectEvents = cms.untracked.PSet()
1204             output.SelectEvents.SelectEvents=cms.vstring()
1205             if isinstance(stream.paths,tuple):
1206                 for path in stream.paths:
1207                     output.SelectEvents.SelectEvents.append(path.label())
1208             else:
1209                 output.SelectEvents.SelectEvents.append(stream.paths.label())
1210 
1211 
1212 
1213         if isinstance(stream.content,str):
1214             evtPset=getattr(self.process,stream.content)
1215             for p in evtPset.parameters_():
1216                 setattr(output,p,getattr(evtPset,p))
1217             if not self._options.inlineEventContent:
1218                 def doNotInlineEventContent(instance,label = "process."+stream.content+".outputCommands"):
1219                     return label
1220                 output.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent
1221         else:
1222             output.outputCommands = stream.content
1223 
1224 
1225         output.fileName = cms.untracked.string(self._options.dirout+stream.name+'.root')
1226 
1227         output.dataset  = cms.untracked.PSet( dataTier = stream.dataTier,
1228                                               filterName = cms.untracked.string(stream.name))
1229 
1230         if self._options.filtername:
1231             output.dataset.filterName= cms.untracked.string(self._options.filtername+"_"+stream.name)
1232 
1233         #add an automatic flushing to limit memory consumption
1234         output.eventAutoFlushCompressedSize=cms.untracked.int32(5*1024*1024)
1235 
1236         if workflow in ("producers,full"):
1237             if isinstance(stream.paths,tuple):
1238                 for path in stream.paths:
1239                     self.schedule.append(path)
1240             else:
1241                 self.schedule.append(stream.paths)
1242 
1243 
1244         # in case of relvals we don't want to have additional outputs
1245         if (not self._options.relval) and workflow in ("full","output"):
1246             self.additionalOutputs[name] = output
1247             setattr(self.process,name,output)
1248 
1249         if workflow == 'output':
1250             # adjust the select events to the proper trigger results from previous process
1251             filterList = output.SelectEvents.SelectEvents
1252             for i, filter in enumerate(filterList):
1253                 filterList[i] = filter+":"+self._options.triggerResultsProcess
1254 
1255         return output
1256 
1257     #----------------------------------------------------------------------------
1258     # here the methods to create the steps. Of course we are doing magic here ;)
1259     # prepare_STEPNAME modifies self.process and what else's needed.
1260     #----------------------------------------------------------------------------
1261 
1262     def loadDefaultOrSpecifiedCFF(self, stepSpec, defaultCFF, defaultSEQ=''):
1263         _dotsplit = stepSpec.split('.')
1264         if ( len(_dotsplit)==1 ):
1265             if '/' in _dotsplit[0]:
1266                 _sequence = defaultSEQ if defaultSEQ else stepSpec 
1267                 _cff = _dotsplit[0]
1268             else:
1269                 _sequence = stepSpec
1270                 _cff = defaultCFF
1271         elif ( len(_dotsplit)==2 ):
1272             _cff,_sequence  = _dotsplit
1273         else:
1274             print("sub sequence configuration must be of the form dir/subdir/cff.a+b+c or cff.a")
1275             print(stepSpec,"not recognized")
1276             raise
1277         l=self.loadAndRemember(_cff)
1278         return l,_sequence,_cff
1279 
1280     def scheduleSequence(self,seq,prefix,what='Path'):
1281         if '*' in seq:
1282             #create only one path with all sequences in it
1283             for i,s in enumerate(seq.split('*')):
1284                 if i==0:
1285                     setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, s) ))
1286                 else:
1287                     p=getattr(self.process,prefix)
1288                     tmp = getattr(self.process, s)
1289                     if isinstance(tmp, cms.Task):
1290                         p.associate(tmp)
1291                     else:
1292                         p+=tmp
1293             self.schedule.append(getattr(self.process,prefix))
1294             return
1295         else:
1296             #create as many path as many sequences
1297             if not '+' in seq:
1298                 if self.nextScheduleIsConditional:
1299                     self.conditionalPaths.append(prefix)
1300                 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, seq) ))
1301                 self.schedule.append(getattr(self.process,prefix))
1302             else:
1303                 for i,s in enumerate(seq.split('+')):
1304                     sn=prefix+'%d'%(i)
1305                     setattr(self.process,sn,getattr(cms,what)( getattr(self.process, s) ))
1306                     self.schedule.append(getattr(self.process,sn))
1307             return
1308 
1309     def scheduleSequenceAtEnd(self,seq,prefix):
1310         self.scheduleSequence(seq,prefix,what='EndPath')
1311         return
1312 
1313     def prepare_ALCAPRODUCER(self, stepSpec = None):
1314         self.prepare_ALCA(stepSpec, workflow = "producers")
1315 
1316     def prepare_ALCAOUTPUT(self, stepSpec = None):
1317         self.prepare_ALCA(stepSpec, workflow = "output")
1318 
1319     def prepare_ALCA(self, stepSpec = None, workflow = 'full'):
1320         """ Enrich the process with alca streams """
1321         alcaConfig,sequence,_=self.loadDefaultOrSpecifiedCFF(stepSpec,self.ALCADefaultCFF)
1322 
1323         MAXLEN=31 #the alca producer name should be shorter than 31 chars as per https://cms-talk.web.cern.ch/t/alcaprompt-datasets-not-loaded-in-dbs/11146/2
1324         # decide which ALCA paths to use
1325         alcaList = sequence.split("+")
1326         for alca in alcaList:
1327             if (len(alca)>MAXLEN):
1328                 raise Exception("The following alca "+str(alca)+" name (with length "+str(len(alca))+" chars) cannot be accepted because it exceeds the DBS constraints on the length of the name of the ALCARECOs producers ("+str(MAXLEN)+")!")
1329 
1330         maxLevel=0
1331         from Configuration.AlCa.autoAlca import autoAlca, AlCaNoConcurrentLumis
1332         # support @X from autoAlca.py, and recursion support: i.e T0:@Mu+@EG+...
1333         self.expandMapping(alcaList,autoAlca)
1334         self.AlCaPaths=[]
1335         for name in alcaConfig.__dict__:
1336             alcastream = getattr(alcaConfig,name)
1337             shortName = name.replace('ALCARECOStream','')
1338             if shortName in alcaList and isinstance(alcastream,cms.FilteredStream):
1339                 if shortName in AlCaNoConcurrentLumis:
1340                     print("Setting numberOfConcurrentLuminosityBlocks=1 because of AlCa sequence {}".format(shortName))
1341                     self._options.nConcurrentLumis = 1
1342                     self._options.nConcurrentIOVs = 1
1343                 output = self.addExtraStream(name,alcastream, workflow = workflow)
1344                 self.executeAndRemember('process.ALCARECOEventContent.outputCommands.extend(process.OutALCARECO'+shortName+'_noDrop.outputCommands)')
1345                 self.AlCaPaths.append(shortName)
1346                 if 'DQM' in alcaList:
1347                     if not self._options.inlineEventContent and hasattr(self.process,name):
1348                         self.executeAndRemember('process.' + name + '.outputCommands.append("keep *_MEtoEDMConverter_*_*")')
1349                     else:
1350                         output.outputCommands.append("keep *_MEtoEDMConverter_*_*")
1351 
1352                 #rename the HLT process name in the alca modules
1353                 if self._options.hltProcess or 'HLT' in self.stepMap:
1354                     if isinstance(alcastream.paths,tuple):
1355                         for path in alcastream.paths:
1356                             self.renameHLTprocessInSequence(path.label())
1357                     else:
1358                         self.renameHLTprocessInSequence(alcastream.paths.label())
1359 
1360                 for i in range(alcaList.count(shortName)):
1361                     alcaList.remove(shortName)
1362 
1363             # DQM needs a special handling
1364             elif name == 'pathALCARECODQM' and 'DQM' in alcaList:
1365                 path = getattr(alcaConfig,name)
1366                 self.schedule.append(path)
1367                 alcaList.remove('DQM')
1368 
1369             if isinstance(alcastream,cms.Path):
1370                 #black list the alca path so that they do not appear in the cfg
1371                 self.blacklist_paths.append(alcastream)
1372 
1373 
1374         if len(alcaList) != 0:
1375             available=[]
1376             for name in alcaConfig.__dict__:
1377                 alcastream = getattr(alcaConfig,name)
1378                 if isinstance(alcastream,cms.FilteredStream):
1379                     available.append(name.replace('ALCARECOStream',''))
1380             print("The following alcas could not be found "+str(alcaList))
1381             print("available ",available)
1382             #print "verify your configuration, ignoring for now"
1383             raise Exception("The following alcas could not be found "+str(alcaList))
1384 
1385     def prepare_LHE(self, stepSpec = None):
1386             #load the fragment
1387             ##make it loadable
1388         loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','').replace('/','.')
1389         print("Loading lhe fragment from",loadFragment)
1390         __import__(loadFragment)
1391         self.process.load(loadFragment)
1392         ##inline the modules
1393         self._options.inlineObjects+=','+stepSpec
1394 
1395         getattr(self.process,stepSpec).nEvents = self._options.number
1396 
1397         #schedule it
1398         self.process.lhe_step = cms.Path( getattr( self.process,stepSpec)  )
1399         self.excludedPaths.append("lhe_step")
1400         self.schedule.append( self.process.lhe_step )
1401 
1402     def prepare_GEN(self, stepSpec = None):
1403         """ load the fragment of generator configuration """
1404         loadFailure=False
1405         #remove trailing .py
1406         #support old style .cfi by changing into something.cfi into something_cfi
1407         #remove python/ from the name
1408         loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','')
1409         #standard location of fragments
1410         if not '/' in loadFragment:
1411             loadFragment='Configuration.Generator.'+loadFragment
1412         else:
1413             loadFragment=loadFragment.replace('/','.')
1414         try:
1415             print("Loading generator fragment from",loadFragment)
1416             __import__(loadFragment)
1417         except:
1418             loadFailure=True
1419             #if self.process.source and self.process.source.type_()=='EmptySource':
1420             if not (self._options.filein or self._options.dasquery):
1421                 raise Exception("Neither gen fragment of input files provided: this is an inconsistent GEN step configuration")
1422 
1423         if not loadFailure:
1424             from Configuration.Generator.concurrentLumisDisable import noConcurrentLumiGenerators
1425 
1426             generatorModule=sys.modules[loadFragment]
1427             genModules=generatorModule.__dict__
1428             #remove lhe producer module since this should have been
1429             #imported instead in the LHE step
1430             if self.LHEDefaultSeq in genModules:
1431                 del genModules[self.LHEDefaultSeq]
1432 
1433             if self._options.hideGen:
1434                 self.loadAndRemember(loadFragment)
1435             else:
1436                 self.process.load(loadFragment)
1437                 # expose the objects from that fragment to the configuration
1438                 import FWCore.ParameterSet.Modules as cmstypes
1439                 for name in genModules:
1440                     theObject = getattr(generatorModule,name)
1441                     if isinstance(theObject, cmstypes._Module):
1442                         self._options.inlineObjects=name+','+self._options.inlineObjects
1443                         if theObject.type_() in noConcurrentLumiGenerators:
1444                             print("Setting numberOfConcurrentLuminosityBlocks=1 because of generator {}".format(theObject.type_()))
1445                             self._options.nConcurrentLumis = 1
1446                             self._options.nConcurrentIOVs = 1
1447                     elif isinstance(theObject, cms.Sequence) or isinstance(theObject, cmstypes.ESProducer):
1448                         self._options.inlineObjects+=','+name
1449 
1450             if stepSpec == self.GENDefaultSeq or stepSpec == 'pgen_genonly':
1451                 if 'ProductionFilterSequence' in genModules and ('generator' in genModules):
1452                     self.productionFilterSequence = 'ProductionFilterSequence'
1453                 elif 'generator' in genModules:
1454                     self.productionFilterSequence = 'generator'
1455 
1456         """ Enrich the schedule with the rest of the generation step """
1457         _,_genSeqName,_=self.loadDefaultOrSpecifiedCFF(stepSpec,self.GENDefaultCFF)
1458 
1459         if True:
1460             try:
1461                 from Configuration.StandardSequences.VtxSmeared import VtxSmeared
1462                 cffToBeLoaded=VtxSmeared[self._options.beamspot]
1463                 self.loadAndRemember(cffToBeLoaded)
1464             except ImportError:
1465                 raise Exception("VertexSmearing type or beamspot "+self._options.beamspot+" unknown.")
1466 
1467             if self._options.scenario == 'HeavyIons':
1468                 if self._options.pileup=='HiMixGEN':
1469                     self.loadAndRemember("Configuration/StandardSequences/GeneratorMix_cff")
1470                 elif self._options.pileup=='HiMixEmbGEN':
1471                     self.loadAndRemember("Configuration/StandardSequences/GeneratorEmbMix_cff")
1472                 else:
1473                     self.loadAndRemember("Configuration/StandardSequences/GeneratorHI_cff")
1474 
1475         self.process.generation_step = cms.Path( getattr(self.process,_genSeqName) )
1476         self.schedule.append(self.process.generation_step)
1477 
1478         #register to the genstepfilter the name of the path (static right now, but might evolve)
1479         self.executeAndRemember('process.genstepfilter.triggerConditions=cms.vstring("generation_step")')
1480 
1481         if 'reGEN' in self.stepMap or stepSpec == 'pgen_smear':
1482             #stop here
1483             return
1484 
1485         """ Enrich the schedule with the summary of the filter step """
1486         #the gen filter in the endpath
1487         self.loadAndRemember("GeneratorInterface/Core/genFilterSummary_cff")
1488         self.scheduleSequenceAtEnd('genFilterSummary','genfiltersummary_step')
1489         return
1490 
1491     def prepare_SIM(self, stepSpec = None):
1492         """ Enrich the schedule with the simulation step"""
1493         _,_simSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.SIMDefaultCFF)
1494         if not self._options.fast:
1495             if self._options.gflash==True:
1496                 self.loadAndRemember("Configuration/StandardSequences/GFlashSIM_cff")
1497 
1498             if self._options.magField=='0T':
1499                 self.executeAndRemember("process.g4SimHits.UseMagneticField = cms.bool(False)")
1500         else:
1501             if self._options.magField=='0T':
1502                 self.executeAndRemember("process.fastSimProducer.detectorDefinition.magneticFieldZ = cms.untracked.double(0.)")
1503 
1504         self.scheduleSequence(_simSeq,'simulation_step')
1505         return
1506 
1507     def prepare_DIGI(self, stepSpec = None):
1508         """ Enrich the schedule with the digitisation step"""
1509         _,_digiSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.DIGIDefaultCFF)
1510 
1511         if self._options.gflash==True:
1512             self.loadAndRemember("Configuration/StandardSequences/GFlashDIGI_cff")
1513 
1514         if _digiSeq == 'pdigi_valid' or _digiSeq == 'pdigi_hi':
1515             self.executeAndRemember("process.mix.digitizers = cms.PSet(process.theDigitizersValid)")
1516 
1517         if _digiSeq != 'pdigi_nogen' and _digiSeq != 'pdigi_valid_nogen' and _digiSeq != 'pdigi_hi_nogen' and not self.process.source.type_()=='EmptySource' and not self._options.filetype == "LHE":
1518             if self._options.inputEventContent=='':
1519                 self._options.inputEventContent='REGEN'
1520             else:
1521                 self._options.inputEventContent=self._options.inputEventContent+',REGEN'
1522 
1523 
1524         self.scheduleSequence(_digiSeq,'digitisation_step')
1525         return
1526 
1527     def prepare_CFWRITER(self, stepSpec = None):
1528         """ Enrich the schedule with the crossing frame writer step"""
1529         self.loadAndRemember(self.CFWRITERDefaultCFF)
1530         self.scheduleSequence('pcfw','cfwriter_step')
1531         return
1532 
1533     def prepare_DATAMIX(self, stepSpec = None):
1534         """ Enrich the schedule with the digitisation step"""
1535         self.loadAndRemember(self.DATAMIXDefaultCFF)
1536         self.scheduleSequence('pdatamix','datamixing_step')
1537 
1538         if self._options.pileup_input:
1539             theFiles=''
1540             if self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:'):
1541                 theFiles=filesFromDASQuery('file dataset = %s'%(self._options.pileup_input[4:],),self._options.pileup_dasoption)[0]
1542             elif self._options.pileup_input.startswith("filelist:"):
1543                 theFiles= (filesFromList(self._options.pileup_input[9:]))[0]
1544             else:
1545                 theFiles=self._options.pileup_input.split(',')
1546             #print theFiles
1547             self.executeAndRemember( "process.mixData.input.fileNames = cms.untracked.vstring(%s)"%(  theFiles ) )
1548 
1549         return
1550 
1551     def prepare_DIGI2RAW(self, stepSpec = None):
1552         _,_digi2rawSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.DIGI2RAWDefaultCFF)
1553         self.scheduleSequence(_digi2rawSeq,'digi2raw_step')
1554         return
1555 
1556     def prepare_REPACK(self, stepSpec = None):
1557         _,_repackSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.REPACKDefaultCFF)
1558         self.scheduleSequence(_repackSeq,'digi2repack_step')
1559         return
1560 
1561     def loadPhase2GTMenu(self, menuFile: str):
1562         import importlib
1563         menuPath = f'L1Trigger.Configuration.Phase2GTMenus.{menuFile}'
1564         menuModule = importlib.import_module(menuPath)
1565         
1566         theMenu = menuModule.menu
1567         triggerPaths = [] #we get a list of paths in each of these files to schedule
1568 
1569         for triggerPathFile in theMenu:
1570             self.loadAndRemember(triggerPathFile) #this load and remember will set the algo variable of the algoblock later
1571 
1572             triggerPathModule = importlib.import_module(triggerPathFile)
1573             for objName in dir(triggerPathModule):
1574                 obj = getattr(triggerPathModule, objName)
1575                 objType = type(obj)
1576                 if objType == cms.Path:
1577                     triggerPaths.append(objName)
1578         
1579         triggerScheduleList = [getattr(self.process, name) for name in triggerPaths] #get the actual paths to put in the schedule
1580         self.schedule.extend(triggerScheduleList) #put them in the schedule for later
1581     
1582     # create the L1 GT step
1583     # We abuse the stepSpec a bit as a way to specify a menu
1584     def prepare_L1P2GT(self, stepSpec=None):
1585         """ Run the GT emulation sequence on top of the L1 emulation step """
1586         self.loadAndRemember(self.L1P2GTDefaultCFF)
1587         self.scheduleSequence('l1tGTProducerSequence', 'Phase2L1GTProducer')
1588         self.scheduleSequence('l1tGTAlgoBlockProducerSequence', 'Phase2L1GTAlgoBlockProducer')
1589         if stepSpec == None:
1590             defaultMenuFile = "prototype_2023_v1_0_0"
1591             self.loadPhase2GTMenu(menuFile = defaultMenuFile)
1592         else:
1593             self.loadPhase2GTMenu(menuFile = stepSpec)
1594 
1595     def prepare_L1(self, stepSpec = None):
1596         """ Enrich the schedule with the L1 simulation step"""
1597         assert(stepSpec == None)
1598         self.loadAndRemember(self.L1EMDefaultCFF)
1599         self.scheduleSequence('SimL1Emulator','L1simulation_step')
1600         return
1601 
1602     def prepare_L1REPACK(self, stepSpec = None):
1603         """ Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection"""
1604         supported = ['GT','GT1','GT2','GCTGT','Full','FullSimTP','FullMC','Full2015Data','uGT','CalouGT']
1605         if stepSpec in supported:
1606             self.loadAndRemember('Configuration/StandardSequences/SimL1EmulatorRepack_%s_cff'% stepSpec)
1607             if self._options.scenario == 'HeavyIons':
1608                 self.renameInputTagsInSequence("SimL1Emulator","rawDataCollector","rawDataRepacker")
1609             self.scheduleSequence('SimL1Emulator','L1RePack_step')
1610         else:
1611             print("L1REPACK with '",stepSpec,"' is not supported! Supported choices are: ",supported)
1612             raise Exception('unsupported feature')
1613 
1614     def prepare_HLT(self, stepSpec = None):
1615         """ Enrich the schedule with the HLT simulation step"""
1616         if not stepSpec:
1617             print("no specification of the hlt menu has been given, should never happen")
1618             raise  Exception('no HLT specifications provided')
1619 
1620         if '@' in stepSpec:
1621             # case where HLT:@something was provided
1622             from Configuration.HLT.autoHLT import autoHLT
1623             key = stepSpec[1:]
1624             if key in autoHLT:
1625                 stepSpec = autoHLT[key]
1626             else:
1627                 raise ValueError('no HLT mapping key "%s" found in autoHLT' % key)
1628 
1629         if ',' in stepSpec:
1630             #case where HLT:something:something was provided
1631             self.executeAndRemember('import HLTrigger.Configuration.Utilities')
1632             optionsForHLT = {}
1633             if self._options.scenario == 'HeavyIons':
1634                 optionsForHLT['type'] = 'HIon'
1635             else:
1636                 optionsForHLT['type'] = 'GRun'
1637             optionsForHLTConfig = ', '.join('%s=%s' % (key, repr(val)) for (key, val) in optionsForHLT.items())
1638             if stepSpec == 'run,fromSource':
1639                 if hasattr(self.process.source,'firstRun'):
1640                     self.executeAndRemember('process.loadHltConfiguration("run:%%d"%%(process.source.firstRun.value()),%s)'%(optionsForHLTConfig))
1641                 elif hasattr(self.process.source,'setRunNumber'):
1642                     self.executeAndRemember('process.loadHltConfiguration("run:%%d"%%(process.source.setRunNumber.value()),%s)'%(optionsForHLTConfig))
1643                 else:
1644                     raise Exception(f'Cannot replace menu to load {stepSpec}')
1645             else:
1646                 self.executeAndRemember('process.loadHltConfiguration("%s",%s)'%(stepSpec.replace(',',':'),optionsForHLTConfig))
1647         else:
1648             self.loadAndRemember('HLTrigger/Configuration/HLT_%s_cff' % stepSpec)
1649 
1650         if self._options.isMC:
1651             self._options.customisation_file.append("HLTrigger/Configuration/customizeHLTforMC.customizeHLTforMC")
1652 
1653         if self._options.name != 'HLT':
1654             self.additionalCommands.append('from HLTrigger.Configuration.CustomConfigs import ProcessName')
1655             self.additionalCommands.append('process = ProcessName(process)')
1656             self.additionalCommands.append('')
1657             from HLTrigger.Configuration.CustomConfigs import ProcessName
1658             self.process = ProcessName(self.process)
1659 
1660         if self.process.schedule == None:
1661             raise Exception('the HLT step did not attach a valid schedule to the process')
1662 
1663         self.scheduleIndexOfFirstHLTPath = len(self.schedule)
1664         [self.blacklist_paths.append(path) for path in self.process.schedule if isinstance(path,(cms.Path,cms.EndPath))]
1665 
1666         # this is a fake, to be removed with fastim migration and HLT menu dump
1667         if self._options.fast:
1668             if not hasattr(self.process,'HLTEndSequence'):
1669                 self.executeAndRemember("process.HLTEndSequence = cms.Sequence( process.dummyModule )")
1670 
1671 
1672     def prepare_RAW2RECO(self, stepSpec = None):
1673         if ','in stepSpec:
1674             seqReco,seqDigi=stepSpec.spli(',')
1675         else:
1676             print(f"RAW2RECO requires two specifications {stepSpec} insufficient")
1677 
1678         self.prepare_RAW2DIGI(seqDigi)
1679         self.prepare_RECO(seqReco)
1680         return
1681 
1682     def prepare_RAW2DIGI(self, stepSpec = "RawToDigi"):
1683         _,_raw2digiSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.RAW2DIGIDefaultCFF)
1684         self.scheduleSequence(_raw2digiSeq,'raw2digi_step')
1685         return
1686 
1687     def prepare_PATFILTER(self, stepSpec = None):
1688         self.loadAndRemember("PhysicsTools/PatAlgos/slimming/metFilterPaths_cff")
1689         from PhysicsTools.PatAlgos.slimming.metFilterPaths_cff import allMetFilterPaths
1690         for filt in allMetFilterPaths:
1691             self.schedule.append(getattr(self.process,'Flag_'+filt))
1692 
1693     def prepare_L1HwVal(self, stepSpec = 'L1HwVal'):
1694         ''' Enrich the schedule with L1 HW validation '''
1695         self.loadDefaultOrSpecifiedCFF(stepSpec,self.L1HwValDefaultCFF)
1696         print('\n\n\n DEPRECATED this has no action \n\n\n')
1697         return
1698 
1699     def prepare_L1Reco(self, stepSpec = "L1Reco"):
1700         ''' Enrich the schedule with L1 reconstruction '''
1701         _,_l1recoSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.L1RecoDefaultCFF)
1702         self.scheduleSequence(_l1recoSeq,'L1Reco_step')
1703         return
1704 
1705     def prepare_L1TrackTrigger(self, stepSpec = "L1TrackTrigger"):
1706         ''' Enrich the schedule with L1 reconstruction '''
1707         _,_l1tracktriggerSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.L1TrackTriggerDefaultCFF)
1708         self.scheduleSequence(_l1tracktriggerSeq,'L1TrackTrigger_step')
1709         return
1710 
1711     def prepare_FILTER(self, stepSpec = None):
1712         ''' Enrich the schedule with a user defined filter sequence '''
1713         ## load the relevant part
1714         filterConfig,filterSeq = stepSpec.split('.')
1715         filterConfig=self.load(filterConfig)
1716         ## print it in the configuration
1717         class PrintAllModules(object):
1718             def __init__(self):
1719                 self.inliner=''
1720                 pass
1721             def enter(self,visitee):
1722                 try:
1723                     label=visitee.label()
1724                     ##needs to be in reverse order
1725                     self.inliner=label+','+self.inliner
1726                 except:
1727                     pass
1728             def leave(self,v): pass
1729 
1730         expander=PrintAllModules()
1731         getattr(self.process,filterSeq).visit( expander )
1732         self._options.inlineObjects+=','+expander.inliner
1733         self._options.inlineObjects+=','+filterSeq
1734 
1735         ## put the filtering path in the schedule
1736         self.scheduleSequence(filterSeq,'filtering_step')
1737         self.nextScheduleIsConditional=True
1738         ## put it before all the other paths
1739         self.productionFilterSequence = filterSeq
1740 
1741         return
1742 
1743     def prepare_RECO(self, stepSpec = "reconstruction"):
1744         ''' Enrich the schedule with reconstruction '''
1745         _,_recoSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.RECODefaultCFF)
1746         self.scheduleSequence(_recoSeq,'reconstruction_step')
1747         return
1748 
1749     def prepare_RECOSIM(self, stepSpec = "recosim"):
1750         ''' Enrich the schedule with reconstruction '''
1751         _,_recosimSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.RECOSIMDefaultCFF)
1752         self.scheduleSequence(_recosimSeq,'recosim_step')
1753         return
1754 
1755     def prepare_RECOBEFMIX(self, stepSpec = "reconstruction"):
1756         ''' Enrich the schedule with the part of reconstruction that is done before mixing in FastSim'''
1757         if not self._options.fast:
1758             print("ERROR: this step is only implemented for FastSim")
1759             sys.exit()
1760         _,_recobefmixSeq,_ = self.loadDefaultOrSpecifiedCFF(self.RECOBEFMIXDefaultSeq,self.RECOBEFMIXDefaultCFF)
1761         self.scheduleSequence(_recobefmixSeq,'reconstruction_befmix_step')
1762         return
1763 
1764     def prepare_PAT(self, stepSpec = "miniAOD"):
1765         ''' Enrich the schedule with PAT '''
1766         self.prepare_PATFILTER(self)
1767         self.loadDefaultOrSpecifiedCFF(stepSpec,self.PATDefaultCFF)
1768         self.labelsToAssociate.append('patTask')
1769         if self._options.isData:
1770             self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllData")
1771         else:
1772             if self._options.fast:
1773                 self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllMCFastSim")
1774             else:
1775                 self._options.customisation_file_unsch.insert(0,"PhysicsTools/PatAlgos/slimming/miniAOD_tools.miniAOD_customizeAllMC")
1776 
1777         if self._options.hltProcess:
1778             if len(self._options.customise_commands) > 1:
1779                 self._options.customise_commands = self._options.customise_commands + " \n"
1780             self._options.customise_commands = self._options.customise_commands + "process.patTrigger.processName = \""+self._options.hltProcess+"\"\n"
1781             self._options.customise_commands = self._options.customise_commands + "process.slimmedPatTrigger.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1782             self._options.customise_commands = self._options.customise_commands + "process.patMuons.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1783 
1784 #            self.renameHLTprocessInSequence(sequence)
1785 
1786         return
1787 
1788     def prepare_PATGEN(self, stepSpec = "miniGEN"):
1789         ''' Enrich the schedule with PATGEN '''
1790         self.loadDefaultOrSpecifiedCFF(stepSpec,self.PATGENDefaultCFF) #this is unscheduled
1791         self.labelsToAssociate.append('patGENTask')
1792         if self._options.isData:
1793             raise Exception("PATGEN step can only run on MC")
1794         return
1795 
1796     def prepare_NANO(self, stepSpec = '' ):
1797         print(f"in prepare_nano {stepSpec}")
1798         ''' Enrich the schedule with NANO '''
1799         if not '@' in stepSpec:
1800             _,_nanoSeq,_nanoCff = self.loadDefaultOrSpecifiedCFF(stepSpec,self.NANODefaultCFF,self.NANODefaultSeq)
1801         else:
1802             _nanoSeq = stepSpec
1803             _nanoCff = self.NANODefaultCFF
1804 
1805         print(_nanoSeq)
1806         # create full specified sequence using autoNANO
1807         from PhysicsTools.NanoAOD.autoNANO import autoNANO, expandNanoMapping
1808         # if not a autoNANO mapping, load an empty customization, which later will be converted into the default.
1809         _nanoCustoms = _nanoSeq.split('+') if '@' in stepSpec else ['']
1810         _nanoSeq = _nanoSeq.split('+')
1811         expandNanoMapping(_nanoSeq, autoNANO, 'sequence')
1812         expandNanoMapping(_nanoCustoms, autoNANO, 'customize')
1813         # make sure there are no duplicates while preserving the ordering
1814         _nanoSeq = list(sorted(set(_nanoSeq), key=_nanoSeq.index))
1815         _nanoCustoms = list(sorted(set(_nanoCustoms), key=_nanoCustoms.index))
1816         # replace empty sequence with default
1817         _nanoSeq = [seq if seq!='' else f"{self.NANODefaultCFF}.{self.NANODefaultSeq}" for seq in _nanoSeq]
1818         _nanoCustoms = [cust if cust!='' else self.NANODefaultCustom for cust in _nanoCustoms]
1819         # build and inject the sequence
1820         if len(_nanoSeq) < 1 and '@' in stepSpec:
1821             raise Exception(f'The specified mapping: {stepSpec} generates an empty NANO sequence. Please provide a valid mapping')
1822         _seqToSchedule = []
1823         for _subSeq in _nanoSeq:
1824             if '.' in _subSeq:
1825                 _cff,_seq = _subSeq.split('.')
1826                 print("NANO: scheduling:",_seq,"from",_cff)
1827                 self.loadAndRemember(_cff)
1828                 _seqToSchedule.append(_seq)
1829             elif '/' in _subSeq:
1830                 self.loadAndRemember(_subSeq)
1831                 _seqToSchedule.append(self.NANODefaultSeq)
1832             else:
1833                 print("NANO: scheduling:",_subSeq)
1834                 _seqToSchedule.append(_subSeq)
1835         self.scheduleSequence('+'.join(_seqToSchedule), 'nanoAOD_step')
1836 
1837         # add the customisations
1838         for custom in _nanoCustoms:
1839             custom_path = custom if '.' in custom else '.'.join([_nanoCff,custom])
1840             # customization order can be important for NANO, here later specified customise take precedence
1841             self._options.customisation_file.append(custom_path)
1842         if self._options.hltProcess:
1843             if len(self._options.customise_commands) > 1:
1844                 self._options.customise_commands = self._options.customise_commands + " \n"
1845             self._options.customise_commands = self._options.customise_commands + "process.unpackedPatTrigger.triggerResults= cms.InputTag( 'TriggerResults::"+self._options.hltProcess+"' )\n"
1846 
1847     def prepare_NANOGEN(self, stepSpec = "nanoAOD"):
1848         ''' Enrich the schedule with NANOGEN '''
1849         # TODO: Need to modify this based on the input file type
1850         fromGen = any([x in self.stepMap for x in ['LHE', 'GEN', 'AOD']])
1851         _,_nanogenSeq,_nanogenCff = self.loadDefaultOrSpecifiedCFF(stepSpec,self.NANOGENDefaultCFF)
1852         self.scheduleSequence(_nanogenSeq,'nanoAOD_step')
1853         custom = "customizeNanoGEN" if fromGen else "customizeNanoGENFromMini"
1854         if self._options.runUnscheduled:
1855             self._options.customisation_file_unsch.insert(0, '.'.join([_nanogenCff, custom]))
1856         else:
1857             self._options.customisation_file.insert(0, '.'.join([_nanogenCff, custom]))
1858 
1859     def prepare_SKIM(self, stepSpec = "all"):
1860         ''' Enrich the schedule with skimming fragments'''
1861         skimConfig,sequence,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.SKIMDefaultCFF)
1862 
1863         stdHLTProcName = 'HLT'
1864         newHLTProcName = self._options.hltProcess
1865         customiseForReHLT = (newHLTProcName or (stdHLTProcName in self.stepMap)) and (newHLTProcName != stdHLTProcName)
1866         if customiseForReHLT:
1867             print("replacing %s process name - step SKIM:%s will use '%s'" % (stdHLTProcName, sequence, newHLTProcName))
1868 
1869         ## support @Mu+DiJet+@Electron configuration via autoSkim.py
1870         from Configuration.Skimming.autoSkim import autoSkim
1871         skimlist = sequence.split('+')
1872         self.expandMapping(skimlist,autoSkim)
1873 
1874         #print("dictionary for skims:", skimConfig.__dict__)
1875         for skim in skimConfig.__dict__:
1876             skimstream = getattr(skimConfig, skim)
1877 
1878             # blacklist AlCa paths so that they do not appear in the cfg
1879             if isinstance(skimstream, cms.Path):
1880                 self.blacklist_paths.append(skimstream)
1881             # if enabled, apply "hltProcess" renaming to Sequences
1882             elif isinstance(skimstream, cms.Sequence):
1883                 if customiseForReHLT:
1884                     self.renameHLTprocessInSequence(skim, proc = newHLTProcName, HLTprocess = stdHLTProcName, verbosityLevel = 0)
1885 
1886             if not isinstance(skimstream, cms.FilteredStream):
1887                 continue
1888 
1889             shortname = skim.replace('SKIMStream','')
1890             if (sequence=="all"):
1891                 self.addExtraStream(skim,skimstream)
1892             elif (shortname in skimlist):
1893                 self.addExtraStream(skim,skimstream)
1894                 #add a DQM eventcontent for this guy
1895                 if self._options.datatier=='DQM':
1896                     self.process.load(self.EVTCONTDefaultCFF)
1897                     skimstreamDQM = cms.FilteredStream(
1898                             responsible = skimstream.responsible,
1899                             name = skimstream.name+'DQM',
1900                             paths = skimstream.paths,
1901                             selectEvents = skimstream.selectEvents,
1902                             content = self._options.datatier+'EventContent',
1903                             dataTier = cms.untracked.string(self._options.datatier)
1904                             )
1905                     self.addExtraStream(skim+'DQM',skimstreamDQM)
1906                 for i in range(skimlist.count(shortname)):
1907                     skimlist.remove(shortname)
1908 
1909         if (skimlist.__len__()!=0 and sequence!="all"):
1910             print('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
1911             raise Exception('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
1912 
1913 
1914     def prepare_USER(self, stepSpec = None):
1915         ''' Enrich the schedule with a user defined sequence '''
1916         _,_userSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.USERDefaultCFF)
1917         self.scheduleSequence(_userSeq,'user_step')
1918         return
1919 
1920     def prepare_POSTRECO(self, stepSpec = None):
1921         """ Enrich the schedule with the postreco step """
1922         self.loadAndRemember(self.POSTRECODefaultCFF)
1923         self.scheduleSequence('postreco_generator','postreco_step')
1924         return
1925 
1926 
1927     def prepare_VALIDATION(self, stepSpec = 'validation'):
1928         print(f"{stepSpec} in preparing validation")
1929         _,sequence,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.VALIDATIONDefaultCFF)
1930         from Validation.Configuration.autoValidation import autoValidation
1931         #in case VALIDATION:something:somethingelse -> something,somethingelse
1932         if sequence.find(',')!=-1:
1933             prevalSeqName=sequence.split(',')[0].split('+')
1934             valSeqName=sequence.split(',')[1].split('+')
1935             self.expandMapping(prevalSeqName,autoValidation,index=0)
1936             self.expandMapping(valSeqName,autoValidation,index=1)
1937         else:
1938             if '@' in sequence:
1939                 prevalSeqName=sequence.split('+')
1940                 valSeqName=sequence.split('+')
1941                 self.expandMapping(prevalSeqName,autoValidation,index=0)
1942                 self.expandMapping(valSeqName,autoValidation,index=1)
1943             else:
1944                 postfix=''
1945                 if sequence:
1946                     postfix='_'+sequence
1947                 prevalSeqName=['prevalidation'+postfix]
1948                 valSeqName=['validation'+postfix]
1949                 if not hasattr(self.process,valSeqName[0]):
1950                     prevalSeqName=['']
1951                     valSeqName=[sequence]
1952 
1953         def NFI(index):
1954             ##name from index, required to keep backward compatibility
1955             if index==0:
1956                 return ''
1957             else:
1958                 return '%s'%index
1959 
1960 
1961         #rename the HLT process in validation steps
1962         if ('HLT' in self.stepMap and not self._options.fast) or self._options.hltProcess:
1963             for s in valSeqName+prevalSeqName:
1964                 if s:
1965                     self.renameHLTprocessInSequence(s)
1966         for (i,s) in enumerate(prevalSeqName):
1967             if s:
1968                 setattr(self.process,'prevalidation_step%s'%NFI(i),  cms.Path( getattr(self.process, s)) )
1969                 self.schedule.append(getattr(self.process,'prevalidation_step%s'%NFI(i)))
1970 
1971         for (i,s) in enumerate(valSeqName):
1972             setattr(self.process,'validation_step%s'%NFI(i), cms.EndPath( getattr(self.process, s)))
1973             self.schedule.append(getattr(self.process,'validation_step%s'%NFI(i)))
1974 
1975         #needed in case the miniAODValidation sequence is run starting from AODSIM
1976         if 'PAT' in self.stepMap and not 'RECO' in self.stepMap:
1977             return
1978 
1979         if not 'DIGI' in self.stepMap and not self._options.fast and not any(map( lambda s : s.startswith('genvalid'), valSeqName)):
1980             if self._options.restoreRNDSeeds==False and not self._options.restoreRNDSeeds==True:
1981                 self._options.restoreRNDSeeds=True
1982 
1983         if not 'DIGI' in self.stepMap and not self._options.isData and not self._options.fast:
1984             self.executeAndRemember("process.mix.playback = True")
1985             self.executeAndRemember("process.mix.digitizers = cms.PSet()")
1986             self.executeAndRemember("for a in process.aliases: delattr(process, a)")
1987             self._options.customisation_file.append("SimGeneral/MixingModule/fullMixCustomize_cff.setCrossingFrameOn")
1988 
1989         if hasattr(self.process,"genstepfilter") and len(self.process.genstepfilter.triggerConditions):
1990             #will get in the schedule, smoothly
1991             for (i,s) in enumerate(valSeqName):
1992                 getattr(self.process,'validation_step%s'%NFI(i)).insert(0, self.process.genstepfilter)
1993 
1994         return
1995 
1996 
1997     class MassSearchReplaceProcessNameVisitor(object):
1998         """Visitor that travels within a cms.Sequence, looks for a parameter and replace its value
1999         It will climb down within PSets, VPSets and VInputTags to find its target"""
2000         def __init__(self, paramSearch, paramReplace, verbose=False, whitelist=()):
2001             self._paramReplace = paramReplace
2002             self._paramSearch = paramSearch
2003             self._verbose = verbose
2004             self._whitelist = whitelist
2005 
2006         def doIt(self, pset, base):
2007             if isinstance(pset, cms._Parameterizable):
2008                 for name in pset.parameters_().keys():
2009                     # skip whitelisted parameters
2010                     if name in self._whitelist:
2011                         continue
2012                     # if I use pset.parameters_().items() I get copies of the parameter values
2013                     # so I can't modify the nested pset
2014                     value = getattr(pset, name)
2015                     valueType = type(value)
2016                     if valueType in [cms.PSet, cms.untracked.PSet, cms.EDProducer]:
2017                         self.doIt(value,base+"."+name)
2018                     elif valueType in [cms.VPSet, cms.untracked.VPSet]:
2019                         for (i,ps) in enumerate(value): self.doIt(ps, "%s.%s[%d]"%(base,name,i) )
2020                     elif valueType in [cms.string, cms.untracked.string]:
2021                         if value.value() == self._paramSearch:
2022                             if self._verbose: print("set string process name %s.%s %s ==> %s"% (base, name, value, self._paramReplace))
2023                             setattr(pset, name,self._paramReplace)
2024                     elif valueType in [cms.VInputTag, cms.untracked.VInputTag]:
2025                         for (i,n) in enumerate(value):
2026                             if not isinstance(n, cms.InputTag):
2027                                 n=cms.InputTag(n)
2028                             if n.processName == self._paramSearch:
2029                                 # VInputTag can be declared as a list of strings, so ensure that n is formatted correctly
2030                                 if self._verbose:print("set process name %s.%s[%d] %s ==> %s " % (base, name, i, n, self._paramReplace))
2031                                 setattr(n,"processName",self._paramReplace)
2032                                 value[i]=n
2033                     elif valueType in [cms.vstring, cms.untracked.vstring]:
2034                         for (i,n) in enumerate(value):
2035                             if n==self._paramSearch:
2036                                 getattr(pset,name)[i]=self._paramReplace
2037                     elif valueType in [cms.InputTag, cms.untracked.InputTag]:
2038                         if value.processName == self._paramSearch:
2039                             if self._verbose: print("set process name %s.%s %s ==> %s " % (base, name, value, self._paramReplace))
2040                             setattr(getattr(pset, name),"processName",self._paramReplace)
2041 
2042         def enter(self,visitee):
2043             label = ''
2044             try:
2045                 label = visitee.label()
2046             except AttributeError:
2047                 label = '<Module not in a Process>'
2048             except:
2049                 label = 'other execption'
2050             self.doIt(visitee, label)
2051 
2052         def leave(self,visitee):
2053             pass
2054 
2055     #visit a sequence to repalce all input tags
2056     def renameInputTagsInSequence(self,sequence,oldT="rawDataCollector",newT="rawDataRepacker"):
2057         print("Replacing all InputTag %s => %s"%(oldT,newT))
2058         from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag
2059         massSearchReplaceAnyInputTag(getattr(self.process,sequence),oldT,newT)
2060         loadMe='from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag'
2061         if not loadMe in self.additionalCommands:
2062             self.additionalCommands.append(loadMe)
2063         self.additionalCommands.append('massSearchReplaceAnyInputTag(process.%s,"%s","%s",False,True)'%(sequence,oldT,newT))
2064 
2065     #change the process name used to address HLT results in any sequence
2066     def renameHLTprocessInSequence(self, sequence, proc=None, HLTprocess='HLT', verbosityLevel=1):
2067         if proc == None:
2068             proc = self._options.hltProcess if self._options.hltProcess else self.process.name_()
2069         if proc == HLTprocess:
2070             return
2071         # look up all module in sequence
2072         if verbosityLevel > 0:
2073             print("replacing %s process name - sequence %s will use '%s'" % (HLTprocess, sequence, proc))
2074         verboseVisit = (verbosityLevel > 1)
2075         getattr(self.process,sequence).visit(
2076             ConfigBuilder.MassSearchReplaceProcessNameVisitor(HLTprocess, proc, whitelist = ("subSystemFolder",), verbose = verboseVisit))
2077         if 'from Configuration.Applications.ConfigBuilder import ConfigBuilder' not in self.additionalCommands:
2078             self.additionalCommands.append('from Configuration.Applications.ConfigBuilder import ConfigBuilder')
2079         self.additionalCommands.append(
2080             'process.%s.visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor("%s", "%s", whitelist = ("subSystemFolder",), verbose = %s))'
2081             % (sequence, HLTprocess, proc, verboseVisit))
2082 
2083     def expandMapping(self,seqList,mapping,index=None):
2084         maxLevel=30
2085         level=0
2086         while '@' in repr(seqList) and level<maxLevel:
2087             level+=1
2088             for specifiedCommand in seqList:
2089                 if specifiedCommand.startswith('@'):
2090                     location=specifiedCommand[1:]
2091                     if not location in mapping:
2092                         raise Exception("Impossible to map "+location+" from "+repr(mapping))
2093                     mappedTo=mapping[location]
2094                     if index!=None:
2095                         mappedTo=mappedTo[index]
2096                     seqList.remove(specifiedCommand)
2097                     seqList.extend(mappedTo.split('+'))
2098                     break;
2099         if level==maxLevel:
2100             raise Exception("Could not fully expand "+repr(seqList)+" from "+repr(mapping))
2101 
2102     def prepare_DQM(self, stepSpec = 'DQMOffline'):
2103         # this one needs replacement
2104 
2105         # any 'DQM' job should use DQMStore in non-legacy mode (but not HARVESTING)
2106         self.loadAndRemember("DQMServices/Core/DQMStoreNonLegacy_cff")
2107         _,_dqmSeq,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.DQMOFFLINEDefaultCFF)
2108         sequenceList=_dqmSeq.split('+')
2109         postSequenceList=_dqmSeq.split('+')
2110         from DQMOffline.Configuration.autoDQM import autoDQM
2111         self.expandMapping(sequenceList,autoDQM,index=0)
2112         self.expandMapping(postSequenceList,autoDQM,index=1)
2113 
2114         if len(set(sequenceList))!=len(sequenceList):
2115             sequenceList=list(OrderedSet(sequenceList))
2116             print("Duplicate entries for DQM:, using",sequenceList)
2117 
2118         pathName='dqmoffline_step'
2119         for (i,_sequence) in enumerate(sequenceList):
2120             if (i!=0):
2121                 pathName='dqmoffline_%d_step'%(i)
2122 
2123             if 'HLT' in self.stepMap.keys() or self._options.hltProcess:
2124                 self.renameHLTprocessInSequence(_sequence)
2125 
2126             setattr(self.process,pathName, cms.EndPath( getattr(self.process,_sequence ) ) )
2127             self.schedule.append(getattr(self.process,pathName))
2128 
2129             if hasattr(self.process,"genstepfilter") and len(self.process.genstepfilter.triggerConditions):
2130                 #will get in the schedule, smoothly
2131                 getattr(self.process,pathName).insert(0,self.process.genstepfilter)
2132 
2133 
2134         pathName='dqmofflineOnPAT_step'
2135         for (i,_sequence) in enumerate(postSequenceList):
2136             #Fix needed to avoid duplication of sequences not defined in autoDQM or without a PostDQM
2137             if (sequenceList[i]==postSequenceList[i]):
2138                       continue
2139             if (i!=0):
2140                 pathName='dqmofflineOnPAT_%d_step'%(i)
2141 
2142             setattr(self.process,pathName, cms.EndPath( getattr(self.process, _sequence ) ) )
2143             self.schedule.append(getattr(self.process,pathName))
2144 
2145     def prepare_HARVESTING(self, stepSpec = None):
2146         """ Enrich the process with harvesting step """
2147         self.DQMSaverCFF='Configuration/StandardSequences/DQMSaver'+self._options.harvesting+'_cff'
2148         self.loadAndRemember(self.DQMSaverCFF)
2149 
2150         harvestingConfig,sequence,_ = self.loadDefaultOrSpecifiedCFF(stepSpec,self.HARVESTINGDefaultCFF)
2151 
2152         # decide which HARVESTING paths to use
2153         harvestingList = sequence.split("+")
2154         from DQMOffline.Configuration.autoDQM import autoDQM
2155         from Validation.Configuration.autoValidation import autoValidation
2156         import copy
2157         combined_mapping = copy.deepcopy( autoDQM )
2158         combined_mapping.update( autoValidation )
2159         self.expandMapping(harvestingList,combined_mapping,index=-1)
2160 
2161         if len(set(harvestingList))!=len(harvestingList):
2162             harvestingList=list(OrderedSet(harvestingList))
2163             print("Duplicate entries for HARVESTING, using",harvestingList)
2164 
2165         for name in harvestingList:
2166             if not name in harvestingConfig.__dict__:
2167                 print(name,"is not a possible harvesting type. Available are",harvestingConfig.__dict__.keys())
2168                 # trigger hard error, like for other sequence types
2169                 getattr(self.process, name)
2170                 continue
2171             harvestingstream = getattr(harvestingConfig,name)
2172             if isinstance(harvestingstream,cms.Path):
2173                 self.schedule.append(harvestingstream)
2174                 self.blacklist_paths.append(harvestingstream)
2175             if isinstance(harvestingstream,cms.Sequence):
2176                 setattr(self.process,name+"_step",cms.Path(harvestingstream))
2177                 self.schedule.append(getattr(self.process,name+"_step"))
2178 
2179         # # NOTE: the "hltProcess" option currently does nothing in the HARVEST step
2180         # if self._options.hltProcess or ('HLT' in self.stepMap):
2181         #     pass
2182 
2183         self.scheduleSequence('DQMSaver','dqmsave_step')
2184         return
2185 
2186     def prepare_ALCAHARVEST(self, stepSpec = None):
2187         """ Enrich the process with AlCaHarvesting step """
2188         harvestingConfig = self.loadAndRemember(self.ALCAHARVESTDefaultCFF)
2189         sequence=stepSpec.split(".")[-1]
2190 
2191         # decide which AlcaHARVESTING paths to use
2192         harvestingList = sequence.split("+")
2193 
2194 
2195 
2196         from Configuration.AlCa.autoPCL import autoPCL
2197         self.expandMapping(harvestingList,autoPCL)
2198 
2199         for name in harvestingConfig.__dict__:
2200             harvestingstream = getattr(harvestingConfig,name)
2201             if name in harvestingList and isinstance(harvestingstream,cms.Path):
2202                 self.schedule.append(harvestingstream)
2203                 if isinstance(getattr(harvestingConfig,"ALCAHARVEST" + name + "_dbOutput"), cms.VPSet) and \
2204                    isinstance(getattr(harvestingConfig,"ALCAHARVEST" + name + "_metadata"), cms.VPSet):
2205                     self.executeAndRemember("process.PoolDBOutputService.toPut.extend(process.ALCAHARVEST" + name + "_dbOutput)")
2206                     self.executeAndRemember("process.pclMetadataWriter.recordsToMap.extend(process.ALCAHARVEST" + name + "_metadata)")
2207                 else:
2208                     self.executeAndRemember("process.PoolDBOutputService.toPut.append(process.ALCAHARVEST" + name + "_dbOutput)")
2209                     self.executeAndRemember("process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVEST" + name + "_metadata)")
2210                 harvestingList.remove(name)
2211         # append the common part at the end of the sequence
2212         lastStep = getattr(harvestingConfig,"ALCAHARVESTDQMSaveAndMetadataWriter")
2213         self.schedule.append(lastStep)
2214 
2215         if len(harvestingList) != 0 and 'dummyHarvesting' not in harvestingList :
2216             print("The following harvesting could not be found : ", harvestingList)
2217             raise Exception("The following harvesting could not be found : "+str(harvestingList))
2218 
2219 
2220 
2221     def prepare_ENDJOB(self, stepSpec = 'endOfProcess'):
2222         _,_endjobSeq,_=self.loadDefaultOrSpecifiedCFF(stepSpec,self.ENDJOBDefaultCFF)
2223         self.scheduleSequenceAtEnd(_endjobSeq,'endjob_step')
2224         return
2225 
2226     def finalizeFastSimHLT(self):
2227         self.process.reconstruction = cms.Path(self.process.reconstructionWithFamos)
2228         self.schedule.append(self.process.reconstruction)
2229 
2230 
2231     def build_production_info(self, evt_type, evtnumber):
2232         """ Add useful info for the production. """
2233         self.process.configurationMetadata=cms.untracked.PSet\
2234                                             (version=cms.untracked.string("$Revision: 1.19 $"),
2235                                              name=cms.untracked.string("Applications"),
2236                                              annotation=cms.untracked.string(evt_type+ " nevts:"+str(evtnumber))
2237                                              )
2238 
2239         self.addedObjects.append(("Production Info","configurationMetadata"))
2240 
2241 
2242     def create_process(self):
2243         self.pythonCfgCode =  "# Auto generated configuration file\n"
2244         self.pythonCfgCode += "# using: \n# "+__version__[1:-1]+"\n# "+__source__[1:-1]+'\n'
2245         self.pythonCfgCode += "# with command line options: "+self._options.arguments+'\n'
2246         self.pythonCfgCode += "import FWCore.ParameterSet.Config as cms\n\n"
2247 
2248         # now set up the modifies
2249         modifiers=[]
2250         modifierStrings=[]
2251         modifierImports=[]
2252 
2253         if hasattr(self._options,"era") and self._options.era :
2254         # Multiple eras can be specified in a comma seperated list
2255             from Configuration.StandardSequences.Eras import eras
2256             for requestedEra in self._options.era.split(",") :
2257                 modifierStrings.append(requestedEra)
2258                 modifierImports.append(eras.pythonCfgLines[requestedEra])
2259                 modifiers.append(getattr(eras,requestedEra))
2260 
2261 
2262         if hasattr(self._options,"procModifiers") and self._options.procModifiers:
2263             import importlib
2264             thingsImported=[]
2265             for c in self._options.procModifiers:
2266                 thingsImported.extend(c.split(","))
2267             for pm in thingsImported:
2268                 modifierStrings.append(pm)
2269                 modifierImports.append('from Configuration.ProcessModifiers.'+pm+'_cff import '+pm)
2270                 modifiers.append(getattr(importlib.import_module('Configuration.ProcessModifiers.'+pm+'_cff'),pm))
2271 
2272         self.pythonCfgCode += '\n'.join(modifierImports)+'\n\n'
2273         self.pythonCfgCode += "process = cms.Process('"+self._options.name+"'" # Start of the line, finished after the loop
2274 
2275 
2276         if len(modifierStrings)>0:
2277             self.pythonCfgCode+= ','+','.join(modifierStrings)
2278         self.pythonCfgCode+=')\n\n'
2279 
2280         #yes, the cfg code gets out of sync here if a process is passed in. That could be fixed in the future
2281         #assuming there is some way for the fwk to get the list of modifiers (and their stringified name)
2282         if self.process == None:
2283             if len(modifiers)>0:
2284                 self.process = cms.Process(self._options.name,*modifiers)
2285             else:
2286                 self.process = cms.Process(self._options.name)
2287 
2288 
2289 
2290 
2291     def prepare(self, doChecking = False):
2292         """ Prepare the configuration string and add missing pieces."""
2293 
2294         self.loadAndRemember(self.EVTCONTDefaultCFF)  #load the event contents regardless
2295         self.addMaxEvents()
2296         if self.with_input:
2297             self.addSource()
2298         self.addStandardSequences()
2299         ##adding standard sequences might change the inputEventContent option and therefore needs to be finalized after
2300         self.completeInputCommand()
2301         self.addConditions()
2302 
2303 
2304         outputModuleCfgCode=""
2305         if not 'HARVESTING' in self.stepMap.keys() and not 'ALCAHARVEST' in self.stepMap.keys() and not 'ALCAOUTPUT' in self.stepMap.keys() and self.with_output:
2306             outputModuleCfgCode=self.addOutput()
2307 
2308         self.addCommon()
2309 
2310         self.pythonCfgCode += "# import of standard configurations\n"
2311         for module in self.imports:
2312             self.pythonCfgCode += ("process.load('"+module+"')\n")
2313 
2314         # production info
2315         if not hasattr(self.process,"configurationMetadata"):
2316             self.build_production_info(self._options.evt_type, self._options.number)
2317         else:
2318             #the PSet was added via a load
2319             self.addedObjects.append(("Production Info","configurationMetadata"))
2320 
2321         self.pythonCfgCode +="\n"
2322         for comment,object in self.addedObjects:
2323             if comment!="":
2324                 self.pythonCfgCode += "\n# "+comment+"\n"
2325             self.pythonCfgCode += dumpPython(self.process,object)
2326 
2327         # dump the output definition
2328         self.pythonCfgCode += "\n# Output definition\n"
2329         self.pythonCfgCode += outputModuleCfgCode
2330 
2331         # dump all additional outputs (e.g. alca or skim streams)
2332         self.pythonCfgCode += "\n# Additional output definition\n"
2333         #I do not understand why the keys are not normally ordered.
2334         nl=sorted(self.additionalOutputs.keys())
2335         for name in nl:
2336             output = self.additionalOutputs[name]
2337             self.pythonCfgCode += "process.%s = %s" %(name, output.dumpPython())
2338             tmpOut = cms.EndPath(output)
2339             setattr(self.process,name+'OutPath',tmpOut)
2340             self.schedule.append(tmpOut)
2341 
2342         # dump all additional commands
2343         self.pythonCfgCode += "\n# Other statements\n"
2344         for command in self.additionalCommands:
2345             self.pythonCfgCode += command + "\n"
2346 
2347         #comma separated list of objects that deserve to be inlined in the configuration (typically from a modified config deep down)
2348         for object in self._options.inlineObjects.split(','):
2349             if not object:
2350                 continue
2351             if not hasattr(self.process,object):
2352                 print('cannot inline -'+object+'- : not known')
2353             else:
2354                 self.pythonCfgCode +='\n'
2355                 self.pythonCfgCode +=dumpPython(self.process,object)
2356 
2357         if self._options.pileup=='HiMixEmbGEN':
2358             self.pythonCfgCode += "\nprocess.generator.embeddingMode=cms.int32(1)\n"
2359 
2360         # dump all paths
2361         self.pythonCfgCode += "\n# Path and EndPath definitions\n"
2362         for path in self.process.paths:
2363             if getattr(self.process,path) not in self.blacklist_paths:
2364                 self.pythonCfgCode += dumpPython(self.process,path)
2365 
2366         for endpath in self.process.endpaths:
2367             if getattr(self.process,endpath) not in self.blacklist_paths:
2368                 self.pythonCfgCode += dumpPython(self.process,endpath)
2369 
2370         # dump the schedule
2371         self.pythonCfgCode += "\n# Schedule definition\n"
2372 
2373         # handling of the schedule
2374         pathNames = ['process.'+p.label_() for p in self.schedule]
2375         if self.process.schedule == None:
2376             self.process.schedule = cms.Schedule()
2377             for item in self.schedule:
2378                 self.process.schedule.append(item)
2379             result = 'process.schedule = cms.Schedule('+','.join(pathNames)+')\n'
2380         else:
2381             if not isinstance(self.scheduleIndexOfFirstHLTPath, int):
2382                 raise Exception('the schedule was imported from a cff in HLTrigger.Configuration, but the final index of the first HLT path is undefined')
2383 
2384             for index, item in enumerate(self.schedule):
2385                 if index < self.scheduleIndexOfFirstHLTPath:
2386                     self.process.schedule.insert(index, item)
2387                 else:
2388                     self.process.schedule.append(item)
2389 
2390             result = "# process.schedule imported from cff in HLTrigger.Configuration\n"
2391             for index, item in enumerate(pathNames[:self.scheduleIndexOfFirstHLTPath]):
2392                 result += 'process.schedule.insert('+str(index)+', '+item+')\n'
2393             if self.scheduleIndexOfFirstHLTPath < len(pathNames):
2394                 result += 'process.schedule.extend(['+','.join(pathNames[self.scheduleIndexOfFirstHLTPath:])+'])\n'
2395 
2396         self.pythonCfgCode += result
2397 
2398         for labelToAssociate in self.labelsToAssociate:
2399             self.process.schedule.associate(getattr(self.process, labelToAssociate))
2400             self.pythonCfgCode += 'process.schedule.associate(process.' + labelToAssociate + ')\n'
2401 
2402         from PhysicsTools.PatAlgos.tools.helpers import associatePatAlgosToolsTask
2403         associatePatAlgosToolsTask(self.process)
2404         self.pythonCfgCode+="from PhysicsTools.PatAlgos.tools.helpers import associatePatAlgosToolsTask\n"
2405         self.pythonCfgCode+="associatePatAlgosToolsTask(process)\n"
2406 
2407         overrideThreads = (self._options.nThreads != 1)
2408         overrideConcurrentLumis = (self._options.nConcurrentLumis != defaultOptions.nConcurrentLumis)
2409         overrideConcurrentIOVs = (self._options.nConcurrentIOVs != defaultOptions.nConcurrentIOVs)
2410 
2411         if overrideThreads or overrideConcurrentLumis or overrideConcurrentIOVs:
2412             self.pythonCfgCode +="\n"
2413             self.pythonCfgCode +="#Setup FWK for multithreaded\n"
2414             if overrideThreads:
2415                 self.pythonCfgCode +="process.options.numberOfThreads = {}\n".format(self._options.nThreads)
2416                 self.pythonCfgCode +="process.options.numberOfStreams = {}\n".format(self._options.nStreams)
2417                 self.process.options.numberOfThreads = self._options.nThreads
2418                 self.process.options.numberOfStreams = self._options.nStreams
2419             if overrideConcurrentLumis:
2420                 self.pythonCfgCode +="process.options.numberOfConcurrentLuminosityBlocks = {}\n".format(self._options.nConcurrentLumis)
2421                 self.process.options.numberOfConcurrentLuminosityBlocks = self._options.nConcurrentLumis
2422             if overrideConcurrentIOVs:
2423                 self.pythonCfgCode +="process.options.eventSetup.numberOfConcurrentIOVs = {}\n".format(self._options.nConcurrentIOVs)
2424                 self.process.options.eventSetup.numberOfConcurrentIOVs = self._options.nConcurrentIOVs
2425 
2426         if self._options.accelerators is not None:
2427             accelerators = self._options.accelerators.split(',')
2428             self.pythonCfgCode += "\n"
2429             self.pythonCfgCode += "# Enable only these accelerator backends\n"
2430             self.pythonCfgCode += "process.load('Configuration.StandardSequences.Accelerators_cff')\n"
2431             self.pythonCfgCode += "process.options.accelerators = ['" + "', '".join(accelerators) + "']\n"
2432             self.process.load('Configuration.StandardSequences.Accelerators_cff')
2433             self.process.options.accelerators = accelerators
2434 
2435         #repacked version
2436         if self._options.isRepacked:
2437             self.pythonCfgCode +="\n"
2438             self.pythonCfgCode +="from Configuration.Applications.ConfigBuilder import MassReplaceInputTag\n"
2439             self.pythonCfgCode +="MassReplaceInputTag(process, new=\"rawDataMapperByLabel\", old=\"rawDataCollector\")\n"
2440             MassReplaceInputTag(self.process, new="rawDataMapperByLabel", old="rawDataCollector")
2441 
2442         # special treatment in case of production filter sequence 2/2
2443         if self.productionFilterSequence and not (self._options.pileup=='HiMixEmbGEN'):
2444             self.pythonCfgCode +='# filter all path with the production filter sequence\n'
2445             self.pythonCfgCode +='for path in process.paths:\n'
2446             if len(self.conditionalPaths):
2447                 self.pythonCfgCode +='\tif not path in %s: continue\n'%str(self.conditionalPaths)
2448             if len(self.excludedPaths):
2449                 self.pythonCfgCode +='\tif path in %s: continue\n'%str(self.excludedPaths)
2450             self.pythonCfgCode +='\tgetattr(process,path).insert(0, process.%s)\n'%(self.productionFilterSequence,)
2451             pfs = getattr(self.process,self.productionFilterSequence)
2452             for path in self.process.paths:
2453                 if not path in self.conditionalPaths: continue
2454                 if path in self.excludedPaths: continue
2455                 getattr(self.process,path).insert(0, pfs)
2456 
2457 
2458         # dump customise fragment
2459         self.pythonCfgCode += self.addCustomise()
2460 
2461         if self._options.runUnscheduled:
2462             print("--runUnscheduled is deprecated and not necessary anymore, and will be removed soon. Please update your command line.")
2463         # Keep the "unscheduled customise functions" separate for now,
2464         # there are customize functions given by users (in our unit
2465         # tests) that need to be run before the "unscheduled customise
2466         # functions"
2467         self.pythonCfgCode += self.addCustomise(1)
2468 
2469         self.pythonCfgCode += self.addCustomiseCmdLine()
2470 
2471         if hasattr(self.process,"logErrorHarvester"):
2472             #configure logErrorHarvester to wait for same EDProducers to finish as the OutputModules
2473             self.pythonCfgCode +="\n#Have logErrorHarvester wait for the same EDProducers to finish as those providing data for the OutputModule\n"
2474             self.pythonCfgCode +="from FWCore.Modules.logErrorHarvester_cff import customiseLogErrorHarvesterUsingOutputCommands\n"
2475             self.pythonCfgCode +="process = customiseLogErrorHarvesterUsingOutputCommands(process)\n"
2476             from FWCore.Modules.logErrorHarvester_cff import customiseLogErrorHarvesterUsingOutputCommands
2477             self.process = customiseLogErrorHarvesterUsingOutputCommands(self.process)
2478 
2479         # Temporary hack to put the early delete customization after
2480         # everything else
2481         #
2482         # FIXME: remove when no longer needed
2483         self.pythonCfgCode += "\n# Add early deletion of temporary data products to reduce peak memory need\n"
2484         self.pythonCfgCode += "from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete\n"
2485         self.pythonCfgCode += "process = customiseEarlyDelete(process)\n"
2486         self.pythonCfgCode += "# End adding early deletion\n"
2487         from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete
2488         self.process = customiseEarlyDelete(self.process)
2489 
2490         imports = cms.specialImportRegistry.getSpecialImports()
2491         if len(imports) > 0:
2492             #need to inject this at the top
2493             index = self.pythonCfgCode.find("import FWCore.ParameterSet.Config")
2494             #now find the end of line
2495             index = self.pythonCfgCode.find("\n",index)
2496             self.pythonCfgCode = self.pythonCfgCode[:index]+ "\n" + "\n".join(imports)+"\n" +self.pythonCfgCode[index:]
2497 
2498 
2499         # make the .io file
2500 
2501         if self._options.io:
2502             #io=open(self._options.python_filename.replace('.py','.io'),'w')
2503             if not self._options.io.endswith('.io'): self._option.io+='.io'
2504             io=open(self._options.io,'w')
2505             ioJson={}
2506             if hasattr(self.process.source,"fileNames"):
2507                 if len(self.process.source.fileNames.value()):
2508                     ioJson['primary']=self.process.source.fileNames.value()
2509             if hasattr(self.process.source,"secondaryFileNames"):
2510                 if len(self.process.source.secondaryFileNames.value()):
2511                     ioJson['secondary']=self.process.source.secondaryFileNames.value()
2512             if self._options.pileup_input and (self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:')):
2513                 ioJson['pileup']=self._options.pileup_input[4:]
2514             for (o,om) in self.process.outputModules_().items():
2515                 ioJson[o]=om.fileName.value()
2516             ioJson['GT']=self.process.GlobalTag.globaltag.value()
2517             if self.productionFilterSequence:
2518                 ioJson['filter']=self.productionFilterSequence
2519             import json
2520             io.write(json.dumps(ioJson))
2521         return
2522