Applications/python/cmsDriverOptions.py

0001 #! /usr/bin/env python3
0002
0003 # A Pyrelval Wrapper
0004
0005 import optparse
0006 import sys
0007 import os
0008 import re
0009 import Configuration.Applications
0010 from Configuration.Applications.ConfigBuilder import ConfigBuilder, defaultOptions
0011 import traceback
0012 from functools import reduce
0013
0014 def checkModifier(era):
0015     from FWCore.ParameterSet.Config import Modifier, ModifierChain
0016     return isinstance( era, Modifier ) or isinstance( era, ModifierChain )
0017
0018 def checkOptions():
0019     return
0020
0021 def adaptOptions():
0022     return
0023
0024 def OptionsFromCommand(command):
0025     items=command.split()
0026     if items[0] != 'cmsDriver.py':
0027         return None
0028     items.append('--evt_type')
0029     items.append(items[1])
0030     options=OptionsFromItems(items[2:])
0031     options.arguments = command
0032     return options
0033
0034 def OptionsFromCommandLine():
0035     import sys
0036     options=OptionsFromItems(sys.argv[1:])
0037     # memorize the command line arguments
0038     options.arguments = reduce(lambda x, y: x+' '+y, sys.argv[1:])
0039     return options
0040
0041 def OptionsFromItems(items):
0042     import sys
0043     from Configuration.Applications.Options import parser
0044
0045     options = parser.parse_args(items)
0046
0047     if options.conditions=="help":
0048         from Configuration.AlCa import autoCond
0049         possible=""
0050         for k in autoCond.autoCond:
0051             possible+="\nauto:"+k+" -> "+str(autoCond.autoCond[k])
0052         parser.error("Possibilities for the --conditions option: "+possible)
0053
0054
0055     #################################
0056     # Check parameters for validity #
0057     #################################
0058
0059     # check in case of ALCAOUTPUT case for alca splitting
0060     if options.triggerResultsProcess == None and "ALCAOUTPUT" in options.step:
0061         print("ERROR: If ALCA splitting is requested, the name of the process in which the alca producers ran needs to be specified. E.g. via --triggerResultsProcess RECO")
0062         sys.exit(1)
0063
0064     if not options.evt_type:
0065         options.evt_type=sys.argv[1]
0066
0067     #now adjust the given parameters before passing it to the ConfigBuilder
0068
0069     #trail a "/" to dirin and dirout
0070     if options.dirin!='' and (not options.dirin.endswith('/')):    options.dirin+='/'
0071     if options.dirout!='' and (not options.dirout.endswith('/')):  options.dirout+='/'
0072
0073     # Build the IO files if necessary.
0074     # The default form of the files is:
0075     # <type>_<energy>_<step>.root
0076     prec_step = {"NONE":"",
0077                  "FILTER":"",
0078                  "ALL":"",
0079                  "LHE":"",
0080                  "GEN":"",
0081                  "reGEN":"",
0082                  "SIM":"GEN",
0083                  "reSIM":"SIM",
0084                  "DIGI":"SIM",
0085                  "reDIGI":"DIGI",
0086                  "L1REPACK":"RAW",
0087                  "L1P2GT":"RAW",
0088                  "HLT":"RAW",
0089                  "RECO":"DIGI",
0090                  "ALCA":"RECO",
0091                  "ANA":"RECO",
0092                  "SKIM":"RECO",
0093                  "DIGI2RAW":"DIGI",
0094                  "RAW2DIGI":"DIGI2RAW",
0095                  "RAW2RECO":"DIGI2RAW",
0096                  "DATAMIX":"DIGI",
0097                  "DIGI2RAW":"DATAMIX",
0098                  "HARVESTING":"RECO",
0099                  "ALCAHARVEST":"RECO",
0100                  "PAT":"RECO",
0101                  "NANO":"PAT",
0102                  "PATGEN":"GEN"}
0103
0104     trimmedEvtType=options.evt_type.split('/')[-1]
0105
0106     #get the list of steps, without their options
0107     options.trimmedStep=[]
0108     for s in options.step.split(','):
0109         step=s.split(':')[0]
0110         options.trimmedStep.append(step)
0111     first_step=options.trimmedStep[0]
0112
0113     #replace step aliases
0114     # this does not affect options.trimmedStep which still contains 'NONE'
0115     stepsAliases={
0116         'NONE':'',
0117         'ALL':'GEN,SIM,DIGI,L1,DIGI2RAW,HLT:GRun,RAW2DIGI,RECO,POSTRECO,VALIDATION,DQM',
0118         'DATA_CHAIN':'RAW2DIGI,RECO,POSTRECO,DQM'
0119         }
0120     if options.step in stepsAliases:
0121         options.step=stepsAliases[options.step]
0122
0123     options.step = options.step.replace("SIM_CHAIN","GEN,SIM,DIGI,L1,DIGI2RAW")
0124
0125     # add on the end of job sequence...
0126     addEndJob = True
0127     if ("FASTSIM" in options.step and not "VALIDATION" in options.step) or "HARVESTING" in options.step or "ALCAHARVEST" in options.step or "ALCAOUTPUT" in options.step or options.step == "":
0128         addEndJob = False
0129     if ("SKIM" in options.step and not "RECO" in options.step):
0130         addEndJob = False
0131     if ("ENDJOB" in options.step):
0132         addEndJob = False
0133     if ('DQMIO' in options.datatier):
0134         addEndJob = False
0135     if addEndJob:
0136         options.step=options.step+',ENDJOB'
0137
0138
0139     #determine the type of file on input
0140     if options.filetype==defaultOptions.filetype:
0141         if options.filein.lower().endswith(".lhe") or options.filein.lower().endswith(".lhef") or options.filein.startswith("lhe:"):
0142             options.filetype="LHE"
0143         elif options.filein.startswith("mcdb:"):
0144             print("This is a deprecated way of selecting lhe files from article number. Please use lhe:article argument to --filein")
0145             options.filein=options.filein.replace('mcdb:','lhe:')
0146             options.filetype="LHE"
0147         elif options.filein.lower().endswith(".rntpl"):
0148             options.filetype="EDM_RNTUPLE"
0149         else:
0150             options.filetype="EDM"
0151
0152     filesuffix = {"LHE": "lhe", "EDM": "root", "MCDB": "", "DQM":"root", "EDM_RNTUPLE":"rntpl"}[options.filetype]
0153
0154     if options.filein=="" and not (first_step in ("ALL","GEN","LHE","SIM_CHAIN")):
0155         options.dirin="file:"+options.dirin.replace('file:','')
0156         options.filein=trimmedEvtType+"_"+prec_step[first_step]+"."+filesuffix
0157
0158
0159     # Prepare the canonical file name for output / config file etc
0160     #   (EventType_STEP1_STEP2_..._PU)
0161     standardFileName = ""
0162     standardFileName = trimmedEvtType+"_"+"_".join(options.trimmedStep)
0163     standardFileName = standardFileName.replace(",","_").replace(".","_")
0164     if options.pileup != "NoPileUp":
0165         standardFileName += "_PU"
0166
0167
0168     # if no output file name given, set it to default
0169     if options.fileout=="" and not first_step in ("HARVESTING", "ALCAHARVEST"):
0170         options.fileout = standardFileName+".root"
0171
0172     # Prepare the name of the config file
0173     if not options.python_filename:
0174         options.python_filename = standardFileName+'.py'
0175
0176     print(options.step)
0177
0178
0179     # Setting name of process
0180     # if not set explicitly it needs some thinking
0181     if not options.name:
0182         if 'reSIM' in options.trimmedStep:
0183             options.name = 'RESIM'
0184         elif 'reDIGI' in options.trimmedStep:
0185             options.name = 'REDIGI'
0186         elif 'HLT' in options.trimmedStep:
0187             options.name = 'HLT'
0188         elif 'RECO' in options.trimmedStep:
0189             options.name = 'RECO'
0190         elif options.trimmedStep == ['NONE'] and options.filetype in ('LHE', 'MCDB'):
0191             options.name = 'LHE'
0192         elif len(options.trimmedStep)==0:
0193             options.name = 'PROCESS'
0194         else:
0195             options.name = options.trimmedStep[-1]
0196
0197     # check to be sure that people run the harvesting as a separate step
0198     isHarvesting = False
0199     isOther = False
0200
0201     if "HARVESTING" in options.trimmedStep and len(options.trimmedStep) > 1:
0202         raise Exception("The Harvesting step must be run alone")
0203
0204     # if not specified by user try to guess whether MC or DATA
0205     if not options.isData and not options.isMC:
0206         if 'LHE' in options.trimmedStep or 'LHE' in options.datatier:
0207             options.isMC=True
0208         if 'GEN' in options.trimmedStep or 'GEN' in options.datatier:
0209             options.isMC=True
0210         if 'SIM' in options.trimmedStep:
0211             options.isMC=True
0212         if 'CFWRITER' in options.trimmedStep:
0213             options.isMC=True
0214         if 'DIGI' in options.trimmedStep:
0215             options.isMC=True
0216         if 'DIGI2RAW' in options.trimmedStep:
0217             options.isMC=True
0218         if (not (options.eventcontent == None)) and 'SIM' in options.eventcontent:
0219             options.isMC=True
0220         if 'SIM' in options.datatier:
0221             options.isMC=True
0222         if 'VALIDATION' in options.trimmedStep:
0223             options.isMC=True
0224         if options.era and 'Phase2' in options.era:
0225             options.isMC=True
0226         if options.isMC:
0227             print('We have determined that this is simulation (if not, rerun cmsDriver.py with --data)')
0228         else:
0229             print('We have determined that this is real data (if not, rerun cmsDriver.py with --mc)')
0230             options.isData=True
0231
0232     if options.profile:
0233         if options.profile and options.prefix:
0234             raise Exception("--profile and --prefix are incompatible")
0235         profilerType = 'pp'
0236         profileOpts = options.profile.split(':')
0237         if len(profileOpts):
0238             profilerType = profileOpts[0].replace("=", " ")
0239
0240         if profilerType == "pp":
0241             options.profileTypeLabel = "performance"
0242         elif profilerType == "mp":
0243             options.profileTypeLabel = "memory"
0244         elif profilerType.startswith("fp "):
0245             options.profileTypeLabel = profilerType.replace("fp ", "")
0246         else:
0247             raise Exception("Not a valid profiler type %s. Alternatives are pp, mp, fp=<function>."%(profilerType))
0248
0249         options.prefix = "igprof -t cmsRun -%s" % profilerType
0250
0251     if options.heap_profile:
0252         if options.prefix:
0253             raise Exception("--heap_profile and --prefix are incompatible")
0254         options.prefix = "env MALLOC_CONF=prof:true,prof_accum:true,prof_prefix:jeprof.out LD_PRELOAD=libjemalloc-prof.so "
0255
0256     if options.maxmem_profile:
0257         if options.prefix:
0258             raise Exception("--maxmem_profile and --prefix are incompatible")
0259         options.prefix = "env LD_PRELOAD=libPerfToolsAllocMonitorPreload.so:libPerfToolsMaxMemoryPreload.so "
0260
0261     # If an "era" argument was supplied make sure it is one of the valid possibilities
0262     if options.era :
0263         from Configuration.StandardSequences.Eras import eras
0264         # Split the string by commas to check individual eras
0265         requestedEras = options.era.split(",")
0266         # Check that the entry is a valid era
0267         for eraName in requestedEras :
0268             if not hasattr( eras, eraName ) or not checkModifier(getattr(eras,eraName)): # Not valid, so print a helpful message
0269                 validOptions="" # Create a stringified list of valid options to print to the user
0270                 for key in eras.__dict__ :
0271                     if checkModifier(eras.__dict__[key]):
0272                         if validOptions!="" : validOptions+=", "
0273                         validOptions+="'"+key+"'"
0274                 raise Exception( "'%s' is not a valid option for '--era'. Valid options are %s." % (eraName, validOptions) )
0275     # If the "--fast" option was supplied automatically enable the fastSim era
0276     if options.fast :
0277         if options.era:
0278             options.era+=",fastSim"
0279         else :
0280             options.era="fastSim"
0281
0282     # options incompatible with fastsim
0283     if options.fast and not options.scenario == "pp":
0284         raise Exception("ERROR: the --option fast is only compatible with the default scenario (--scenario=pp)")
0285     if options.fast and 'HLT' in options.trimmedStep:
0286         raise Exception("ERROR: the --option fast is incompatible with HLT (HLT is no longer available in FastSim)")
0287
0288     return options
0289