Applications/python/cmsDriverOptions.py

0001 #! /usr/bin/env python3
0002
0003 # A Pyrelval Wrapper
0004
0005 import optparse
0006 import sys
0007 import os
0008 import re
0009 import Configuration.Applications
0010 from Configuration.Applications.ConfigBuilder import ConfigBuilder, defaultOptions
0011 import traceback
0012 from functools import reduce
0013
0014 def checkModifier(era):
0015     from FWCore.ParameterSet.Config import Modifier, ModifierChain
0016     return isinstance( era, Modifier ) or isinstance( era, ModifierChain )
0017
0018 def checkOptions():
0019     return
0020
0021 def adaptOptions():
0022     return
0023
0024 def OptionsFromCommand(command):
0025     items=command.split()
0026     if items[0] != 'cmsDriver.py':
0027         return None
0028     items.append('--evt_type')
0029     items.append(items[1])
0030     options=OptionsFromItems(items[2:])
0031     options.arguments = command
0032     return options
0033
0034 def OptionsFromCommandLine():
0035     import sys
0036     options=OptionsFromItems(sys.argv[1:])
0037     # memorize the command line arguments
0038     options.arguments = reduce(lambda x, y: x+' '+y, sys.argv[1:])
0039     return options
0040
0041 def OptionsFromItems(items):
0042     import sys
0043     from Configuration.Applications.Options import parser
0044
0045     options = parser.parse_args(items)
0046
0047     if options.conditions=="help":
0048         from Configuration.AlCa import autoCond
0049         possible=""
0050         for k in autoCond.autoCond:
0051             possible+="\nauto:"+k+" -> "+str(autoCond.autoCond[k])
0052         parser.error("Possibilities for the --conditions option: "+possible)
0053
0054
0055     #################################
0056     # Check parameters for validity #
0057     #################################
0058
0059     # check in case of ALCAOUTPUT case for alca splitting
0060     if options.triggerResultsProcess == None and "ALCAOUTPUT" in options.step:
0061         print("ERROR: If ALCA splitting is requested, the name of the process in which the alca producers ran needs to be specified. E.g. via --triggerResultsProcess RECO")
0062         sys.exit(1)
0063
0064     if not options.evt_type:
0065         options.evt_type=sys.argv[1]
0066
0067     #now adjust the given parameters before passing it to the ConfigBuilder
0068
0069     #trail a "/" to dirin and dirout
0070     if options.dirin!='' and (not options.dirin.endswith('/')):    options.dirin+='/'
0071     if options.dirout!='' and (not options.dirout.endswith('/')):  options.dirout+='/'
0072
0073     # Build the IO files if necessary.
0074     # The default form of the files is:
0075     # <type>_<energy>_<step>.root
0076     prec_step = {"NONE":"",
0077                  "FILTER":"",
0078                  "ALL":"",
0079                  "LHE":"",
0080                  "GEN":"",
0081                  "reGEN":"",
0082                  "SIM":"GEN",
0083                  "reSIM":"SIM",
0084                  "DIGI":"SIM",
0085                  "reDIGI":"DIGI",
0086                  "L1REPACK":"RAW",
0087                  "HLT":"RAW",
0088                  "RECO":"DIGI",
0089                  "ALCA":"RECO",
0090                  "ANA":"RECO",
0091                  "SKIM":"RECO",
0092                  "DIGI2RAW":"DIGI",
0093                  "RAW2DIGI":"DIGI2RAW",
0094                  "RAW2RECO":"DIGI2RAW",
0095                  "DATAMIX":"DIGI",
0096                  "DIGI2RAW":"DATAMIX",
0097                  "HARVESTING":"RECO",
0098                  "ALCAHARVEST":"RECO",
0099                  "PAT":"RECO",
0100                  "NANO":"PAT",
0101                  "PATGEN":"GEN"}
0102
0103     trimmedEvtType=options.evt_type.split('/')[-1]
0104
0105     #get the list of steps, without their options
0106     options.trimmedStep=[]
0107     for s in options.step.split(','):
0108         step=s.split(':')[0]
0109         options.trimmedStep.append(step)
0110     first_step=options.trimmedStep[0]
0111
0112     #replace step aliases
0113     # this does not affect options.trimmedStep which still contains 'NONE'
0114     stepsAliases={
0115         'NONE':'',
0116         'ALL':'GEN,SIM,DIGI,L1,DIGI2RAW,HLT:GRun,RAW2DIGI,RECO,POSTRECO,VALIDATION,DQM',
0117         'DATA_CHAIN':'RAW2DIGI,RECO,POSTRECO,DQM'
0118         }
0119     if options.step in stepsAliases:
0120         options.step=stepsAliases[options.step]
0121
0122     options.step = options.step.replace("SIM_CHAIN","GEN,SIM,DIGI,L1,DIGI2RAW")
0123
0124     # add on the end of job sequence...
0125     addEndJob = True
0126     if ("FASTSIM" in options.step and not "VALIDATION" in options.step) or "HARVESTING" in options.step or "ALCAHARVEST" in options.step or "ALCAOUTPUT" in options.step or options.step == "":
0127         addEndJob = False
0128     if ("SKIM" in options.step and not "RECO" in options.step):
0129         addEndJob = False
0130     if ("ENDJOB" in options.step):
0131         addEndJob = False
0132     if ('DQMIO' in options.datatier):
0133         addEndJob = False
0134     if addEndJob:
0135         options.step=options.step+',ENDJOB'
0136
0137
0138     #determine the type of file on input
0139     if options.filetype==defaultOptions.filetype:
0140         if options.filein.lower().endswith(".lhe") or options.filein.lower().endswith(".lhef") or options.filein.startswith("lhe:"):
0141             options.filetype="LHE"
0142         elif options.filein.startswith("mcdb:"):
0143             print("This is a deprecated way of selecting lhe files from article number. Please use lhe:article argument to --filein")
0144             options.filein=options.filein.replace('mcdb:','lhe:')
0145             options.filetype="LHE"
0146         else:
0147             options.filetype="EDM"
0148
0149     filesuffix = {"LHE": "lhe", "EDM": "root", "MCDB": "", "DQM":"root"}[options.filetype]
0150
0151     if options.filein=="" and not (first_step in ("ALL","GEN","LHE","SIM_CHAIN")):
0152         options.dirin="file:"+options.dirin.replace('file:','')
0153         options.filein=trimmedEvtType+"_"+prec_step[first_step]+"."+filesuffix
0154
0155
0156     # Prepare the canonical file name for output / config file etc
0157     #   (EventType_STEP1_STEP2_..._PU)
0158     standardFileName = ""
0159     standardFileName = trimmedEvtType+"_"+"_".join(options.trimmedStep)
0160     standardFileName = standardFileName.replace(",","_").replace(".","_")
0161     if options.pileup != "NoPileUp":
0162         standardFileName += "_PU"
0163
0164
0165     # if no output file name given, set it to default
0166     if options.fileout=="" and not first_step in ("HARVESTING", "ALCAHARVEST"):
0167         options.fileout = standardFileName+".root"
0168
0169     # Prepare the name of the config file
0170     if not options.python_filename:
0171         options.python_filename = standardFileName+'.py'
0172
0173     print(options.step)
0174
0175
0176     # Setting name of process
0177     # if not set explicitly it needs some thinking
0178     if not options.name:
0179         if 'reSIM' in options.trimmedStep:
0180             options.name = 'RESIM'
0181         elif 'reDIGI' in options.trimmedStep:
0182             options.name = 'REDIGI'
0183         elif 'HLT' in options.trimmedStep:
0184             options.name = 'HLT'
0185         elif 'RECO' in options.trimmedStep:
0186             options.name = 'RECO'
0187         elif options.trimmedStep == ['NONE'] and options.filetype in ('LHE', 'MCDB'):
0188             options.name = 'LHE'
0189         elif len(options.trimmedStep)==0:
0190             options.name = 'PROCESS'
0191         else:
0192             options.name = options.trimmedStep[-1]
0193
0194     # check to be sure that people run the harvesting as a separate step
0195     isHarvesting = False
0196     isOther = False
0197
0198     if "HARVESTING" in options.trimmedStep and len(options.trimmedStep) > 1:
0199         raise Exception("The Harvesting step must be run alone")
0200
0201     # if not specified by user try to guess whether MC or DATA
0202     if not options.isData and not options.isMC:
0203         if 'LHE' in options.trimmedStep or 'LHE' in options.datatier:
0204             options.isMC=True
0205         if 'GEN' in options.trimmedStep or 'GEN' in options.datatier:
0206             options.isMC=True
0207         if 'SIM' in options.trimmedStep:
0208             options.isMC=True
0209         if 'CFWRITER' in options.trimmedStep:
0210             options.isMC=True
0211         if 'DIGI' in options.trimmedStep:
0212             options.isMC=True
0213         if 'DIGI2RAW' in options.trimmedStep:
0214             options.isMC=True
0215         if (not (options.eventcontent == None)) and 'SIM' in options.eventcontent:
0216             options.isMC=True
0217         if 'SIM' in options.datatier:
0218             options.isMC=True
0219         if 'VALIDATION' in options.trimmedStep:
0220             options.isMC=True
0221         if options.era and 'Phase2' in options.era:
0222             options.isMC=True
0223         if options.isMC:
0224             print('We have determined that this is simulation (if not, rerun cmsDriver.py with --data)')
0225         else:
0226             print('We have determined that this is real data (if not, rerun cmsDriver.py with --mc)')
0227             options.isData=True
0228
0229     if options.profile:
0230         if options.profile and options.prefix:
0231             raise Exception("--profile and --prefix are incompatible")
0232         profilerType = 'pp'
0233         profileOpts = options.profile.split(':')
0234         if len(profileOpts):
0235             profilerType = profileOpts[0].replace("=", " ")
0236
0237         if profilerType == "pp":
0238             options.profileTypeLabel = "performance"
0239         elif profilerType == "mp":
0240             options.profileTypeLabel = "memory"
0241         elif profilerType.startswith("fp "):
0242             options.profileTypeLabel = profilerType.replace("fp ", "")
0243         else:
0244             raise Exception("Not a valid profiler type %s. Alternatives are pp, mp, fp=<function>."%(profilerType))
0245
0246         options.prefix = "igprof -t cmsRun -%s" % profilerType
0247
0248     if options.heap_profile:
0249         if options.prefix:
0250             raise Exception("--heap_profile and --prefix are incompatible")
0251         options.prefix = "env MALLOC_CONF=prof:true,prof_accum:true,prof_prefix:jeprof.out LD_PRELOAD=libjemalloc-prof.so "
0252
0253     if options.maxmem_profile:
0254         if options.prefix:
0255             raise Exception("--maxmem_profile and --prefix are incompatible")
0256         options.prefix = "env LD_PRELOAD=libPerfToolsAllocMonitorPreload.so:libPerfToolsMaxMemoryPreload.so "
0257
0258     # If an "era" argument was supplied make sure it is one of the valid possibilities
0259     if options.era :
0260         from Configuration.StandardSequences.Eras import eras
0261         # Split the string by commas to check individual eras
0262         requestedEras = options.era.split(",")
0263         # Check that the entry is a valid era
0264         for eraName in requestedEras :
0265             if not hasattr( eras, eraName ) or not checkModifier(getattr(eras,eraName)): # Not valid, so print a helpful message
0266                 validOptions="" # Create a stringified list of valid options to print to the user
0267                 for key in eras.__dict__ :
0268                     if checkModifier(eras.__dict__[key]):
0269                         if validOptions!="" : validOptions+=", "
0270                         validOptions+="'"+key+"'"
0271                 raise Exception( "'%s' is not a valid option for '--era'. Valid options are %s." % (eraName, validOptions) )
0272     # If the "--fast" option was supplied automatically enable the fastSim era
0273     if options.fast :
0274         if options.era:
0275             options.era+=",fastSim"
0276         else :
0277             options.era="fastSim"
0278
0279     # options incompatible with fastsim
0280     if options.fast and not options.scenario == "pp":
0281         raise Exception("ERROR: the --option fast is only compatible with the default scenario (--scenario=pp)")
0282     if options.fast and 'HLT' in options.trimmedStep:
0283         raise Exception("ERROR: the --option fast is incompatible with HLT (HLT is no longer available in FastSim)")
0284
0285     return options
0286