DataProcessing/python/Merge.py

0001 #!/usr/bin/env python3
0002 """
0003 _Merge_
0004
0005 Module that generates standard merge job configurations for use in any
0006 standard processing
0007
0008 """
0009
0010
0011 from FWCore.ParameterSet.Config import Process, EndPath
0012 from FWCore.ParameterSet.Modules import OutputModule, Source, Service
0013 import FWCore.ParameterSet.Types as CfgTypes
0014
0015
0016 def mergeProcess(*inputFiles, **options):
0017     """
0018     _mergeProcess_
0019
0020     Creates and returns a merge process that will merge the provided
0021     filenames
0022
0023     supported options:
0024
0025     - process_name : name of the process, defaults to Merge
0026     - outputmod_label : label of the output module, defaults to Merged
0027     - newDQMIO : specifies if the new DQM format should be used to merge the files
0028     - output_file : sets the output file name
0029     - output_lfn : sets the output LFN
0030     - mergeNANO : to merge NanoAOD
0031     - bypassVersionCheck : to bypass version check in case merging happened in lower version of CMSSW (i.e. UL HLT case). This will be FALSE by default.
0032
0033     """
0034     #  //
0035     # // process supported options
0036     #//
0037     processName = options.get("process_name", "Merge")
0038     outputModLabel = options.get("outputmod_label", "Merged")
0039     outputFilename = options.get("output_file", "Merged.root")
0040     outputLFN = options.get("output_lfn", None)
0041     dropDQM = options.get("drop_dqm", False)
0042     newDQMIO = options.get("newDQMIO", False)
0043     mergeNANO = options.get("mergeNANO", False)
0044     bypassVersionCheck = options.get("bypassVersionCheck", False)
0045     #  //
0046     # // build process
0047     #//
0048     process = Process(processName)
0049
0050     #  //
0051     # // input source
0052     #//
0053     if newDQMIO:
0054         process.source = Source("DQMRootSource", reScope = CfgTypes.untracked.string(""))
0055         process.add_(Service("DQMStore"))
0056     else:
0057         process.source = Source("PoolSource")
0058         if bypassVersionCheck:
0059             process.source.bypassVersionCheck = CfgTypes.untracked.bool(True)
0060         if dropDQM:
0061             process.source.inputCommands = CfgTypes.untracked.vstring('keep *','drop *_EDMtoMEConverter_*_*')
0062         if not mergeNANO:
0063             process.source.noRunLumiSort = CfgTypes.untracked.bool(True)
0064     process.source.fileNames = CfgTypes.untracked(CfgTypes.vstring())
0065     for entry in inputFiles:
0066         process.source.fileNames.append(str(entry))
0067
0068     #  //
0069     # // output module
0070     #//
0071     if newDQMIO:
0072         outMod = OutputModule("DQMRootOutputModule")
0073     elif mergeNANO:
0074         import Configuration.EventContent.EventContent_cff
0075         outMod = OutputModule("NanoAODOutputModule",Configuration.EventContent.EventContent_cff.NANOAODEventContent.clone())
0076     else:
0077         outMod = OutputModule("PoolOutputModule")
0078         outMod.mergeJob = CfgTypes.untracked.bool(True)
0079         outMod.eventAuxiliaryBasketSize = CfgTypes.untracked.int32(2*1024*1024)
0080
0081     outMod.fileName = CfgTypes.untracked.string(outputFilename)
0082     if outputLFN != None:
0083         outMod.logicalFileName = CfgTypes.untracked.string(outputLFN)
0084     setattr(process, outputModLabel, outMod)
0085
0086     process.outputPath = EndPath(outMod)
0087
0088     return process