python/Workflow/DTTtrigWorkflow.py

0001 from __future__ import absolute_import
0002 import os
0003 import logging
0004 import glob
0005
0006 from . import tools
0007 import FWCore.ParameterSet.Config as cms
0008 from .DTWorkflow import DTWorkflow
0009
0010 log = logging.getLogger(__name__)
0011
0012 class DTttrigWorkflow( DTWorkflow ):
0013     """ This class creates and performce / submits ttrig workflow jobs"""
0014     def __init__(self, options):
0015         # call parent constructor
0016         super( DTttrigWorkflow, self ).__init__( options )
0017
0018         self.outpath_command_tag = "TTrigCalibration"
0019         # Dict to map workflow modes to output file name
0020         output_file_dict ={ "timeboxes" : "DTTimeBoxes.root",
0021                             "residuals" : 'residuals.root',
0022                             "validation" : "DQM.root"
0023                             }
0024         # Dict to map workflow modes to output folders in main path
0025         self.outpath_workflow_mode_dict = {
0026                                             "timeboxes" : "TimeBoxes",
0027                                             "residuals" : "Residuals",
0028                                             "validation" : "TtrigValidation"
0029                                            }
0030        # Dict to map workflow modes to database file name
0031         self.output_db_dict = { "timeboxes" : {
0032                                           "write": "ttrig_timeboxes_uncorrected_"
0033                                                     + self.tag
0034                                                     + ".db",
0035                                           "correction": "ttrig_timeboxes_"
0036                                                         + self.tag
0037                                                         + ".db"
0038                                          },
0039                            "residuals" : "ttrig_residuals_" + self.tag + ".db"}
0040         self.output_file = output_file_dict[self.options.workflow_mode]
0041         self.output_files = [self.output_file]
0042
0043     def prepare_workflow(self):
0044         """ Generalized function to prepare workflow dependent on workflow mode"""
0045         function_name = "prepare_" + self.options.workflow_mode + "_" + self.options.command
0046         try:
0047             fill_function = getattr(self, function_name)
0048         except AttributeError:
0049             errmsg = "Class `{}` does not implement `{}`"
0050             raise NotImplementedError( errmsg.format(self.__class__.__name__,
0051                                                      function_name))
0052         log.debug("Preparing workflow with function %s" % function_name)
0053         # call chosen function
0054         fill_function()
0055
0056     def get_output_db(self, workflow_mode, command):
0057         output_db_file = self.output_db_dict[workflow_mode]
0058         if isinstance(output_db_file, dict):
0059             return output_db_file[command]
0060         return output_db_file
0061     ####################################################################
0062     #                     Prepare functions for timeboxes              #
0063     ####################################################################
0064     def prepare_timeboxes_submit(self):
0065         self.pset_name = 'dtTTrigCalibration_cfg.py'
0066         self.pset_template = 'CalibMuon.DTCalibration.dtTTrigCalibration_cfg'
0067         if self.options.datasettype == "Cosmics":
0068             self.pset_template = 'CalibMuon.DTCalibration.dtTTrigCalibration_cosmics_cfg'
0069         log.debug('Using pset template ' + self.pset_template)
0070         self.process = tools.loadCmsProcess(self.pset_template)
0071         self.process.GlobalTag.globaltag = self.options.globaltag
0072         self.process.dtTTrigCalibration.rootFileName = self.output_file
0073         self.process.dtTTrigCalibration.digiLabel = self.digilabel
0074
0075         if self.options.inputDBTag:
0076             self.add_local_custom_db()
0077         self.prepare_common_submit()
0078         self.write_pset_file()
0079
0080     def prepare_timeboxes_check(self):
0081         self.load_options_command("submit")
0082
0083     def prepare_timeboxes_write(self):
0084         self.output_db_file = self.output_db_dict[self.options.workflow_mode]
0085         if isinstance(self.output_db_dict[self.options.workflow_mode], dict):
0086             self.output_db_file = self.output_db_file[self.options.command]
0087         self.prepare_common_write()
0088         merged_file = os.path.join(self.result_path, self.output_file)
0089         ttrig_uncorrected_db = os.path.join(self.result_path,
0090                                             self.get_output_db("timeboxes", "write"))
0091         self.pset_name = 'dtTTrigWriter_cfg.py'
0092         self.pset_template = "CalibMuon.DTCalibration.dtTTrigWriter_cfg"
0093         self.process = tools.loadCmsProcess(self.pset_template)
0094         self.process.dtTTrigWriter.rootFileName = "file:///" + merged_file
0095         self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_uncorrected_db
0096         self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
0097         self.write_pset_file()
0098
0099     def prepare_timeboxes_correction(self):
0100         self.load_options_command("submit")
0101         self.pset_name = 'dtTTrigCorrection_cfg.py'
0102         self.pset_template = "CalibMuon.DTCalibration.dtTTrigCorrection_cfg"
0103         ttrig_timeboxes_db = "ttrig_timeboxes_"+ self.tag + ".db"
0104         ttrig_timeboxes_db = os.path.join(self.result_path,
0105                                           self.get_output_db("timeboxes", "correction"))
0106         self.process = tools.loadCmsProcess(self.pset_template)
0107         self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
0108         self.process.source.firstRun = self.options.run
0109         self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_timeboxes_db
0110
0111         if not self.options.inputCalibDB:
0112             self.options.inputCalibDB = os.path.join(self.result_path,
0113                                             self.get_output_db( "timeboxes", "write" ) )
0114         if self.options.inputVDriftDB:
0115             log.warning("Options inputVDriftDB has no effect for timeboxes correction")
0116         if self.options.inputT0DB:
0117             log.warning("Options inputT0DB has no effect for timeboxes correction")
0118         self.add_local_calib_db(local=True)
0119         self.write_pset_file()
0120
0121     def prepare_timeboxes_dump(self):
0122         self.pset_name = 'dumpDBToFile_ResidCorr_cfg.py'
0123         self.pset_template = 'CalibMuon.DTCalibration.dumpDBToFile_ttrig_cfg'
0124         if self.options.input_dumpDB:
0125             try:
0126                 test = self.result_path
0127                 self.load_options_command("correction")
0128             except:
0129                 pass
0130             dbpath = os.path.abspath(self.options.input_dumpDB)
0131         else:
0132             dbpath = os.path.abspath( os.path.join(self.result_path,
0133                                                    self.get_output_db("timeboxes",
0134                                                                       "correction")))
0135         self.prepare_common_dump(dbpath)
0136         self.write_pset_file()
0137
0138     def prepare_timeboxes_all(self):
0139         # individual prepare functions for all tasks will be called in
0140         # main implementation of all
0141         self.all_commands=["submit", "check", "write", "correction", "dump"]
0142
0143     ####################################################################
0144     #                      prepare functions for residuals             #
0145     ####################################################################
0146     def prepare_residuals_submit(self):
0147         self.pset_name = 'dtResidualCalibration_cfg.py'
0148         self.pset_template = 'CalibMuon.DTCalibration.dtResidualCalibration_cfg'
0149         if self.options.datasettype == "Cosmics":
0150             self.pset_template = 'CalibMuon.DTCalibration.dtResidualCalibration_cosmics_cfg'
0151         self.process = tools.loadCmsProcess(self.pset_template)
0152         #~ self.process.GlobalTag.globaltag = cms.string(self.options.globaltag)
0153         self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
0154         self.process.dtResidualCalibration.rootFileName = self.output_file
0155         self.prepare_common_submit()
0156         if self.options.fromMuons:
0157             self.process.dtResidualCalibration.Muons = cms.InputTag("muons")
0158         if self.options.histoRange:
0159             self.process.dtResidualCalibration.histogramRange = cms.double(float(self.options.histoRange))
0160         if self.options.inputCalibDB:
0161             self.add_local_calib_db()
0162         self.write_pset_file()
0163
0164     def prepare_residuals_check(self):
0165         self.load_options_command("submit")
0166
0167     def prepare_residuals_correction(self):
0168         self.pset_name = "dtTTrigResidualCorrection_cfg.py"
0169         self.pset_template = 'CalibMuon.DTCalibration.dtTTrigResidualCorrection_cfg'
0170         self.process = tools.loadCmsProcess(self.pset_template)
0171         self.load_options_command("submit")
0172         self.process.source.firstRun = cms.untracked.uint32(self.options.run)
0173         self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
0174
0175         tag = self.prepare_common_write()
0176         if self.options.inputT0DB:
0177             log.warning("Option inputT0DB not supported for residual corrections")
0178
0179         if self.options.inputDBTag:
0180             self.add_local_custom_db(local=True)
0181         if self.options.inputVDriftDB:
0182             self.add_local_vdrift_db(local=True)
0183         if self.options.inputCalibDB:
0184             self.add_local_calib_db(local=True)
0185         # Change DB label if running on Cosmics
0186         if self.options.datasettype == "Cosmics":
0187             self.process.dtTTrigResidualCorrection.dbLabel = 'cosmics'
0188             self.process.dtTTrigResidualCorrection.correctionAlgoConfig.dbLabel = 'cosmics'
0189         ttrig_ResidCorr_db = os.path.abspath( os.path.join(self.result_path,
0190                                               self.get_output_db("residuals", "write")))
0191         self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_ResidCorr_db
0192         rootfile_path = os.path.abspath( os.path.join(self.result_path, self.output_file))
0193         merged_file = os.path.join(self.result_path, self.output_file)
0194         self.process.dtTTrigResidualCorrection.correctionAlgoConfig.residualsRootFile = merged_file
0195         self.write_pset_file()
0196
0197     def prepare_residuals_dump(self):
0198         self.pset_name = 'dumpDBToFile_ResidCorr_cfg.py'
0199         self.pset_template = 'CalibMuon.DTCalibration.dumpDBToFile_ttrig_cfg'
0200         if self.options.input_dumpDB:
0201             dbpath = os.path.abspath(self.options.input_dumpDB)
0202         else:
0203             try:
0204                 test = self.result_path
0205                 self.load_options_command("write")
0206             except:
0207                 pass
0208             dbpath = os.path.abspath( os.path.join(self.result_path,
0209                                                    self.get_output_db("residuals",
0210                                                                       "write")))
0211         self.prepare_common_dump(dbpath)
0212         self.write_pset_file()
0213
0214     def prepare_residuals_all(self):
0215         # individual prepare functions for all tasks will be called in
0216         # main implementation of all
0217         self.all_commands=["submit", "check", "correction", "dump"]
0218
0219     ####################################################################
0220     #                   prepare functions for validation               #
0221     ####################################################################
0222
0223     def prepare_validation_submit(self):
0224         self.pset_name = 'dtCalibValidation_cfg.py'
0225         self.pset_template = 'CalibMuon.DTCalibration.dtCalibValidation_cfg'
0226         if self.options.datasettype == "Cosmics":
0227             self.pset_template = 'CalibMuon.DTCalibration.dtCalibValidation_cosmics_cfg'
0228         if self.options.fromMuons:
0229             self.pset_template = 'CalibMuon.DTCalibration.dtCalibValidationFromMuons_cfg'
0230         self.process = tools.loadCmsProcess(self.pset_template)
0231         self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
0232         self.prepare_common_submit()
0233         if self.options.inputCalibDB:
0234             self.add_local_calib_db()
0235         self.write_pset_file()
0236
0237     def prepare_validation_check(self):
0238         self.load_options_command("submit")
0239
0240     def prepare_validation_write(self):
0241         self.pset_name = 'dtDQMClient_cfg.py'
0242         self.pset_template = 'CalibMuon.DTCalibration.dtDQMClient_cfg'
0243         self.process = tools.loadCmsProcess(self.pset_template)
0244         self.prepare_common_write(do_hadd = False)
0245         dqm_files = glob.glob(os.path.join( self.local_path,
0246                                             "unmerged_results",
0247                                             "DQM_*.root"))
0248         dqm_files[:] = ["file://"+txt for txt in dqm_files]
0249         self.process.source.fileNames =  dqm_files
0250         self.process.dqmSaver.dirName = os.path.abspath(self.result_path)
0251         self.process.dqmSaver.workflow = str(self.options.datasetpath)
0252         if self.process.DQMStore.collateHistograms == True:
0253             self.process.dqmSaver.forceRunNumber = self.options.run
0254         self.write_pset_file()
0255
0256     def summary(self):
0257         pass
0258
0259     def prepare_validation_all(self):
0260         # individual prepare functions for all tasks will be called in
0261         # main implementation of all
0262         self.all_commands=["submit", "check","write"]
0263
0264     ####################################################################
0265     #                           CLI creation                           #
0266     ####################################################################
0267     @classmethod
0268     def add_parser_options(cls, subparser_container):
0269         ttrig_parser = subparser_container.add_parser( "ttrig",
0270         #parents=[mutual_parent_parser, common_parent_parser],
0271         help = "" ) # What does ttrig
0272
0273
0274         ################################################################
0275         #                Sub parser options for workflow modes         #
0276         ################################################################
0277         ttrig_subparsers = ttrig_parser.add_subparsers( dest="workflow_mode",
0278             help="Possible workflow modes",)
0279         ## Add all workflow modes for ttrig
0280         ttrig_timeboxes_subparser = ttrig_subparsers.add_parser( "timeboxes",
0281             help = "" )
0282         ttrig_residuals_subparser = ttrig_subparsers.add_parser( "residuals",
0283             help = "" )
0284         ttrig_validation_subparser = ttrig_subparsers.add_parser( "validation",
0285             help = "" )
0286         ################################################################
0287         #        Sub parser options for workflow mode timeboxes        #
0288         ################################################################
0289         ttrig_timeboxes_subparsers = ttrig_timeboxes_subparser.add_subparsers( dest="command",
0290             help="Possible commands for timeboxes")
0291         ttrig_timeboxes_submit_parser = ttrig_timeboxes_subparsers.add_parser(
0292             "submit",
0293             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0294                     super(DTttrigWorkflow,cls).get_submission_options_parser(),
0295                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0296             help = "Submit job to the GRID via crab3")
0297
0298         ttrig_timeboxes_check_parser = ttrig_timeboxes_subparsers.add_parser(
0299             "check",
0300             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0301                     super(DTttrigWorkflow,cls).get_check_options_parser(),],
0302             help = "Check status of submitted jobs")
0303
0304         ttrig_timeboxes_write_parser = ttrig_timeboxes_subparsers.add_parser(
0305             "write",
0306             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0307                      super(DTttrigWorkflow,cls).get_write_options_parser()
0308                     ],
0309             help = "Write result from root output to text file")
0310
0311         ttrig_timeboxes_correction_parser = ttrig_timeboxes_subparsers.add_parser(
0312             "correction",
0313             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0314                      super(DTttrigWorkflow,cls).get_local_input_db_options_parser()
0315                     ],
0316             help = "Perform correction on uncorrected ttrig database")
0317
0318         ttrig_timeboxes_dump_parser = ttrig_timeboxes_subparsers.add_parser(
0319             "dump",
0320             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0321                      super(DTttrigWorkflow,cls).get_dump_options_parser()],
0322             help = "Dump database to text file")
0323
0324         ttrig_timeboxes_all_parser = ttrig_timeboxes_subparsers.add_parser(
0325             "all",
0326             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0327                      super(DTttrigWorkflow,cls).get_submission_options_parser(),
0328                      super(DTttrigWorkflow,cls).get_check_options_parser(),
0329                      super(DTttrigWorkflow,cls).get_input_db_options_parser(),
0330                      super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0331                      super(DTttrigWorkflow,cls).get_write_options_parser(),
0332                      super(DTttrigWorkflow,cls).get_dump_options_parser()
0333                     ],
0334             help = "Perform all steps: submit, check, write, correction,"\
0335                    "dump in this order")
0336
0337         ################################################################
0338         #        Sub parser options for workflow mode residuals        #
0339         ################################################################
0340         ttrig_residuals_subparsers = ttrig_residuals_subparser.add_subparsers( dest="command",
0341             help="Possible commands for residuals")
0342         ttrig_residuals_submit_parser = ttrig_residuals_subparsers.add_parser(
0343             "submit",
0344             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0345                     super(DTttrigWorkflow,cls).get_submission_options_parser(),
0346                     super(DTttrigWorkflow,cls).get_check_options_parser(),
0347                     super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0348                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0349             help = "Submit job to the GRID via crab3")
0350
0351         ttrig_residuals_check_parser = ttrig_residuals_subparsers.add_parser(
0352             "check",
0353             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0354                     super(DTttrigWorkflow,cls).get_check_options_parser(),],
0355             help = "Check status of submitted jobs")
0356
0357         ttrig_residuals_correct_parser = ttrig_residuals_subparsers.add_parser(
0358             "correction",
0359             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0360                     super(DTttrigWorkflow,cls).get_write_options_parser(),
0361                     super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0362                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0363             help = "Perform residual corrections")
0364         ttrig_residuals_correct_parser.add_argument("--globaltag",
0365             help="Alternative globalTag. Otherwise the gt for sunmission is used")
0366
0367         ttrig_residuals_dump_parser = ttrig_residuals_subparsers.add_parser(
0368             "dump",
0369             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0370                      super(DTttrigWorkflow,cls).get_dump_options_parser()],
0371             help = "Dump database to text file")
0372
0373         ttrig_residuals_all_parser = ttrig_residuals_subparsers.add_parser(
0374             "all",
0375             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0376                     super(DTttrigWorkflow,cls).get_submission_options_parser(),
0377                     super(DTttrigWorkflow,cls).get_check_options_parser(),
0378                     super(DTttrigWorkflow,cls).get_write_options_parser(),
0379                     super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0380                     super(DTttrigWorkflow,cls).get_dump_options_parser(),
0381                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0382             help = "Perform all steps: submit, check, correct, dump")
0383
0384         ################################################################
0385         #        Sub parser options for workflow mode validation       #
0386         ################################################################
0387         ttrig_validation_subparsers = ttrig_validation_subparser.add_subparsers( dest="command",
0388             help="Possible commands for residuals")
0389
0390         ttrig_validation_submit_parser = ttrig_validation_subparsers.add_parser(
0391             "submit",
0392             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0393                     super(DTttrigWorkflow,cls).get_submission_options_parser(),
0394                     super(DTttrigWorkflow,cls).get_check_options_parser(),
0395                     super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0396                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0397             help = "Submit job to the GRID via crab3")
0398
0399
0400         ttrig_validation_check_parser = ttrig_validation_subparsers.add_parser(
0401             "check",
0402             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0403                     super(DTttrigWorkflow,cls).get_check_options_parser(),],
0404             help = "Check status of submitted jobs")
0405
0406         ttrig_validation_write_parser = ttrig_validation_subparsers.add_parser(
0407             "write",
0408             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0409                     super(DTttrigWorkflow,cls).get_write_options_parser()],
0410             help = "Create summary for validation")
0411
0412         ttrig_validation_all_parser = ttrig_validation_subparsers.add_parser(
0413             "all",
0414             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0415                     super(DTttrigWorkflow,cls).get_submission_options_parser(),
0416                     super(DTttrigWorkflow,cls).get_check_options_parser(),
0417                     super(DTttrigWorkflow,cls).get_write_options_parser(),
0418                     super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0419                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0420             help = "Perform all steps: submit, check, summary")