python/Workflow/DTTtrigWorkflow.py

0001 import os
0002 import logging
0003 import glob
0004
0005 from . import tools
0006 import FWCore.ParameterSet.Config as cms
0007 from .DTWorkflow import DTWorkflow
0008
0009 log = logging.getLogger(__name__)
0010
0011 class DTttrigWorkflow( DTWorkflow ):
0012     """ This class creates and performce / submits ttrig workflow jobs"""
0013     def __init__(self, options):
0014         # call parent constructor
0015         super( DTttrigWorkflow, self ).__init__( options )
0016
0017         self.outpath_command_tag = "TTrigCalibration"
0018         # Dict to map workflow modes to output file name
0019         output_file_dict ={ "timeboxes" : "DTTimeBoxes.root",
0020                             "residuals" : 'residuals.root',
0021                             "validation" : "DQM.root"
0022                             }
0023         # Dict to map workflow modes to output folders in main path
0024         self.outpath_workflow_mode_dict = {
0025                                             "timeboxes" : "TimeBoxes",
0026                                             "residuals" : "Residuals",
0027                                             "validation" : "TtrigValidation"
0028                                            }
0029        # Dict to map workflow modes to database file name
0030         self.output_db_dict = { "timeboxes" : {
0031                                           "write": "ttrig_timeboxes_uncorrected_"
0032                                                     + self.tag
0033                                                     + ".db",
0034                                           "correction": "ttrig_timeboxes_"
0035                                                         + self.tag
0036                                                         + ".db"
0037                                          },
0038                            "residuals" : "ttrig_residuals_" + self.tag + ".db"}
0039         self.output_file = output_file_dict[self.options.workflow_mode]
0040         self.output_files = [self.output_file]
0041
0042     def prepare_workflow(self):
0043         """ Generalized function to prepare workflow dependent on workflow mode"""
0044         function_name = "prepare_" + self.options.workflow_mode + "_" + self.options.command
0045         try:
0046             fill_function = getattr(self, function_name)
0047         except AttributeError:
0048             errmsg = "Class `{}` does not implement `{}`"
0049             raise NotImplementedError( errmsg.format(self.__class__.__name__,
0050                                                      function_name))
0051         log.debug("Preparing workflow with function %s" % function_name)
0052         # call chosen function
0053         fill_function()
0054
0055     def get_output_db(self, workflow_mode, command):
0056         output_db_file = self.output_db_dict[workflow_mode]
0057         if isinstance(output_db_file, dict):
0058             return output_db_file[command]
0059         return output_db_file
0060     ####################################################################
0061     #                     Prepare functions for timeboxes              #
0062     ####################################################################
0063     def prepare_timeboxes_submit(self):
0064         self.pset_name = 'dtTTrigCalibration_cfg.py'
0065         self.pset_template = 'CalibMuon.DTCalibration.dtTTrigCalibration_cfg'
0066         if self.options.datasettype == "Cosmics":
0067             self.pset_template = 'CalibMuon.DTCalibration.dtTTrigCalibration_cosmics_cfg'
0068         log.debug('Using pset template ' + self.pset_template)
0069         self.process = tools.loadCmsProcess(self.pset_template)
0070         self.process.GlobalTag.globaltag = self.options.globaltag
0071         self.process.dtTTrigCalibration.rootFileName = self.output_file
0072         self.process.dtTTrigCalibration.digiLabel = self.digilabel
0073
0074         if self.options.inputDBTag:
0075             self.add_local_custom_db()
0076         self.prepare_common_submit()
0077         self.write_pset_file()
0078
0079     def prepare_timeboxes_check(self):
0080         self.load_options_command("submit")
0081
0082     def prepare_timeboxes_write(self):
0083         self.output_db_file = self.output_db_dict[self.options.workflow_mode]
0084         if isinstance(self.output_db_dict[self.options.workflow_mode], dict):
0085             self.output_db_file = self.output_db_file[self.options.command]
0086         self.prepare_common_write()
0087         merged_file = os.path.join(self.result_path, "Run"+str(self.options.run)+"_"+self.output_file)
0088         ttrig_uncorrected_db = os.path.join(self.result_path,
0089                                             self.get_output_db("timeboxes", "write"))
0090         self.pset_name = 'dtTTrigWriter_cfg.py'
0091         self.pset_template = "CalibMuon.DTCalibration.dtTTrigWriter_cfg"
0092         self.process = tools.loadCmsProcess(self.pset_template)
0093         self.process.dtTTrigWriter.rootFileName = "file:///" + merged_file
0094         self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_uncorrected_db
0095         self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
0096         self.write_pset_file()
0097
0098     def prepare_timeboxes_correction(self):
0099         self.load_options_command("submit")
0100         self.pset_name = 'dtTTrigCorrection_cfg.py'
0101         self.pset_template = "CalibMuon.DTCalibration.dtTTrigCorrection_cfg"
0102         ttrig_timeboxes_db = "ttrig_timeboxes_"+ self.tag + ".db"
0103         ttrig_timeboxes_db = os.path.join(self.result_path,
0104                                           self.get_output_db("timeboxes", "correction"))
0105         self.process = tools.loadCmsProcess(self.pset_template)
0106         self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
0107         self.process.source.firstRun = self.options.run
0108         self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_timeboxes_db
0109
0110         if not self.options.inputCalibDB:
0111             self.options.inputCalibDB = os.path.join(self.result_path,
0112                                             self.get_output_db( "timeboxes", "write" ) )
0113         if self.options.inputVDriftDB:
0114             log.warning("Options inputVDriftDB has no effect for timeboxes correction")
0115         if self.options.inputT0DB:
0116             log.warning("Options inputT0DB has no effect for timeboxes correction")
0117         self.add_local_calib_db(local=True)
0118         self.write_pset_file()
0119
0120     def prepare_timeboxes_dump(self):
0121         self.pset_name = 'dumpDBToFile_ResidCorr_cfg.py'
0122         self.pset_template = 'CalibMuon.DTCalibration.dumpDBToFile_ttrig_cfg'
0123         if self.options.input_dumpDB:
0124             try:
0125                 test = self.result_path
0126                 self.load_options_command("correction")
0127             except:
0128                 pass
0129             dbpath = os.path.abspath(self.options.input_dumpDB)
0130         else:
0131             dbpath = os.path.abspath( os.path.join(self.result_path,
0132                                                    self.get_output_db("timeboxes",
0133                                                                       "correction")))
0134         self.prepare_common_dump(dbpath)
0135         self.write_pset_file()
0136
0137     def prepare_timeboxes_all(self):
0138         # prepare functions for all tasks that will be called in the main implementation of 'all'
0139         self.all_commands=["submit", "check", "write", "correction", "dump"]
0140
0141     ####################################################################
0142     #                      prepare functions for residuals             #
0143     ####################################################################
0144     def prepare_residuals_submit(self):
0145         self.pset_name = 'dtResidualCalibration_cfg.py'
0146         self.pset_template = 'CalibMuon.DTCalibration.dtResidualCalibration_cfg'
0147         if self.options.datasettype == "Cosmics":
0148             self.pset_template = 'CalibMuon.DTCalibration.dtResidualCalibration_cosmics_cfg'
0149         self.process = tools.loadCmsProcess(self.pset_template)
0150         self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
0151         self.process.dtResidualCalibration.rootFileName = self.output_file
0152         self.prepare_common_submit()
0153         if self.options.fromMuons:
0154             self.process.dtResidualCalibration.Muons = cms.InputTag("muons")
0155         if self.options.histoRange:
0156             self.process.dtResidualCalibration.histogramRange = cms.double(float(self.options.histoRange))
0157         if self.options.inputCalibDB:
0158             self.add_local_calib_db()
0159         self.write_pset_file()
0160
0161     def prepare_residuals_check(self):
0162         self.load_options_command("submit")
0163
0164     def prepare_residuals_correction(self):
0165         self.pset_name = "dtTTrigResidualCorrection_cfg.py"
0166         self.pset_template = 'CalibMuon.DTCalibration.dtTTrigResidualCorrection_cfg'
0167         self.process = tools.loadCmsProcess(self.pset_template)
0168         self.load_options_command("submit")
0169         self.process.source.firstRun = cms.untracked.uint32(self.options.run)
0170         self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
0171
0172         self.prepare_common_write()
0173         if self.options.inputT0DB:
0174             log.warning("Option inputT0DB not supported for residual corrections")
0175
0176         if self.options.inputDBTag:
0177             self.add_local_custom_db(local=True)
0178         if self.options.inputVDriftDB:
0179             self.add_local_vdrift_db(local=True)
0180         if self.options.inputCalibDB:
0181             self.add_local_calib_db(local=True)
0182         # Change DB label if running on Cosmics
0183         if self.options.datasettype == "Cosmics":
0184             self.process.dtTTrigResidualCorrection.dbLabel = 'cosmics'
0185             self.process.dtTTrigResidualCorrection.correctionAlgoConfig.dbLabel = 'cosmics'
0186         ttrig_ResidCorr_db = os.path.abspath( os.path.join(self.result_path,
0187                                               self.get_output_db("residuals", "write")))
0188         self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_ResidCorr_db
0189         merged_file = os.path.join(self.result_path, "Run"+str(self.options.run)+"_"+self.output_file)
0190         self.process.dtTTrigResidualCorrection.correctionAlgoConfig.residualsRootFile = merged_file
0191         self.write_pset_file()
0192
0193     def prepare_residuals_dump(self):
0194         self.pset_name = 'dumpDBToFile_ResidCorr_cfg.py'
0195         self.pset_template = 'CalibMuon.DTCalibration.dumpDBToFile_ttrig_cfg'
0196         if self.options.input_dumpDB:
0197             dbpath = os.path.abspath(self.options.input_dumpDB)
0198         else:
0199             try:
0200                 test = self.result_path
0201                 self.load_options_command("write")
0202             except:
0203                 pass
0204             dbpath = os.path.abspath( os.path.join(self.result_path,
0205                                                    self.get_output_db("residuals",
0206                                                                       "write")))
0207         self.prepare_common_dump(dbpath)
0208         self.write_pset_file()
0209
0210     def prepare_residuals_all(self):
0211         # individual prepare functions for all tasks will be called in
0212         # main implementation of all
0213         self.all_commands=["submit", "check", "correction", "dump"]
0214
0215     ####################################################################
0216     #                   prepare functions for validation               #
0217     ####################################################################
0218
0219     def prepare_validation_submit(self):
0220         self.pset_name = 'dtCalibValidation_cfg.py'
0221         self.pset_template = 'CalibMuon.DTCalibration.dtCalibValidation_cfg'
0222         if self.options.datasettype == "Cosmics":
0223             self.pset_template = 'CalibMuon.DTCalibration.dtCalibValidation_cosmics_cfg'
0224         if self.options.fromMuons:
0225             self.pset_template = 'CalibMuon.DTCalibration.dtCalibValidationFromMuons_cfg'
0226         self.process = tools.loadCmsProcess(self.pset_template)
0227         self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
0228         self.prepare_common_submit()
0229         if self.options.inputCalibDB:
0230             self.add_local_calib_db()
0231         self.write_pset_file()
0232
0233     def prepare_validation_check(self):
0234         self.load_options_command("submit")
0235
0236     def prepare_validation_write(self):
0237         self.pset_name = 'dtDQMClient_cfg.py'
0238         self.pset_template = 'CalibMuon.DTCalibration.dtDQMClient_cfg'
0239         self.process = tools.loadCmsProcess(self.pset_template)
0240         (crab_tag, crab_folder) = self.prepare_common_write(do_hadd = False)
0241         print("Crab Folder:",  crab_folder)
0242         print("Crab Tag:", crab_tag, "\t Run number =", self.options.run)
0243         #print("local path:", self.local_path)
0244         dqm_files = glob.glob(os.path.join( crab_folder, 'results', "DQM_*.root"))
0245         dqm_files[:] = ["file://"+txt for txt in dqm_files]
0246
0247         self.process.source.fileNames =  dqm_files
0248         self.process.dqmSaver.dirName = os.path.abspath(self.result_path)
0249         # This parameter does not matter, but it has to be in the format /A/B/C:
0250         self.process.dqmSaver.workflow = "/DT/Calib/Validation"
0251         #if self.process.DQMStore.collateHistograms == True:
0252         self.process.dqmSaver.forceRunNumber = self.options.run
0253         self.write_pset_file()
0254
0255     def summary(self):
0256         pass
0257
0258     def prepare_validation_all(self):
0259         # individual prepare functions for all tasks will be called in
0260         # main implementation of all
0261         self.all_commands=["submit", "check","write"]
0262
0263     ####################################################################
0264     #                           CLI creation                           #
0265     ####################################################################
0266     @classmethod
0267     def add_parser_options(cls, subparser_container):
0268         ttrig_parser = subparser_container.add_parser( "ttrig",
0269         #parents=[mutual_parent_parser, common_parent_parser],
0270         help = "" ) # What does ttrig
0271
0272
0273         ################################################################
0274         #                Sub parser options for workflow modes         #
0275         ################################################################
0276         ttrig_subparsers = ttrig_parser.add_subparsers( dest="workflow_mode",
0277             help="Possible workflow modes",)
0278         ## Add all workflow modes for ttrig
0279         ttrig_timeboxes_subparser = ttrig_subparsers.add_parser( "timeboxes",
0280             help = "" )
0281         ttrig_residuals_subparser = ttrig_subparsers.add_parser( "residuals",
0282             help = "" )
0283         ttrig_validation_subparser = ttrig_subparsers.add_parser( "validation",
0284             help = "" )
0285         ################################################################
0286         #        Sub parser options for workflow mode timeboxes        #
0287         ################################################################
0288         ttrig_timeboxes_subparsers = ttrig_timeboxes_subparser.add_subparsers( dest="command",
0289             help="Possible commands for timeboxes")
0290         ttrig_timeboxes_submit_parser = ttrig_timeboxes_subparsers.add_parser(
0291             "submit",
0292             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0293                     super(DTttrigWorkflow,cls).get_submission_options_parser(),
0294                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0295             help = "Submit job to the GRID via crab3")
0296
0297         ttrig_timeboxes_check_parser = ttrig_timeboxes_subparsers.add_parser(
0298             "check",
0299             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0300                     super(DTttrigWorkflow,cls).get_check_options_parser(),],
0301             help = "Check status of submitted jobs")
0302
0303         ttrig_timeboxes_write_parser = ttrig_timeboxes_subparsers.add_parser(
0304             "write",
0305             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0306                      super(DTttrigWorkflow,cls).get_write_options_parser()
0307                     ],
0308             help = "Write result from root output to text file")
0309
0310         ttrig_timeboxes_correction_parser = ttrig_timeboxes_subparsers.add_parser(
0311             "correction",
0312             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0313                      super(DTttrigWorkflow,cls).get_local_input_db_options_parser()
0314                     ],
0315             help = "Perform correction on uncorrected ttrig database")
0316
0317         ttrig_timeboxes_dump_parser = ttrig_timeboxes_subparsers.add_parser(
0318             "dump",
0319             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0320                      super(DTttrigWorkflow,cls).get_dump_options_parser()],
0321             help = "Dump database to text file")
0322
0323         ttrig_timeboxes_all_parser = ttrig_timeboxes_subparsers.add_parser(
0324             "all",
0325             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0326                      super(DTttrigWorkflow,cls).get_submission_options_parser(),
0327                      super(DTttrigWorkflow,cls).get_check_options_parser(),
0328                      super(DTttrigWorkflow,cls).get_input_db_options_parser(),
0329                      super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0330                      super(DTttrigWorkflow,cls).get_write_options_parser(),
0331                      super(DTttrigWorkflow,cls).get_dump_options_parser()
0332                     ],
0333             help = "Perform all steps: submit, check, write, correction,"\
0334                    "dump in this order")
0335
0336         ################################################################
0337         #        Sub parser options for workflow mode residuals        #
0338         ################################################################
0339         ttrig_residuals_subparsers = ttrig_residuals_subparser.add_subparsers( dest="command",
0340             help="Possible commands for residuals")
0341         ttrig_residuals_submit_parser = ttrig_residuals_subparsers.add_parser(
0342             "submit",
0343             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0344                     super(DTttrigWorkflow,cls).get_submission_options_parser(),
0345                     super(DTttrigWorkflow,cls).get_check_options_parser(),
0346                     super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0347                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0348             help = "Submit job to the GRID via crab3")
0349
0350         ttrig_residuals_check_parser = ttrig_residuals_subparsers.add_parser(
0351             "check",
0352             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0353                     super(DTttrigWorkflow,cls).get_check_options_parser(),],
0354             help = "Check status of submitted jobs")
0355
0356         ttrig_residuals_correct_parser = ttrig_residuals_subparsers.add_parser(
0357             "correction",
0358             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0359                     super(DTttrigWorkflow,cls).get_write_options_parser(),
0360                     super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0361                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0362             help = "Perform residual corrections")
0363         ttrig_residuals_correct_parser.add_argument("--globaltag",
0364             help="Alternative globalTag. Otherwise the gt for sunmission is used")
0365
0366         ttrig_residuals_dump_parser = ttrig_residuals_subparsers.add_parser(
0367             "dump",
0368             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0369                      super(DTttrigWorkflow,cls).get_dump_options_parser()],
0370             help = "Dump database to text file")
0371
0372         ttrig_residuals_all_parser = ttrig_residuals_subparsers.add_parser(
0373             "all",
0374             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0375                     super(DTttrigWorkflow,cls).get_submission_options_parser(),
0376                     super(DTttrigWorkflow,cls).get_check_options_parser(),
0377                     super(DTttrigWorkflow,cls).get_write_options_parser(),
0378                     super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0379                     super(DTttrigWorkflow,cls).get_dump_options_parser(),
0380                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0381             help = "Perform all steps: submit, check, correct, dump")
0382
0383         ################################################################
0384         #        Sub parser options for workflow mode validation       #
0385         ################################################################
0386         ttrig_validation_subparsers = ttrig_validation_subparser.add_subparsers( dest="command",
0387             help="Possible commands for residuals")
0388
0389         ttrig_validation_submit_parser = ttrig_validation_subparsers.add_parser(
0390             "submit",
0391             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0392                     super(DTttrigWorkflow,cls).get_submission_options_parser(),
0393                     super(DTttrigWorkflow,cls).get_check_options_parser(),
0394                     super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0395                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0396             help = "Submit job to the GRID via crab3")
0397
0398
0399         ttrig_validation_check_parser = ttrig_validation_subparsers.add_parser(
0400             "check",
0401             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0402                     super(DTttrigWorkflow,cls).get_check_options_parser(),],
0403             help = "Check status of submitted jobs")
0404
0405         ttrig_validation_write_parser = ttrig_validation_subparsers.add_parser(
0406             "write",
0407             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0408                     super(DTttrigWorkflow,cls).get_write_options_parser()],
0409             help = "Create summary for validation")
0410
0411         ttrig_validation_all_parser = ttrig_validation_subparsers.add_parser(
0412             "all",
0413             parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
0414                     super(DTttrigWorkflow,cls).get_submission_options_parser(),
0415                     super(DTttrigWorkflow,cls).get_check_options_parser(),
0416                     super(DTttrigWorkflow,cls).get_write_options_parser(),
0417                     super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
0418                     super(DTttrigWorkflow,cls).get_input_db_options_parser()],
0419             help = "Perform all steps: submit, check, summary")