File indexing completed on 2021-10-31 23:54:40
0001 from __future__ import print_function
0002 from __future__ import absolute_import
0003 import os
0004 import re
0005 import sys
0006 import shutil
0007 import subprocess
0008 import urllib.request
0009 import multiprocessing
0010
0011 import ROOT
0012 ROOT.gROOT.SetBatch(True)
0013 ROOT.PyConfig.IgnoreCommandLineOptions = True
0014
0015 from . import plotting
0016 from . import html
0017
0018
0019 _globalTags = {
0020 "CMSSW_6_2_0": {"default": "PRE_ST62_V8"},
0021 "CMSSW_6_2_0_SLHC15": {"UPG2019withGEM": "DES19_62_V8", "UPG2023SHNoTaper": "DES23_62_V1"},
0022 "CMSSW_6_2_0_SLHC17": {"UPG2019withGEM": "DES19_62_V8", "UPG2023SHNoTaper": "DES23_62_V1"},
0023 "CMSSW_6_2_0_SLHC20": {"UPG2019withGEM": "DES19_62_V8", "UPG2023SHNoTaper": "DES23_62_V1_UPG2023SHNoTaper"},
0024 "CMSSW_6_2_0_SLHC22": {"UPG2023SHNoTaper": "PH2_1K_FB_V6_UPG23SHNoTaper",
0025
0026 "2023GReco": "PH2_1K_FB_V6_UPG23SHNoTaper", "2023GRecoPU35": "", "2023GRecoPU140": "", "2023GRecoPU200": "",
0027 "2023tilted": "PH2_1K_FB_V6_UPG23SHNoTaper", "2023tiltedPU35": "", "2023tiltedPU140": "", "2023tiltedPU200": ""},
0028 "CMSSW_6_2_0_SLHC26": {"LHCCRefPU140": "DES23_62_V1_LHCCRefPU140", "LHCCRefPU200": "DES23_62_V1_LHCCRefPU200",
0029
0030 "2023GReco": "", "2023GRecoPU35": "", "2023GRecoPU140": "DES23_62_V1_LHCCRefPU140", "2023GRecoPU200": "DES23_62_V1_LHCCRefPU200",
0031 "2023tilted": "", "2023tiltedPU35": "", "2023tiltedPU140": "DES23_62_V1_LHCCRefPU140", "2023tiltedPU200": "DES23_62_V1_LHCCRefPU200"},
0032 "CMSSW_6_2_0_SLHC27_phase1": {"default": "DES17_62_V8_UPG17"},
0033 "CMSSW_7_0_0": {"default": "POSTLS170_V3", "fullsim_50ns": "POSTLS170_V4"},
0034 "CMSSW_7_0_0_AlcaCSA14": {"default": "POSTLS170_V5_AlcaCSA14", "fullsim_50ns": "POSTLS170_V6_AlcaCSA14"},
0035 "CMSSW_7_0_7_pmx": {"default": "PLS170_V7AN1", "fullsim_50ns": "PLS170_V6AN1"},
0036 "CMSSW_7_0_9_patch3": {"default": "PLS170_V7AN2", "fullsim_50ns": "PLS170_V6AN2"},
0037 "CMSSW_7_0_9_patch3_Premix": {"default": "PLS170_V7AN2", "fullsim_50ns": "PLS170_V6AN2"},
0038 "CMSSW_7_1_0": {"default": "POSTLS171_V15", "fullsim_50ns": "POSTLS171_V16"},
0039 "CMSSW_7_1_9": {"default": "POSTLS171_V17", "fullsim_50ns": "POSTLS171_V18"},
0040 "CMSSW_7_1_9_patch2": {"default": "POSTLS171_V17", "fullsim_50ns": "POSTLS171_V18"},
0041 "CMSSW_7_1_10_patch2": {"default": "MCRUN2_71_V1", "fullsim_50ns": "MCRUN2_71_V0"},
0042 "CMSSW_7_2_0_pre5": {"default": "POSTLS172_V3", "fullsim_50ns": "POSTLS172_V4"},
0043 "CMSSW_7_2_0_pre7": {"default": "PRE_LS172_V11", "fullsim_50ns": "PRE_LS172_V12"},
0044 "CMSSW_7_2_0_pre8": {"default": "PRE_LS172_V15", "fullsim_50ns": "PRE_LS172_V16"},
0045 "CMSSW_7_2_0": {"default": "PRE_LS172_V15", "fullsim_50ns": "PRE_LS172_V16"},
0046 "CMSSW_7_2_0_PHYS14": {"default": "PHYS14_25_V1_Phys14"},
0047 "CMSSW_7_2_2_patch1": {"default": "MCRUN2_72_V1", "fullsim_50ns": "MCRUN2_72_V0"},
0048 "CMSSW_7_2_2_patch1_Fall14DR": {"default": "MCRUN2_72_V3_71XGENSIM"},
0049
0050 "CMSSW_7_3_0_pre1": {"default": "PRE_LS172_V15", "fullsim_50ns": "PRE_LS172_V16"},
0051
0052 "CMSSW_7_3_0_pre2": {"default": "MCRUN2_73_V1", "fullsim_50ns": "MCRUN2_73_V0"},
0053 "CMSSW_7_3_0_pre3": {"default": "MCRUN2_73_V5", "fullsim_50ns": "MCRUN2_73_V4"},
0054 "CMSSW_7_3_0": {"default": "MCRUN2_73_V7", "fullsim_50ns": "MCRUN2_73_V6"},
0055 "CMSSW_7_3_0_71XGENSIM": {"default": "MCRUN2_73_V7_71XGENSIM"},
0056 "CMSSW_7_3_0_71XGENSIM_FIXGT": {"default": "MCRUN2_73_V9_71XGENSIM_FIXGT"},
0057 "CMSSW_7_3_1_patch1": {"default": "MCRUN2_73_V9", "fastsim": "MCRUN2_73_V7"},
0058 "CMSSW_7_3_1_patch1_GenSim_7113": {"default": "MCRUN2_73_V9_GenSim_7113"},
0059 "CMSSW_7_3_3": {"default": "MCRUN2_73_V11", "fullsim_50ns": "MCRUN2_73_V10", "fastsim": "MCRUN2_73_V13"},
0060 "CMSSW_7_4_0_pre1": {"default": "MCRUN2_73_V5", "fullsim_50ns": "MCRUN2_73_V4"},
0061 "CMSSW_7_4_0_pre2": {"default": "MCRUN2_73_V7", "fullsim_50ns": "MCRUN2_73_V6"},
0062 "CMSSW_7_4_0_pre2_73XGENSIM": {"default": "MCRUN2_73_V7_73XGENSIM_Pythia6", "fullsim_50ns": "MCRUN2_73_V6_73XGENSIM_Pythia6"},
0063 "CMSSW_7_4_0_pre5": {"default": "MCRUN2_73_V7", "fullsim_50ns": "MCRUN2_73_V6"},
0064 "CMSSW_7_4_0_pre5_BS": {"default": "MCRUN2_73_V9_postLS1beamspot", "fullsim_50ns": "MCRUN2_73_V8_postLS1beamspot"},
0065 "CMSSW_7_4_0_pre6": {"default": "MCRUN2_74_V1", "fullsim_50ns": "MCRUN2_74_V0"},
0066 "CMSSW_7_4_0_pre8": {"default": "MCRUN2_74_V7", "fullsim_25ns": "MCRUN2_74_V5_AsympMinGT", "fullsim_50ns": "MCRUN2_74_V4_StartupMinGT"},
0067 "CMSSW_7_4_0_pre8_minimal": {"default": "MCRUN2_74_V5_MinGT", "fullsim_25ns": "MCRUN2_74_V5_AsympMinGT", "fullsim_50ns": "MCRUN2_74_V4_StartupMinGT"},
0068 "CMSSW_7_4_0_pre8_25ns_asymptotic": {"default": "MCRUN2_74_V7"},
0069 "CMSSW_7_4_0_pre8_50ns_startup": {"default": "MCRUN2_74_V6"},
0070 "CMSSW_7_4_0_pre8_50ns_asympref": {"default": "MCRUN2_74_V5A_AsympMinGT"},
0071 "CMSSW_7_4_0_pre8_50ns_asymptotic": {"default": "MCRUN2_74_V7A_AsympGT"},
0072 "CMSSW_7_4_0_pre8_ROOT6": {"default": "MCRUN2_74_V7"},
0073 "CMSSW_7_4_0_pre8_pmx": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0074 "CMSSW_7_4_0_pre8_pmx_v2": {"default": "MCRUN2_74_V7_gs_pre7", "fullsim_50ns": "MCRUN2_74_V6_gs_pre7"},
0075 "CMSSW_7_4_0_pre8_pmx_v3": {"default": "MCRUN2_74_V7_bis", "fullsim_50ns": "MCRUN2_74_V6_bis"},
0076 "CMSSW_7_4_0_pre9": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0077 "CMSSW_7_4_0_pre9_ROOT6": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0078 "CMSSW_7_4_0_pre9_extended": {"default": "MCRUN2_74_V7_extended"},
0079 "CMSSW_7_4_0": {"default": "MCRUN2_74_V7_gensim_740pre7", "fullsim_50ns": "MCRUN2_74_V6_gensim_740pre7", "fastsim": "MCRUN2_74_V7"},
0080 "CMSSW_7_4_0_71XGENSIM": {"default": "MCRUN2_74_V7_GENSIM_7_1_15", "fullsim_50ns": "MCRUN2_74_V6_GENSIM_7_1_15"},
0081 "CMSSW_7_4_0_71XGENSIM_PU": {"default": "MCRUN2_74_V7_gs7115_puProd", "fullsim_50ns": "MCRUN2_74_V6_gs7115_puProd"},
0082 "CMSSW_7_4_0_71XGENSIM_PXworst": {"default": "MCRUN2_74_V7C_pxWorst_gs7115", "fullsim_50ns": "MCRUN2_74_V6A_pxWorst_gs7115"},
0083 "CMSSW_7_4_0_71XGENSIM_PXbest": {"default": "MCRUN2_74_V7D_pxBest_gs7115", "fullsim_50ns": "MCRUN2_74_V6B_pxBest_gs7115"},
0084 "CMSSW_7_4_0_pmx": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0085 "CMSSW_7_4_1": {"default": "MCRUN2_74_V9_gensim_740pre7", "fullsim_50ns": "MCRUN2_74_V8_gensim_740pre7", "fastsim": "MCRUN2_74_V9"},
0086 "CMSSW_7_4_1_71XGENSIM": {"default": "MCRUN2_74_V9_gensim71X", "fullsim_50ns": "MCRUN2_74_V8_gensim71X"},
0087 "CMSSW_7_4_1_extended": {"default": "MCRUN2_74_V9_extended"},
0088 "CMSSW_7_4_3": {"default": "MCRUN2_74_V9", "fullsim_50ns": "MCRUN2_74_V8", "fastsim": "MCRUN2_74_V9", "fastsim_25ns": "MCRUN2_74_V9_fixMem"},
0089 "CMSSW_7_4_3_extended": {"default": "MCRUN2_74_V9_ext","fastsim": "MCRUN2_74_V9_fixMem"},
0090 "CMSSW_7_4_3_pmx": {"default": "MCRUN2_74_V9_ext", "fullsim_50ns": "MCRUN2_74_V8", "fastsim": "MCRUN2_74_V9_fixMem"},
0091 "CMSSW_7_4_3_patch1_unsch": {"default": "MCRUN2_74_V9_unsch", "fullsim_50ns": "MCRUN2_74_V8_unsch"},
0092 "CMSSW_7_4_4": {"default": "MCRUN2_74_V9_38Tbis", "fullsim_50ns": "MCRUN2_74_V8_38Tbis"},
0093 "CMSSW_7_4_4_0T": {"default": "MCRUN2_740TV1_0Tv2", "fullsim_50ns": "MCRUN2_740TV0_0TV2", "fullsim_25ns": "MCRUN2_740TV1_0TV2"},
0094 "CMSSW_7_4_6_patch6": {"default": "MCRUN2_74_V9_scheduled", "fullsim_50ns": "MCRUN2_74_V8_scheduled"},
0095 "CMSSW_7_4_6_patch6_unsch": {"default": "MCRUN2_74_V9", "fullsim_50ns": "MCRUN2_74_V8"},
0096 "CMSSW_7_4_6_patch6_noCCC": {"default": "MCRUN2_74_V9_unsch_noCCC", "fullsim_50ns": "MCRUN2_74_V8_unsch_noCCC"},
0097 "CMSSW_7_4_6_patch6_noCCC_v3": {"default": "MCRUN2_74_V9_unsch_noCCC_v3", "fullsim_50ns": "MCRUN2_74_V8_unsch_noCCC_v3"},
0098 "CMSSW_7_4_6_patch6_BS": {"default": "74X_mcRun2_asymptotic_realisticBS_v0_2015Jul24", "fullsim_50ns": "74X_mcRun2_startup_realisticBS_v0_2015Jul24PU", "fullsim_25ns": "74X_mcRun2_asymptotic_realisticBS_v0_2015Jul24PU"},
0099 "CMSSW_7_4_8_patch1_MT": {"default": "MCRUN2_74_V11_mulTrh", "fullsim_50ns": "MCRUN2_74_V10_mulTrh",},
0100 "CMSSW_7_4_12": {"default": "74X_mcRun2_asymptotic_v2", "fullsim_25ns": "74X_mcRun2_asymptotic_v2_v2", "fullsim_50ns": "74X_mcRun2_startup_v2_v2"},
0101 "CMSSW_7_5_0_pre1": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0102 "CMSSW_7_5_0_pre2": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0103 "CMSSW_7_5_0_pre3": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0104 "CMSSW_7_5_0_pre4": {"default": "MCRUN2_75_V1", "fullsim_50ns": "MCRUN2_75_V0"},
0105 "CMSSW_7_5_0_pre5": {"default": "MCRUN2_75_V5", "fullsim_50ns": "MCRUN2_75_V4"},
0106 "CMSSW_7_5_0_pre6": {"default": "75X_mcRun2_asymptotic_v1", "fullsim_50ns": "75X_mcRun2_startup_v1"},
0107 "CMSSW_7_5_0": {"default": "75X_mcRun2_asymptotic_v1", "fullsim_50ns": "75X_mcRun2_startup_v1"},
0108 "CMSSW_7_5_0_71XGENSIM": {"default": "75X_mcRun2_asymptotic_v1_gs7115", "fullsim_50ns": "75X_mcRun2_startup_v1_gs7115"},
0109 "CMSSW_7_5_1": {"default": "75X_mcRun2_asymptotic_v3", "fullsim_50ns": "75X_mcRun2_startup_v3"},
0110 "CMSSW_7_5_1_71XGENSIM": {"default": "75X_mcRun2_asymptotic_v3_gs7118", "fullsim_50ns": "75X_mcRun2_startup_v3_gs7118"},
0111 "CMSSW_7_5_2": {"default": "75X_mcRun2_asymptotic_v5", "fullsim_50ns": "75X_mcRun2_startup_v4"},
0112 "CMSSW_7_6_0_pre1": {"default": "75X_mcRun2_asymptotic_v1", "fullsim_50ns": "75X_mcRun2_startup_v1"},
0113 "CMSSW_7_6_0_pre2": {"default": "75X_mcRun2_asymptotic_v2", "fullsim_50ns": "75X_mcRun2_startup_v2"},
0114 "CMSSW_7_6_0_pre3": {"default": "75X_mcRun2_asymptotic_v2", "fullsim_50ns": "75X_mcRun2_startup_v2"},
0115 "CMSSW_7_6_0_pre4": {"default": "76X_mcRun2_asymptotic_v1", "fullsim_50ns": "76X_mcRun2_startup_v1"},
0116 "CMSSW_7_6_0_pre5": {"default": "76X_mcRun2_asymptotic_v1", "fullsim_50ns": "76X_mcRun2_startup_v1"},
0117 "CMSSW_7_6_0_pre6": {"default": "76X_mcRun2_asymptotic_v4", "fullsim_50ns": "76X_mcRun2_startup_v4"},
0118 "CMSSW_7_6_0_pre7": {"default": "76X_mcRun2_asymptotic_v5", "fullsim_50ns": "76X_mcRun2_startup_v5", "fastsim": "76X_mcRun2_asymptotic_v5_resub"},
0119 "CMSSW_7_6_0": {"default": "76X_mcRun2_asymptotic_v11", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0120 "CMSSW_7_6_0_71XGENSIM": {"default": "76X_mcRun2_asymptotic_v11_gs7120p2rlBS", "fullsim_50ns": "76X_mcRun2_startup_v11_gs7120p2rlBS"},
0121 "CMSSW_7_6_2": {"default": "76X_mcRun2_asymptotic_v12", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0122 "CMSSW_7_6_3_patch2_0T": {"default": "76X_mcRun2_0T_v1_0Tv1GT"},
0123 "CMSSW_8_0_0_pre1": {"default": "76X_mcRun2_asymptotic_v11", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0124 "CMSSW_8_0_0_pre2": {"default": "76X_mcRun2_asymptotic_v12", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0125 "CMSSW_8_0_0_pre2_phase1": {"default": "76X_upgrade2017_design_v7"},
0126 "CMSSW_8_0_0_pre2_phase1_rereco": {"default": "76X_upgrade2017_design_v7_rereco"},
0127 "CMSSW_8_0_0_pre3_phase1": {"default": "76X_upgrade2017_design_v8"},
0128 "CMSSW_8_0_0_pre3_phase1_pythia8": {"default": "76X_upgrade2017_design_v8_pythia8"},
0129 "CMSSW_8_0_0_pre4": {"default": "76X_mcRun2_asymptotic_v13", "fullsim_50ns": "76X_mcRun2_startup_v12"},
0130 "CMSSW_8_0_0_pre4_phase1": {"default": "76X_upgrade2017_design_v8_UPG17"},
0131 "CMSSW_8_0_0_pre4_phase1_13TeV": {"default": "76X_upgrade2017_design_v8_UPG17"},
0132 "CMSSW_8_0_0_pre4_ecal15fb": {"default": "80X_mcRun2_asymptotic_2016EcalTune_15fb_v0_ecal15fbm1"},
0133 "CMSSW_8_0_0_pre4_ecal30fb": {"default": "80X_mcRun2_asymptotic_2016EcalTune_30fb_v0_ecal30fbm1"},
0134 "CMSSW_8_0_0_pre5": {"default": "80X_mcRun2_asymptotic_v1", "fullsim_50ns": "80X_mcRun2_startup_v1"},
0135 "CMSSW_8_0_0_pre5_phase1": {"default": "80X_upgrade2017_design_v1_UPG17"},
0136 "CMSSW_8_0_0_pre6": {"default": "80X_mcRun2_asymptotic_v4"},
0137 "CMSSW_8_0_0_pre6_phase1": {"default": "80X_upgrade2017_design_v3_UPG17"},
0138 "CMSSW_8_0_0_pre6_MT": {"default": "80X_mcRun2_asymptotic_v4_multiCoreResub"},
0139 "CMSSW_8_0_0": {"default": "80X_mcRun2_asymptotic_v4"},
0140 "CMSSW_8_0_0_patch1_phase1": {"default": "80X_upgrade2017_design_v4_UPG17"},
0141 "CMSSW_8_0_0_patch1_phase1_rereco": {"default": "80X_upgrade2017_design_v4_UPG17_rereco"},
0142 "CMSSW_8_0_0_patch2": {"default": "80X_mcRun2_asymptotic_v5_refGT"},
0143 "CMSSW_8_0_0_patch2_pixDynIneff": {"default": "80X_mcRun2_asymptotic_v5_2016PixDynIneff_targetGT"},
0144
0145
0146 "CMSSW_8_0_0_patch2": {"default": "80X_mcRun2_asymptotic_v5_refGT_resub"},
0147 "CMSSW_8_0_0_patch2_pixDynIneff": {"default": "80X_mcRun2_asymptotic_v5_2016PixDynIneff_targetGT_resub"},
0148 "CMSSW_8_0_1": {"default": "80X_mcRun2_asymptotic_v6"},
0149 "CMSSW_8_0_1_71XGENSIM": {"default": "80X_mcRun2_asymptotic_v6_gs7120p2"},
0150 "CMSSW_8_0_1_gcc530": {"default": "80X_mcRun2_asymptotic_v6_gcc530"},
0151 "CMSSW_8_0_3_71XGENSIM": {"default": "80X_mcRun2_asymptotic_2016_v3_gs7120p2NewGTv3"},
0152 "CMSSW_8_0_3_71XGENSIM_hcal": {"default": "80X_mcRun2_asymptotic_2016_v3_gs71xNewGtHcalCust"},
0153 "CMSSW_8_0_3_71XGENSIM_tec": {"default": "80X_mcRun2_asymptotic_SiStripBad_TEC_CL62_for2016_v1_mc_gs7120p2TrkCoolLoop"},
0154 "CMSSW_8_0_5": {"default": "80X_mcRun2_asymptotic_v12_gs7120p2", "fastsim": "80X_mcRun2_asymptotic_v12"},
0155 "CMSSW_8_0_5_pmx": {"default": "80X_mcRun2_asymptotic_v12_gs7120p2_resub", "fastsim": "80X_mcRun2_asymptotic_v12"},
0156 "CMSSW_8_0_10": {"default": "80X_mcRun2_asymptotic_v14"},
0157 "CMSSW_8_0_10_patch1_BS": {"default": "80X_mcRun2_asymptotic_RealisticBS_25ns_13TeV2016_v1_mc_realisticBS2016"},
0158 "CMSSW_8_0_11": {"default": "80X_mcRun2_asymptotic_v14"},
0159 "CMSSW_8_0_15": {"default": "80X_mcRun2_asymptotic_v16_gs7120p2", "fastsim": "80X_mcRun2_asymptotic_v16"},
0160 "CMSSW_8_0_16": {"default": "80X_mcRun2_asymptotic_v16_gs7120p2", "fastsim": "80X_mcRun2_asymptotic_v16"},
0161 "CMSSW_8_0_16_Tranche4GT": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_gs7120p2_Tranch4GT",
0162 "fastsim": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_Tranch4GT", "RelValTTbar": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_Tr4GT_resub"}, "fastsim_25ns": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_Tranch4GT"},
0163 "CMSSW_8_0_16_Tranche4GT_v2": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v2_Tr4GT_v2"},
0164 "CMSSW_8_0_16_Tranche4GT_pmx": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_gs7120p2_Tranch4GT", "fastsim": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_resub"},
0165 "CMSSW_8_0_19_Tranche4GT_v2": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v2_Tr4GT_v2"},
0166 "CMSSW_8_0_20_Tranche4GT": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v4_Tr4GT_v4"},
0167 "CMSSW_8_0_21_Tranche4GT": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v6_Tr4GT_v6"},
0168 "CMSSW_8_1_0_pre1": {"default": "80X_mcRun2_asymptotic_v6"},
0169 "CMSSW_8_1_0_pre1_phase1": {"default": "80X_upgrade2017_design_v4_UPG17", "fullsim_25ns": "80X_upgrade2017_design_v4_UPG17PU35"},
0170 "CMSSW_8_1_0_pre2": {"default": "80X_mcRun2_asymptotic_v10_gs810pre2", "fastsim": "80X_mcRun2_asymptotic_v10"},
0171 "CMSSW_8_1_0_pre2_phase1": {"default": "80X_upgrade2017_design_v9_UPG17designGT", "fullsim_25ns": "80X_upgrade2017_design_v9_UPG17PU35designGT"},
0172 "CMSSW_8_1_0_pre2_phase1_realGT": {"default": "80X_upgrade2017_realistic_v1_UPG17realGT", "fullsim_25ns": "80X_upgrade2017_realistic_v1_UPG17PU35realGT"},
0173 "CMSSW_8_1_0_pre3": {"default": "80X_mcRun2_asymptotic_v12"},
0174 "CMSSW_8_1_0_pre3_phase1": {"default": "80X_upgrade2017_realistic_v3_UPG17", "fullsim_25ns": "80X_upgrade2017_realistic_v3_UPG17PU35"},
0175 "CMSSW_8_1_0_pre4": {"default": "80X_mcRun2_asymptotic_v13"},
0176 "CMSSW_8_1_0_pre4_phase1": {"default": "80X_upgrade2017_realistic_v4_UPG17", "fullsim_25ns": "80X_upgrade2017_realistic_v4_UPG17PU35"},
0177 "CMSSW_8_1_0_pre5": {"default": "80X_mcRun2_asymptotic_v13"},
0178 "CMSSW_8_1_0_pre5_phase1": {"default": "80X_upgrade2017_realistic_v4_resubUPG17", "fullsim_25ns": "80X_upgrade2017_realistic_v4_resubUPG17PU35"},
0179 "CMSSW_8_1_0_pre6": {"default": "80X_mcRun2_asymptotic_v14"},
0180 "CMSSW_8_1_0_pre6_phase1": {"default": "81X_upgrade2017_realistic_v0_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v0_UPG17PU35"},
0181 "CMSSW_8_1_0_pre7": {"default": "81X_mcRun2_asymptotic_v0"},
0182 "CMSSW_8_1_0_pre7_phase1": {"default": "81X_upgrade2017_realistic_v2_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v2_UPG17PU35"},
0183 "CMSSW_8_1_0_pre7_phase1_newGT": {"default": "81X_upgrade2017_realistic_v3_UPG17newGT", "fullsim_25ns": "81X_upgrade2017_realistic_v3_UPG17PU35newGTresub"},
0184 "CMSSW_8_1_0_pre7_phase2": {"2023GReco": "81X_mcRun2_asymptotic_v0_2023GReco", "2023GRecoPU35": "", "2023GRecoPU140": "81X_mcRun2_asymptotic_v0_2023GRecoPU140resubmit2", "2023GRecoPU200": "81X_mcRun2_asymptotic_v0_2023GRecoPU200resubmit2",
0185 "2023tilted": "81X_mcRun2_asymptotic_v0_2023tilted", "2023tiltedPU35": "", "2023tiltedPU140": "81X_mcRun2_asymptotic_v0_2023tiltedPU140resubmit2", "2023tiltedPU200": "81X_mcRun2_asymptotic_v0_2023tiltedPU200resubmit2"},
0186 "CMSSW_8_1_0_pre8": {"default": "81X_mcRun2_asymptotic_v1"},
0187 "CMSSW_8_1_0_pre8_phase1": {"default": "81X_upgrade2017_realistic_v3_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v3_UPG17PU35"},
0188 "CMSSW_8_1_0_pre8_phase1_newGT": {"default": "81X_upgrade2017_realistic_v4_UPG17newGT", "fullsim_25ns": "81X_upgrade2017_realistic_v4_UPG17PU35newGT"},
0189 "CMSSW_8_1_0_pre8_phase1_newGT2": {"default": "81X_upgrade2017_realistic_v5_UPG17newGTset2", "fullsim_25ns": "81X_upgrade2017_realistic_v5_UPG17PU35newGTset2"},
0190 "CMSSW_8_1_0_pre8_phase2": {"2023GReco": "81X_mcRun2_asymptotic_v1_resub2023GReco", "2023GRecoPU35": "81X_mcRun2_asymptotic_v1_resub2023GRecoPU35", "2023GRecoPU140": "81X_mcRun2_asymptotic_v1_resub2023GRecoPU140", "2023GRecoPU200": "81X_mcRun2_asymptotic_v1_resub2023GRecoPU200",
0191 "2023tilted": "81X_mcRun2_asymptotic_v1_2023tilted", "2023tiltedPU35": "81X_mcRun2_asymptotic_v1_2023tiltedPU", "2023tiltedPU140": "81X_mcRun2_asymptotic_v1_2023tiltedPU140", "2023tiltedPU200": "81X_mcRun2_asymptotic_v1_2023tiltedPU200"},
0192 "CMSSW_8_1_0_pre9": {"default": "81X_mcRun2_asymptotic_v2"},
0193 "CMSSW_8_1_0_pre9_Geant4102": {"default": "81X_mcRun2_asymptotic_v2"},
0194 "CMSSW_8_1_0_pre9_phase1": {"default": "81X_upgrade2017_realistic_v5_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v5_UPG17PU35"},
0195 "CMSSW_8_1_0_pre9_phase1_newGT": {"default": "81X_upgrade2017_realistic_v6_UPG17newGT", "fullsim_25ns": "81X_upgrade2017_realistic_v6_UPG17PU35newGT"},
0196 "CMSSW_8_1_0_pre10": {"default": "81X_mcRun2_asymptotic_v5_recycle", "fullsim_25ns": "81X_mcRun2_asymptotic_v5_resub", "fastsim": "81X_mcRun2_asymptotic_v5"},
0197 "CMSSW_8_1_0_pre10_pmx": {"default": "81X_mcRun2_asymptotic_v5"},
0198 "CMSSW_8_1_0_pre10_phase1": {"default": "81X_upgrade2017_realistic_v9_UPG17resub", "fullsim_25ns": "81X_upgrade2017_realistic_v9_UPG17PU35resub"},
0199 "CMSSW_8_1_0_pre11": {"default": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55", "fullsim_25ns": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55_resub", "fastsim_25ns": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55_resub2"},
0200 "CMSSW_8_1_0_pre11_pmx": {"default": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55"},
0201 "CMSSW_8_1_0_pre11_phase1": {"default": "81X_upgrade2017_realistic_v9_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v9_UPG17PU35"},
0202 "CMSSW_8_1_0_pre12": {"default": "81X_mcRun2_asymptotic_v8", "fullsim_25ns": "81X_mcRun2_asymptotic_v8_resub", "fastsim": "81X_mcRun2_asymptotic_v8_resub", "fastsim_25ns": "81X_mcRun2_asymptotic_v8"},
0203 "CMSSW_8_1_0_pre12_pmx": {"default": "81X_mcRun2_asymptotic_v8_resub", "fastsim_25ns": "81X_mcRun2_asymptotic_v8_rsub"},
0204 "CMSSW_8_1_0_pre12_phase1": {"default": "81X_upgrade2017_realistic_v13"},
0205 "CMSSW_8_1_0_pre12_phase1_newBPix": {"default": "81X_upgrade2017_realistic_newBPix_wAlign_v1_BpixGeom"},
0206 "CMSSW_8_1_0_pre12_phase1_newBPixFPix": {"default": "81X_upgrade2017_realistic_v13_BpixFpixGeom"},
0207 "CMSSW_8_1_0_pre12_phase1_newBPixFPixHCAL": {"default": "81X_upgrade2017_HCALdev_v2_BpixFpixHcalGeom"},
0208 "CMSSW_8_1_0_pre12_phase1_newHCAL": {"default": "81X_upgrade2017_HCALdev_v2_HcalGeom"},
0209 "CMSSW_8_1_0_pre12_phase1_newBPixHCAL": {"default": "81X_upgrade2017_HCALdev_v2_NewBPix_BpixHcalGeom"},
0210 "CMSSW_8_1_0_pre15": {"default": "81X_mcRun2_asymptotic_v11"},
0211 "CMSSW_8_1_0_pre15_HIP": {"default": "81X_mcRun2_asymptotic_v11_hip"},
0212 "CMSSW_8_1_0_pre15_PU": {"default": "81X_mcRun2_asymptotic_v11_M17"},
0213 "CMSSW_8_1_0_pre15_PU_HIP": {"default": "81X_mcRun2_asymptotic_v11_hipM17"},
0214 "CMSSW_8_1_0_pre15_phase1": {"default": "81X_upgrade2017_realistic_v17_BpixFpixHcalGeom",
0215 "Design": "81X_upgrade2017_design_IdealBS_v1_2017design", "Design_fullsim_25ns": "81X_upgrade2017_design_IdealBS_v1_design"},
0216 "CMSSW_8_1_0_pre16": {"default": "81X_mcRun2_asymptotic_v11"},
0217 "CMSSW_8_1_0_pre16_phase1": {"default": "81X_upgrade2017_realistic_v22", "Design": "81X_upgrade2017_design_IdealBS_v6"},
0218 "CMSSW_8_1_0": {"default": "81X_mcRun2_asymptotic_v12"},
0219 "CMSSW_8_1_0_phase1": {"default": "81X_upgrade2017_realistic_v26_HLT2017"},
0220 "CMSSW_9_0_0_pre1": {"default": "90X_mcRun2_asymptotic_v0"},
0221 "CMSSW_9_0_0_pre1_phase1": {"default": "90X_upgrade2017_realistic_v0",
0222 "Design": "90X_upgrade2017_design_IdealBS_v0", "Design_fullsim_25ns": "90X_upgrade2017_design_IdealBS_v0_resub"},
0223 "CMSSW_9_0_0_pre2": {"default": "90X_mcRun2_asymptotic_v0"},
0224 "CMSSW_9_0_0_pre2_ROOT6": {"default": "90X_mcRun2_asymptotic_v0"},
0225 "CMSSW_9_0_0_pre2_phase1": {"default": "90X_upgrade2017_realistic_v0",
0226 "Design": "90X_upgrade2017_design_IdealBS_v0", "Design_fullsim_25ns": "90X_upgrade2017_design_IdealBS_v0_resub"},
0227 "CMSSW_9_0_0_pre4": {"default": "90X_mcRun2_asymptotic_v1", "fullsim_25ns": "90X_mcRun2_asymptotic_v1_resub"},
0228 "CMSSW_9_0_0_pre4_phase1": {"default": "90X_upgrade2017_realistic_v6", "fullsim_25ns_PU50": "Nonexistent",
0229 "Design": "90X_upgrade2017_design_IdealBS_v6"},
0230 "CMSSW_9_0_0_pre4_GS": {"default": "90X_mcRun2_asymptotic_v1_GSval", "fullsim_25ns": "90X_mcRun2_asymptotic_v1_GSval"},
0231 "CMSSW_9_0_0_pre4_phase1_ecalsrb5": {"default": "90X_upgrade2017_realistic_v6_B5"},
0232 "CMSSW_9_0_0_pre4_phase1_ecalsrc1": {"default": "90X_upgrade2017_realistic_v6_C1"},
0233 "CMSSW_9_0_0_pre4_phase1_ecalsrd7": {"default": "90X_upgrade2017_realistic_v6_D7"},
0234 "CMSSW_9_0_0_pre5": {"default": "90X_mcRun2_asymptotic_v4"},
0235 "CMSSW_9_0_0_pre5_pmx": {"default": "90X_mcRun2_asymptotic_v4", "fastsim_25ns": "90X_mcRun2_asymptotic_v4_resub"},
0236 "CMSSW_9_0_0_pre5_phase1": {"default": "90X_upgrade2017_realistic_v15",
0237 "fullsim_25ns_PU35": "90X_upgrade2017_realistic_v15_resub", "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v15_PU50",
0238 "Design": "90X_upgrade2017_design_IdealBS_v15"},
0239 "CMSSW_9_0_0_pre6": {"default": "90X_mcRun2_asymptotic_v4"},
0240 "CMSSW_9_0_0_pre6_phase1": {"default": "90X_upgrade2017_realistic_v15",
0241 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v15_PU50",
0242 "Design": "90X_upgrade2017_design_IdealBS_v15"},
0243 "CMSSW_9_0_0": {"default": "90X_mcRun2_asymptotic_v5"},
0244 "CMSSW_9_0_0_phase1": {"default": "90X_upgrade2017_realistic_v20_resub",
0245 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50_resub",
0246 "Design": "90X_upgrade2017_design_IdealBS_v19_resub"},
0247 "CMSSW_9_0_0_gcc630": {"default": "90X_mcRun2_asymptotic_v5_gcc630"},
0248 "CMSSW_9_0_0_phase1_gcc630": {"default": "90X_upgrade2017_realistic_v20_gcc630"},
0249 "CMSSW_9_0_0_cc7": {"default": "90X_mcRun2_asymptotic_v5_cc7"},
0250 "CMSSW_9_0_0_phase1_cc7": {"default": "90X_upgrade2017_realistic_v20_cc7_rsb"},
0251 "CMSSW_9_0_2_phase1": {"default": "90X_upgrade2017_realistic_v20",
0252 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50",
0253 "Design": "90X_upgrade2017_design_IdealBS_v19"},
0254 "CMSSW_9_1_0_pre1": {"default": "90X_mcRun2_asymptotic_v5"},
0255 "CMSSW_9_1_0_pre1_phase1": {"default": "90X_upgrade2017_realistic_v20",
0256 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50",
0257 "Design": "90X_upgrade2017_design_IdealBS_v19_resub"},
0258 "CMSSW_9_1_0_pre2": {"default": "90X_mcRun2_asymptotic_v5"},
0259 "CMSSW_9_1_0_pre2_phase1": {"default": "90X_upgrade2017_realistic_v20",
0260 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50",
0261 "Design": "90X_upgrade2017_design_IdealBS_v19"},
0262 "CMSSW_9_1_0_pre3": {"default": "91X_mcRun2_asymptotic_v2"},
0263 "CMSSW_9_1_0_pre3_phase1": {"default": "91X_upgrade2017_realistic_v3",
0264 "fullsim_25ns_PU50": "91X_upgrade2017_realistic_v3_PU50_resub",
0265 "Design": "91X_upgrade2017_design_IdealBS_v3"},
0266 "CMSSW_9_2_0": {"default": "91X_mcRun2_asymptotic_v3"},
0267 "CMSSW_9_2_0_phase1": {"default": "91X_upgrade2017_realistic_v5",
0268 "fullsim_25ns_PU50": "91X_upgrade2017_realistic_v5_PU50",
0269 "Design": "91X_upgrade2017_design_IdealBS_v5"},
0270 "CMSSW_9_2_0_phase1_PXmap": {"default": "91X_upgrade2017_realistic_v5_pixel_ideal_PXgeom"},
0271 "CMSSW_9_2_1_phase1": {"default": "92X_upgrade2017_realistic_v1",
0272 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v1_PU50",
0273 "Design": "92X_upgrade2017_design_IdealBS_v1"},
0274 "CMSSW_9_2_2": {"default": "91X_mcRun2_asymptotic_v3"},
0275 "CMSSW_9_2_2_phase1": {"default": "92X_upgrade2017_realistic_v1",
0276 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v1_highPU_AVE50",
0277 "Design": "92X_upgrade2017_design_IdealBS_v1"},
0278 "CMSSW_9_2_4_run1": {"default": "91X_mcRun1_realistic_v2"},
0279 "CMSSW_9_2_4": {"default": "91X_mcRun2_asymptotic_v3"},
0280 "CMSSW_9_2_7_phase1": {"default": "92X_upgrade2017_realistic_v7"},
0281 "CMSSW_9_3_0_pre1": {"default": "92X_mcRun2_asymptotic_v2"},
0282 "CMSSW_9_3_0_pre1_phase1": {"default": "92X_upgrade2017_realistic_v7",
0283 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v7_highPU_AVE50",
0284 "Design": "92X_upgrade2017_design_IdealBS_v7"},
0285 "CMSSW_9_3_0_pre1_run1": {"default": "92X_mcRun1_realistic_v2"},
0286 "CMSSW_9_3_0_pre2": {"default": "92X_mcRun2_asymptotic_v2"},
0287 "CMSSW_9_3_0_pre2_phase1": {"default": "92X_upgrade2017_realistic_v7",
0288 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v7_highPU_AVE50_resub",
0289 "Design": "92X_upgrade2017_design_IdealBS_v7"},
0290 "CMSSW_9_3_0_pre3": {"default": "92X_mcRun2_asymptotic_v2"},
0291 "CMSSW_9_3_0_pre3_phase1": {"default": "92X_upgrade2017_realistic_v10_resub",
0292 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v10_highPU_AVE50_resub",
0293 "Design": "92X_upgrade2017_design_IdealBS_v10_resub"},
0294 "CMSSW_9_3_0_pre3_phase1_pmx": {"default": "92X_upgrade2017_realistic_v10_resub2"},
0295 "CMSSW_9_3_0_pre4": {"default": "93X_mcRun2_asymptotic_v0"},
0296 "CMSSW_9_3_0_pre4_phase1": {"default": "93X_mc2017_realistic_v1",
0297 "fullsim_25ns_PU50": "93X_mc2017_realistic_v1_highPU_AVE50",
0298 "Design": "93X_mc2017_design_IdealBS_v1"},
0299 "CMSSW_9_3_0_pre5": {"default": "93X_mcRun2_asymptotic_v0"},
0300 "CMSSW_9_3_0_pre5_phase1": {"default": "93X_mc2017_realistic_v2",
0301 "fullsim_25ns_PU50": "93X_mc2017_realistic_v2_highPU_AVE50",
0302 "Design": "93X_mc2017_design_IdealBS_v2"},
0303 "CMSSW_9_4_0_pre1": {"default": "93X_mcRun2_asymptotic_v1"},
0304 "CMSSW_9_4_0_pre1_phase1": {"default": "93X_mc2017_realistic_v3",
0305 "fullsim_25ns_PU50": "93X_mc2017_realistic_v3_highPU_AVE50",
0306 "Design": "93X_mc2017_design_IdealBS_v3"},
0307 "CMSSW_9_4_0_pre2": {"default": "93X_mcRun2_asymptotic_v2"},
0308 "CMSSW_9_4_0_pre2_phase1": {"default": "94X_mc2017_realistic_v1",
0309 "fullsim_25ns_PU50": "94X_mc2017_realistic_v1_highPU_AVE50",
0310 "Design": "94X_mc2017_design_IdealBS_v0"},
0311 "CMSSW_9_4_0_pre3": {"default": "94X_mcRun2_asymptotic_v0"},
0312 "CMSSW_9_4_0_pre3_phase1": {"default": "94X_mc2017_realistic_v4",
0313 "fullsim_25ns_PU50": "94X_mc2017_realistic_v4_highPU_AVE50",
0314 "Design": "94X_mc2017_design_IdealBS_v4"},
0315 "CMSSW_9_4_0": {"default": "94X_mcRun2_asymptotic_v0"},
0316 "CMSSW_9_4_0_phase1": {"default": "94X_mc2017_realistic_v10",
0317 "fullsim_25ns_PU50": "94X_mc2017_realistic_v10_highPU_AVE50",
0318 "Design": "94X_mc2017_design_IdealBS_v5"},
0319 "CMSSW_10_0_0_pre1": {"default": "94X_mcRun2_asymptotic_v0"},
0320 "CMSSW_10_0_0_pre1_phase1": {"default": "94X_mc2017_realistic_v10",
0321 "fullsim_25ns_PU50": "94X_mc2017_realistic_v10_highPU_AVE50",
0322 "Design": "94X_mc2017_design_IdealBS_v5"},
0323 "CMSSW_10_0_0_pre2": {"default": "100X_mcRun2_asymptotic_v2"},
0324 "CMSSW_10_0_0_pre2_2017": {"default": "100X_mc2017_realistic_v1",
0325 "fullsim_25ns": "100X_mc2017_realistic_v1_resub",
0326 "fullsim_25ns_PU50": "100X_mc2017_realistic_v1_highPU_AVE50",
0327 "Design": "100X_mc2017_design_IdealBS_v1",
0328 "Design_fullsim_25ns_PU50": "Does_not_exist"},
0329 "CMSSW_10_0_0_pre2_2017_pmx": {"default": "100X_mc2017_realistic_v1"},
0330 "CMSSW_10_0_0_pre2_2018": {"default": "100X_upgrade2018_realistic_v1",
0331 "fullsim_25ns": "100X_upgrade2018_realistic_v1_resub",
0332 "Design": "100X_upgrade2018_design_IdealBS_v1",
0333 "Design_fullsim_25ns": "100X_upgrade2018_design_IdealBS_v1_resub"},
0334 "CMSSW_10_0_0_pre3": {"default": "100X_mcRun2_asymptotic_v2"},
0335 "CMSSW_10_0_0_pre3_2017": {"default": "100X_mc2017_realistic_v1_mahiOFF",
0336 "fullsim_25ns_PU50": "100X_mc2017_realistic_v1_highPU_AVE50_mahiOFF",
0337 "Design": "100X_mc2017_design_IdealBS_v1_mahiOFF"},
0338 "CMSSW_10_0_0_pre3_2018": {"default": "100X_upgrade2018_realistic_v4_mahiOFF",
0339 "Design": "100X_upgrade2018_design_IdealBS_v3_mahiOFF"},
0340 "CMSSW_10_0_0_pre3_2018_pmx": {"default": "100X_upgrade2018_realistic_v4",
0341 "Design": "100X_upgrade2018_design_IdealBS_v3"},
0342 "CMSSW_10_0_0_pre3_2017_mahi": {"default": "100X_mc2017_realistic_v1_mahiON",
0343 "fullsim_25ns_PU50": "100X_mc2017_realistic_v1_highPU_AVE50_mahiON",
0344 "Design": "100X_mc2017_design_IdealBS_v1_mahiON"},
0345 "CMSSW_10_0_0_pre3_2018_mahi": {"default": "100X_upgrade2018_realistic_v4_mahiON",
0346 "Design": "100X_upgrade2018_design_IdealBS_v3_mahiON"},
0347 "CMSSW_10_0_0_pre3_GEANT4_2018_mahi": {"default": "100X_upgrade2018_realistic_v4_mahiON"},
0348 "CMSSW_10_0_0_pre3_G4VecGeom2_2018": {"default": "100X_upgrade2018_realistic_v4"},
0349 }
0350
0351 _releasePostfixes = ["_AlcaCSA14", "_PHYS14", "_TEST", "_v2", "_v3", "_pmx", "_Fall14DR", "_FIXGT", "_PU", "_PXbest", "_PXworst", "_hcal", "_tec", "_71XGENSIM", "_73XGENSIM", "_BS", "_GenSim_7113", "_extended",
0352 "_25ns_asymptotic", "_50ns_startup", "_50ns_asympref", "_50ns_asymptotic", "_minimal", "_0T", "_unsch", "_noCCC", "_MT", "_GS", "_rereco", "_pythia8", "_13TeV", "_realGT", "_newGT2", "_newGT", "_phase1", "_phase2", "_2017", "_2018", "_ecal15fb", "_ecal30fb", "_ecalsrb5", "_ecalsrc1", "_ecalsrd7", "_pixDynIneff", "_PXmap", "_gcc530", "_gcc630", "_cc7", "_Tranche4GT", "_newBPixFPixHCAL", "_newBPixFPix", "_newBPixHCAL", "_newBPix", "_newHCAL", "_HIP", "_run1", "_mahi"]
0353 def _stripRelease(release):
0354 for pf in _releasePostfixes:
0355 if pf in release:
0356 return _stripRelease(release.replace(pf, ""))
0357 return release
0358
0359
0360 def _getGlobalTag(sample, release):
0361 """Get a GlobalTag.
0362
0363 Arguments:
0364 sample -- Sample object
0365 release -- CMSSW release string
0366 """
0367 if not release in _globalTags:
0368 print("Release %s not found from globaltag map in validation.py" % release)
0369 sys.exit(1)
0370 gtmap = _globalTags[release]
0371 selectedGT = None
0372 if sample.hasOverrideGlobalTag():
0373 ogt = sample.overrideGlobalTag()
0374 if release in ogt:
0375 gtmap = _globalTags[ogt[release]]
0376 scenario = ""
0377 if sample.hasScenario():
0378 scenario = sample.scenario()
0379 sims = []
0380 if sample.fullsim():
0381 if sample.pileupEnabled():
0382 sim = "fullsim_"+sample.pileupType()
0383 sims.extend([
0384 sim+"_PU%d"%sample.pileupNumber(),
0385 sim
0386 ])
0387 elif sample.fastsim():
0388 sim = "fastsim"
0389 if sample.pileupEnabled():
0390 sim += "_"+sample.pileupType()
0391 sims.append(sim+"_PU%d"%sample.pileupNumber())
0392 sims.append(sim)
0393
0394 selectedGT = None
0395
0396 if scenario != "":
0397 for sim in sims:
0398 selectedGT = gtmap.get(scenario+"_"+sim, None)
0399 if selectedGT is not None:
0400 break
0401
0402 if selectedGT is None:
0403 selectedGT = gtmap.get(scenario, None)
0404
0405 if selectedGT is None:
0406 for sim in sims:
0407 selectedGT = gtmap.get(sim, None)
0408 if selectedGT is not None:
0409 break
0410
0411 if selectedGT is None:
0412 selectedGT = gtmap["default"]
0413
0414 if isinstance(selectedGT, dict):
0415 return selectedGT.get(sample.name(), selectedGT["default"])
0416 else:
0417 return selectedGT
0418
0419
0420 _relvalUrls = {
0421 "6_2_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_6_2_x/",
0422 "7_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_0_x/",
0423 "7_1_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_1_x/",
0424 "7_2_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_2_x/",
0425 "7_3_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_3_x/",
0426 "7_4_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_4_x/",
0427 "7_5_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_5_x/",
0428 "7_6_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_6_x/",
0429 "8_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_8_0_x/",
0430 "8_1_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_8_1_x/",
0431 "9_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_0_x/",
0432 "9_1_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_1_x/",
0433 "9_2_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_2_x/",
0434 "9_3_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_3_x/",
0435 "9_4_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_4_x/",
0436 "10_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_10_0_x/",
0437 }
0438
0439 _doElectronSamples = [
0440 "RelValTTbar",
0441 "RelValSingleElectronPt35",
0442 "RelValSingleElectronPt10",
0443 ]
0444 _doConversionSamples = [
0445 "RelValTTbar",
0446 "RelValH125GGgluonfusion",
0447 ]
0448 _doBHadronSamples = [
0449 "RelValTTbar"
0450 ]
0451
0452 def _getRelValUrl(release):
0453 """Get RelVal download URL for a given release."""
0454 version_re = re.compile("CMSSW_(?P<X>\d+)_(?P<Y>\d+)")
0455 m = version_re.search(release)
0456 if not m:
0457 raise Exception("Regex %s does not match to release version %s" % (version_re.pattern, release))
0458 version = "%s_%s_X" % (m.group("X"), m.group("Y"))
0459 if not version in _relvalUrls:
0460 print("No RelVal URL for version %s, please update _relvalUrls" % version)
0461 sys.exit(1)
0462 return _relvalUrls[version]
0463
0464 def _processPlotsForSample(plotterFolder, sample):
0465 if plotterFolder.onlyForPileup() and not sample.pileupEnabled():
0466 return False
0467 if plotterFolder.onlyForElectron() and not sample.doElectron():
0468 return False
0469 if plotterFolder.onlyForConversion() and not sample.doConversion():
0470 return False
0471 if plotterFolder.onlyForBHadron() and not sample.doBHadron():
0472 return False
0473 return True
0474
0475 class Sample:
0476 """Represents a RelVal sample."""
0477 def __init__(self, sample, append=None, midfix=None, putype=None, punum=0,
0478 fastsim=False, fastsimCorrespondingFullsimPileup=None,
0479 doElectron=None, doConversion=None, doBHadron=None,
0480 version="v1", dqmVersion="0001", scenario=None, overrideGlobalTag=None, appendGlobalTag=""):
0481 """Constructor.
0482
0483 Arguments:
0484 sample -- String for name of the sample
0485
0486 Keyword arguments
0487 append -- String for a variable name within the DWM file names, to be directly appended to sample name (e.g. "HS"; default None)
0488 midfix -- String for a variable name within the DQM file names, to be appended after underscore to "sample name+append" (e.g. "13", "UP15"; default None)
0489 putype -- String for pileup type (e.g. "25ns"/"50ns" for FullSim, "AVE20" for FastSim; default None)
0490 punum -- String for amount of pileup (default None)
0491 fastsim -- Bool indicating the FastSim status (default False)
0492 fastsimCorrespondingFullSimPileup -- String indicating what is the FullSim pileup sample corresponding this FastSim sample. Must be set if fastsim=True and putype!=None (default None)
0493 doElectron -- Bool specifying if electron-specific plots should be produced (default depends on sample)
0494 doConversion -- Bool specifying if conversion-specific plots should be produced (default depends on sample)
0495 doBHadron -- Bool specifying if B-hadron-specific plots should be produced (default depends on sample)
0496 version -- String for dataset/DQM file version (default "v1")
0497 scenario -- Geometry scenario for upgrade samples (default None)
0498 overrideGlobalTag -- GlobalTag obtained from release information (in the form of {"release": "actualRelease"}; default None)
0499 appendGlobalTag -- String to append to GlobalTag (intended for one-time hacks; default "")
0500 """
0501 self._sample = sample
0502 self._append = append
0503 self._midfix = midfix
0504 self._putype = putype
0505 self._punum = punum
0506 self._fastsim = fastsim
0507 self._fastsimCorrespondingFullsimPileup = fastsimCorrespondingFullsimPileup
0508 self._version = version
0509 self._dqmVersion = dqmVersion
0510 self._scenario = scenario
0511 self._overrideGlobalTag = overrideGlobalTag
0512 self._appendGlobalTag = appendGlobalTag
0513
0514 if doElectron is not None:
0515 self._doElectron = doElectron
0516 else:
0517 self._doElectron = (sample in _doElectronSamples)
0518 if doConversion is not None:
0519 self._doConversion = doConversion
0520 else:
0521 self._doConversion = (sample in _doConversionSamples)
0522 if doBHadron is not None:
0523 self._doBHadron = doBHadron
0524 else:
0525 self._doBHadron = (sample in _doBHadronSamples)
0526
0527 if self._fastsim and self.hasPileup() and self._fastsimCorrespondingFullsimPileup is None:
0528 self._fastsimCorrespondingFullsimPileup = self._putype
0529
0530 def digest(self):
0531 """Return a tuple uniquely identifying the sample, to be used e.g. as a key to dict"""
0532 return (self.name(), self.pileupNumber(), self.pileupType(), self.scenario(), self.fastsim())
0533
0534 def sample(self):
0535 """Get the sample name"""
0536 return self._sample
0537
0538 def name(self):
0539 """Get the sample name"""
0540 return self._sample
0541
0542 def label(self):
0543 return self._sample
0544
0545 def hasPileup(self):
0546 """Return True if sample has pileup (for HTML generation)"""
0547 return self._putype is not None
0548
0549 def pileupEnabled(self):
0550 """Return True if pileup plots are enabled (for plot generation)"""
0551 return self.hasPileup()
0552
0553 def pileup(self):
0554 """Return "PU"/"noPU" corresponding the pileup status"""
0555 if self.hasPileup():
0556 return "PU"
0557 else:
0558 return "noPU"
0559
0560 def pileupType(self, release=None):
0561 """Return the pileup type"""
0562 if isinstance(self._putype, dict):
0563 return self._putype.get(release, self._putype["default"])
0564 else:
0565 return self._putype
0566
0567 def pileupNumber(self):
0568 return self._punum
0569
0570 def doElectron(self):
0571 return self._doElectron
0572
0573 def doConversion(self):
0574 return self._doConversion
0575
0576 def doBHadron(self):
0577 return self._doBHadron
0578
0579 def version(self, release=None):
0580 if isinstance(self._version, dict):
0581 return self._version.get(release, self._version["default"])
0582 else:
0583 return self._version
0584
0585 def hasScenario(self):
0586 return self._scenario is not None
0587
0588 def scenario(self):
0589 return self._scenario
0590
0591 def hasOverrideGlobalTag(self):
0592 return self._overrideGlobalTag is not None
0593
0594 def overrideGlobalTag(self):
0595 return self._overrideGlobalTag
0596
0597 def fastsim(self):
0598 """Return True for FastSim sample"""
0599 return self._fastsim
0600
0601 def fullsim(self):
0602 """Return True for FullSim sample"""
0603 return not self._fastsim
0604
0605 def fastsimCorrespondingFullsimPileup(self):
0606 return self._fastsimCorrespondingFullsimPileup
0607
0608 def dirname(self, newRepository, newRelease, newSelection):
0609 """Return the output directory name
0610
0611 Arguments:
0612 newRepository -- String for base directory for output files
0613 newRelease -- String for CMSSW release
0614 newSelection -- String for histogram selection
0615 """
0616 pileup = ""
0617 if self.hasPileup() and not self._fastsim:
0618 pileup = "_"+self._putype
0619 return "{newRepository}/{newRelease}/{newSelection}{pileup}/{sample}".format(
0620 newRepository=newRepository, newRelease=newRelease, newSelection=newSelection,
0621 pileup=pileup, sample=sample)
0622
0623 def filename(self, newRelease):
0624 """Return the DQM file name
0625
0626 Arguments:
0627 newRelease -- String for CMSSW release
0628 """
0629 pileup = ""
0630 fastsim = ""
0631 midfix = ""
0632 sample = self._sample
0633 if self._append is not None:
0634 midfix += self._append
0635 if self._midfix is not None:
0636 midfix += "_"+self._midfix
0637 if self.hasPileup():
0638 if self._fastsim:
0639
0640
0641
0642
0643
0644 pileup = "PU"+self.pileupType(newRelease)+"_"
0645 else:
0646 pileup = "PU"+self.pileupType(newRelease)+"_"
0647 if self._fastsim:
0648 fastsim = "_FastSim"
0649
0650 globalTag = _getGlobalTag(self, newRelease)
0651
0652 fname = 'DQM_V{dqmVersion}_R000000001__{sample}{midfix}__{newrelease}-{pileup}{globaltag}{appendGlobalTag}{fastsim}-{version}__DQMIO.root'.format(
0653 sample=sample, midfix=midfix, newrelease=_stripRelease(newRelease),
0654 pileup=pileup, globaltag=globalTag, appendGlobalTag=self._appendGlobalTag, fastsim=fastsim,
0655 version=self.version(newRelease), dqmVersion=self._dqmVersion
0656 )
0657
0658 return fname
0659
0660 def datasetpattern(self, newRelease):
0661 """Return the dataset pattern
0662
0663 Arguments:
0664 newRelease -- String for CMSSW release
0665 """
0666 pileup = ""
0667 fastsim = ""
0668 digi = ""
0669 if self.hasPileup():
0670 pileup = "-PU_"
0671 if self._fastsim:
0672 fastsim = "_FastSim-"
0673 digi = "DIGI-"
0674 else:
0675 fastsim = "*"
0676 globalTag = _getGlobalTag(self, newRelease)
0677 return "{sample}/{newrelease}-{pileup}{globaltag}{fastsim}{version}/GEN-SIM-{digi}RECO".format(
0678 sample=self._sample, newrelease=newRelease,
0679 pileup=pileup, globaltag=globalTag, fastsim=fastsim, digi=digi,
0680 version=self.version(newRelease)
0681 )
0682
0683 class Validation:
0684 """Base class for Tracking/Vertex validation."""
0685 def __init__(self, fullsimSamples, fastsimSamples, refRelease, refRepository, newRelease, newRepository, newFileModifier=None, selectionName=""):
0686 """Constructor.
0687
0688 Arguments:
0689 fullsimSamples -- List of Sample objects for FullSim samples (may be empty)
0690 fastsimSamples -- List of Sample objects for FastSim samples (may be empty)
0691 refRelease -- String for reference CMSSW release (can be None for no reference release)
0692 newRepository -- String for directory whete to put new files
0693 newRelease -- CMSSW release to be validated
0694 refRepository -- String for directory where reference root files are
0695 newFileModifier -- If given, a function to modify the names of the new files (function takes a string and returns a string)
0696 selectionName -- If given, use this string as the selection name (appended to GlobalTag for directory names)
0697 """
0698 try:
0699 self._newRelease = os.environ["CMSSW_VERSION"]
0700 except KeyError:
0701 print('Error: CMSSW environment variables are not available.', file=sys.stderr)
0702 print(' Please run cmsenv', file=sys.stderr)
0703 sys.exit()
0704
0705 self._fullsimSamples = fullsimSamples
0706 self._fastsimSamples = fastsimSamples
0707 self._refRelease = refRelease
0708 self._refRepository = refRepository
0709 self._newRelease = newRelease
0710 self._newBaseDir = os.path.join(newRepository, self._newRelease)
0711 self._newFileModifier = newFileModifier
0712 self._selectionName = selectionName
0713
0714 def _getDirectoryName(self, *args, **kwargs):
0715 return None
0716
0717 def _getSelectionName(self, *args, **kwargs):
0718 return self._selectionName
0719
0720 def download(self):
0721 """Download DQM files. Requires grid certificate and asks your password for it."""
0722 filenames = [s.filename(self._newRelease) for s in self._fullsimSamples+self._fastsimSamples]
0723 if self._newFileModifier is not None:
0724 filenames = list(map(self._newFileModifier, filenames))
0725 filenames = [f for f in filenames if not os.path.exists(f)]
0726 if len(filenames) == 0:
0727 print("All files already downloaded")
0728 return
0729
0730 relvalUrl = _getRelValUrl(self._newRelease)
0731 urls = [relvalUrl+f for f in filenames]
0732 certfile = os.path.join(os.environ["HOME"], ".globus", "usercert.pem")
0733 if not os.path.exists(certfile):
0734 print("Certificate file {certfile} does not exist, unable to download RelVal files from {url}".format(certfile=certfile, url=relvalUrl))
0735 sys.exit(1)
0736 keyfile = os.path.join(os.environ["HOME"], ".globus", "userkey.pem")
0737 if not os.path.exists(certfile):
0738 print("Private key file {keyfile} does not exist, unable to download RelVal files from {url}".format(keyfile=keyfile, url=relvalUrl))
0739 sys.exit(1)
0740
0741
0742 cmd = ["curl", "--cert-type", "PEM", "--cert", certfile, "--key", keyfile, "-k"]
0743 for u in urls:
0744 cmd.extend(["-O", u])
0745 print("Downloading %d files from RelVal URL %s:" % (len(filenames), relvalUrl))
0746 print(" "+"\n ".join(filenames))
0747 print("Please provide your private key pass phrase when curl asks it")
0748 ret = subprocess.call(cmd)
0749 if ret != 0:
0750 print("Downloading failed with exit code %d" % ret)
0751 sys.exit(1)
0752
0753
0754 allFine = True
0755 for f in filenames:
0756 p = subprocess.Popen(["file", f], stdout=subprocess.PIPE)
0757 stdout = p.communicate()[0]
0758 if p.returncode != 0:
0759 print("file command failed with exit code %d" % p.returncode)
0760 sys.exit(1)
0761 if not "ROOT" in stdout:
0762 print("File {f} is not ROOT, please check the correct version, GlobalTag etc. from {url}".format(f=f, url=relvalUrl))
0763 allFine = False
0764 if os.path.exists(f):
0765 os.remove(f)
0766 if not allFine:
0767 sys.exit(1)
0768
0769 def createHtmlReport(self):
0770 return html.HtmlReport(self._newRelease, self._newBaseDir)
0771
0772 def doPlots(self, plotter, plotterDrawArgs={}, limitSubFoldersOnlyTo=None, htmlReport=html.HtmlReportDummy(), doFastVsFull=True, doPhase2PU=False):
0773 """Create validation plots.
0774
0775 Arguments:
0776 plotter -- plotting.Plotter object that does the plotting
0777
0778 Keyword arguments:
0779 plotterDrawArgs -- Dictionary for additional arguments to Plotter.draw() (default: {})
0780 limitSubFoldersOnlyTo -- If not None, should be a dictionary from string to an object. The string is the name of a PlotFolder, and the object is PlotFolder-type specific to limit the subfolders to be processed. In general case the object is a list of strings, but e.g. for track iteration plots it is a function taking the algo and quality as parameters.
0781 htmlReport -- Object returned by createHtmlReport(), in case HTML report generation is desired
0782 doFastVsFull -- Do FastSim vs. FullSim comparison? (default: True)
0783 doPhase2PU -- Do Phase2 PU 200 vs. 140 comparison (default: False)
0784 """
0785 self._plotter = plotter
0786 self._plotterDrawArgs = plotterDrawArgs
0787
0788
0789 for sample in self._fullsimSamples+self._fastsimSamples:
0790
0791 harvestedFile = sample.filename(self._newRelease)
0792 if not os.path.exists(harvestedFile):
0793 print("Harvested file %s does not exist!" % harvestedFile)
0794 sys.exit(1)
0795
0796 plotterInstance = plotter.readDirs(harvestedFile)
0797 htmlReport.beginSample(sample)
0798 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
0799 if not _processPlotsForSample(plotterFolder, sample):
0800 continue
0801 plotFiles = self._doPlots(sample, harvestedFile, plotterFolder, dqmSubFolder, htmlReport)
0802 htmlReport.addPlots(plotterFolder, dqmSubFolder, plotFiles)
0803
0804
0805 if doFastVsFull:
0806 self._doFastsimFastVsFullPlots(limitSubFoldersOnlyTo, htmlReport)
0807
0808
0809 if doPhase2PU:
0810 self._doPhase2PileupPlots(limitSubFoldersOnlyTo, htmlReport)
0811
0812 def _doFastsimFastVsFullPlots(self, limitSubFoldersOnlyTo, htmlReport):
0813 for fast in self._fastsimSamples:
0814 correspondingFull = None
0815 for full in self._fullsimSamples:
0816 if fast.name() != full.name():
0817 continue
0818 if fast.pileupEnabled():
0819 if not full.pileupEnabled():
0820 continue
0821 if fast.fastsimCorrespondingFullsimPileup() != full.pileupType():
0822 continue
0823 else:
0824 if full.pileupEnabled():
0825 continue
0826
0827 if correspondingFull is None:
0828 correspondingFull = full
0829 else:
0830 raise Exception("Got multiple compatible FullSim samples for FastSim sample %s %s" % (fast.name(), fast.pileup()))
0831 if correspondingFull is None:
0832 print("WARNING: Did not find compatible FullSim sample for FastSim sample %s %s, omitting FastSim vs. FullSim comparison" % (fast.name(), fast.pileup()))
0833 continue
0834
0835
0836 harvestedFile = fast.filename(self._newRelease)
0837 plotterInstance = self._plotter.readDirs(harvestedFile)
0838 htmlReport.beginSample(fast, fastVsFull=True)
0839 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
0840 if not _processPlotsForSample(plotterFolder, fast):
0841 continue
0842 plotFiles = self._doPlotsFastFull(fast, correspondingFull, plotterFolder, dqmSubFolder, htmlReport)
0843 htmlReport.addPlots(plotterFolder, dqmSubFolder, plotFiles)
0844
0845 def _doPhase2PileupPlots(self, limitSubFoldersOnlyTo, htmlReport):
0846 def _stripScenario(name):
0847 puindex = name.find("PU")
0848 if puindex < 0:
0849 return name
0850 return name[:puindex]
0851
0852 pu140samples = {}
0853 for sample in self._fullsimSamples:
0854 if sample.pileupNumber() == 140:
0855 key = (sample.name(), _stripScenario(sample.scenario()))
0856 if key in pu140samples:
0857 raise Exception("Duplicate entry for sample %s in scenario %s" % (sample.name(), sample.scenar()))
0858 pu140samples[key] = sample
0859
0860 for sample in self._fullsimSamples:
0861 if sample.pileupNumber() != 200:
0862 continue
0863 key = (sample.name(), _stripScenario(sample.scenario()))
0864 if not key in pu140samples:
0865 continue
0866
0867 sample_pu140 = pu140samples[key]
0868
0869
0870 harvestedFile = sample.filename(self._newRelease)
0871 plotterInstance = self._plotter.readDirs(harvestedFile)
0872 htmlReport.beginSample(sample, pileupComparison="vs. PU140")
0873 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
0874 if not _processPlotsForSample(plotterFolder, sample):
0875 continue
0876 plotFiles = self._doPlotsPileup(sample_pu140, sample, plotterFolder, dqmSubFolder, htmlReport)
0877 htmlReport.addPlots(plotterFolder, dqmSubFolder, plotFiles)
0878
0879
0880 def _getRefFileAndSelection(self, sample, plotterFolder, dqmSubFolder, selectionNameBase, valname):
0881 if self._refRelease is None:
0882 return (None, "")
0883
0884 refGlobalTag = _getGlobalTag(sample, self._refRelease)
0885 def _createRefSelection(selectionName):
0886 sel = refGlobalTag+selectionNameBase+selectionName
0887 if sample.pileupEnabled():
0888 refPu = sample.pileupType(self._refRelease)
0889 if refPu != "":
0890 sel += "_"+refPu
0891 return sel
0892 refSelection = _createRefSelection(plotterFolder.getSelectionName(dqmSubFolder))
0893
0894
0895 refValFile = None
0896 triedRefValFiles = []
0897 tmp = [self._refRepository, self._refRelease]
0898 if sample.fastsim():
0899 tmp.extend(["fastsim", self._refRelease])
0900 for selName in plotterFolder.getSelectionNameIterator(dqmSubFolder):
0901 refSel = _createRefSelection(selName)
0902 refdir = os.path.join(*(tmp+[refSel, sample.name()]))
0903
0904
0905 refValFilePath = os.path.join(refdir, valname)
0906 if os.path.exists(refValFilePath):
0907 refSelection = refSel
0908 refValFile = ROOT.TFile.Open(refValFilePath)
0909 break
0910 else:
0911 triedRefValFiles.append(refValFilePath)
0912 if refValFile is None:
0913 if len(triedRefValFiles) == 1:
0914 if plotting.verbose:
0915 print("Reference file %s not found" % triedRefValFiles[0])
0916 else:
0917 if plotting.verbose:
0918 print("None of the possible reference files %s not found" % ",".join(triedRefValFiles))
0919
0920 return (refValFile, refSelection)
0921
0922 def _doPlots(self, sample, harvestedFile, plotterFolder, dqmSubFolder, htmlReport):
0923 """Do the real plotting work for a given sample and DQM subfolder"""
0924
0925 newGlobalTag = _getGlobalTag(sample, self._newRelease)
0926
0927
0928 selectionNameBase = "_"+sample.pileup()
0929 newSelection = newGlobalTag+selectionNameBase+plotterFolder.getSelectionName(dqmSubFolder)
0930 if sample.pileupEnabled():
0931 newPu = sample.pileupType(self._newRelease)
0932 if newPu != "":
0933 newSelection += "_"+newPu
0934
0935 valname = "val.{sample}.root".format(sample=sample.name())
0936
0937
0938 (refValFile, refSelection) = self._getRefFileAndSelection(sample, plotterFolder, dqmSubFolder, selectionNameBase, valname)
0939
0940
0941 tmp = []
0942 if sample.fastsim():
0943 tmp.extend(["fastsim", self._newRelease])
0944 tmp.extend([newSelection, sample.name()])
0945 newsubdir = os.path.join(*tmp)
0946 newdir = os.path.join(self._newBaseDir, newsubdir)
0947 if not os.path.exists(newdir):
0948 os.makedirs(newdir)
0949 valnameFullPath = os.path.join(newdir, valname)
0950
0951
0952
0953 newValFile = _copySubDir(harvestedFile, valnameFullPath, plotterFolder.getPossibleDQMFolders(), dqmSubFolder.subfolder if dqmSubFolder is not None else None)
0954 fileList = []
0955
0956
0957 if plotting.verbose:
0958 print("Comparing ref and new {sim} {sample} {translatedFolder}".format(
0959 sim="FullSim" if not sample.fastsim() else "FastSim",
0960 sample=sample.name(), translatedFolder=str(dqmSubFolder.translated) if dqmSubFolder is not None else ""))
0961 rootFiles = [refValFile, newValFile]
0962 legendLabels = [
0963 "%s, %s %s" % (sample.name(), _stripRelease(self._refRelease), refSelection) if self._refRelease is not None else "dummy",
0964 "%s, %s %s" % (sample.name(), _stripRelease(self._newRelease), newSelection)
0965 ]
0966 plotterFolder.create(rootFiles, legendLabels, dqmSubFolder, isPileupSample=sample.pileupEnabled())
0967 fileList.extend(plotterFolder.draw(directory=newdir, **self._plotterDrawArgs))
0968
0969 if len(fileList) > 0:
0970 fileList.append(valnameFullPath)
0971
0972
0973 for tableCreator in plotterFolder.getTableCreators():
0974 htmlReport.addTable(tableCreator.create(rootFiles, legendLabels, dqmSubFolder))
0975
0976 newValFile.Close()
0977 if refValFile is not None:
0978 refValFile.Close()
0979
0980 if len(fileList) == 0:
0981 print("No object found in %s" % plotterFolder.getName())
0982 return []
0983
0984 dups = _findDuplicates(fileList)
0985 if len(dups) > 0:
0986 print("Plotter produced multiple files with names", ", ".join(dups))
0987 print("Typically this is a naming problem in the plotter configuration")
0988 sys.exit(1)
0989
0990
0991 print("Created plots and %s in %s" % (valname, newdir))
0992 return list(map(lambda n: n.replace(newdir, newsubdir), fileList))
0993
0994 def _doPlotsFastFull(self, fastSample, fullSample, plotterFolder, dqmSubFolder, htmlReport):
0995 """Do the real plotting work for FastSim vs. FullSim for a given algorithm, quality flag, and sample."""
0996
0997 fastGlobalTag = _getGlobalTag(fastSample, self._newRelease)
0998 fullGlobalTag = _getGlobalTag(fullSample, self._newRelease)
0999
1000
1001 tmp = plotterFolder.getSelectionName(dqmSubFolder)
1002 fastSelection = fastGlobalTag+"_"+fastSample.pileup()+tmp
1003 fullSelection = fullGlobalTag+"_"+fullSample.pileup()+tmp
1004 if fullSample.pileupEnabled():
1005 fullSelection += "_"+fullSample.pileupType(self._newRelease)
1006 fastSelection += "_"+fastSample.pileupType(self._newRelease)
1007
1008
1009 fastdir = os.path.join(self._newBaseDir, "fastsim", self._newRelease, fastSelection, fastSample.name())
1010 fulldir = os.path.join(self._newBaseDir, fullSelection, fullSample.name())
1011 newsubdir = os.path.join("fastfull", self._newRelease, fastSelection, fastSample.name())
1012 newdir = os.path.join(self._newBaseDir, newsubdir)
1013 if not os.path.exists(newdir):
1014 os.makedirs(newdir)
1015
1016
1017 valname = "val.{sample}.root".format(sample=fastSample.name())
1018 fastValFilePath = os.path.join(fastdir, valname)
1019 if not os.path.exists(fastValFilePath) and plotting.verbose:
1020 print("FastSim file %s not found" % fastValFilePath)
1021 fullValFilePath = os.path.join(fulldir, valname)
1022 if not os.path.exists(fullValFilePath) and plotting.verbose:
1023 print("FullSim file %s not found" % fullValFilePath)
1024
1025 fastValFile = ROOT.TFile.Open(fastValFilePath)
1026 fullValFile = ROOT.TFile.Open(fullValFilePath)
1027
1028
1029 if plotting.verbose:
1030 print("Comparing FullSim and FastSim {sample} {translatedFolder}".format(
1031 sample=fastSample.name(), translatedFolder=str(dqmSubFolder.translated) if dqmSubFolder is not None else ""))
1032 rootFiles = [fullValFile, fastValFile]
1033 legendLabels = [
1034 "FullSim %s, %s %s" % (fullSample.name(), _stripRelease(self._newRelease), fullSelection),
1035 "FastSim %s, %s %s" % (fastSample.name(), _stripRelease(self._newRelease), fastSelection),
1036 ]
1037 plotterFolder.create(rootFiles, legendLabels, dqmSubFolder, isPileupSample=fastSample.pileupEnabled(), requireAllHistograms=True)
1038 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1039
1040
1041 for tableCreator in plotterFolder.getTableCreators():
1042 htmlReport.addTable(tableCreator.create(rootFiles, legendLabels, dqmSubFolder))
1043
1044 fullValFile.Close()
1045 fastValFile.Close()
1046
1047 if len(fileList) == 0:
1048 print("No object found in %s" % plotterFolder.getName())
1049 return []
1050
1051 dups = _findDuplicates(fileList)
1052 if len(dups) > 0:
1053 print("Plotter produced multiple files with names", ", ".join(dups))
1054 print("Typically this is a naming problem in the plotter configuration")
1055 sys.exit(1)
1056
1057
1058 print("Created plots in %s" % (newdir))
1059 return list(map(lambda n: n.replace(newdir, newsubdir), fileList))
1060
1061 def _doPlotsPileup(self, pu140Sample, pu200Sample, plotterFolder, dqmSubFolder, htmlReport):
1062 """Do the real plotting work for two pileup scenarios for a given algorithm, quality flag, and sample."""
1063
1064 pu140GlobalTag = _getGlobalTag(pu140Sample, self._newRelease)
1065 pu200GlobalTag = _getGlobalTag(pu200Sample, self._newRelease)
1066
1067
1068 tmp = plotterFolder.getSelectionName(dqmSubFolder)
1069 pu140Selection = pu140GlobalTag+"_"+pu140Sample.pileup()+tmp+"_"+pu140Sample.pileupType(self._newRelease)
1070 pu200Selection = pu200GlobalTag+"_"+pu200Sample.pileup()+tmp+"_"+pu200Sample.pileupType(self._newRelease)
1071
1072
1073 pu140dir = os.path.join(self._newBaseDir, pu140Selection, pu140Sample.name())
1074 pu200dir = os.path.join(self._newBaseDir, pu200Selection, pu200Sample.name())
1075 newsubdir = os.path.join("pileup", self._newRelease, pu200Selection, pu200Sample.name())
1076 newdir = os.path.join(self._newBaseDir, newsubdir)
1077 if not os.path.exists(newdir):
1078 os.makedirs(newdir)
1079
1080
1081 valname = "val.{sample}.root".format(sample=pu140Sample.name())
1082 pu140ValFilePath = os.path.join(pu140dir, valname)
1083 if not os.path.exists(pu140ValFilePath):
1084 if plotting.verbose:
1085 print("PU140 file %s not found" % pu140ValFilePath)
1086 return []
1087 pu200ValFilePath = os.path.join(pu200dir, valname)
1088 if not os.path.exists(pu200ValFilePath):
1089 if plotting.verbose:
1090 print("PU200 file %s not found" % pu200ValFilePath)
1091 return []
1092
1093 pu140ValFile = ROOT.TFile.Open(pu140ValFilePath)
1094 pu200ValFile = ROOT.TFile.Open(pu200ValFilePath)
1095
1096
1097 if plotting.verbose:
1098 print("Comparing PU140 and PU200 {sample} {translatedFolder}".format(
1099 sample=pu200Sample.name(), translatedFolder=str(dqmSubFolder.translated) if dqmSubFolder is not None else ""))
1100 rootFiles = [pu140ValFile, pu200ValFile]
1101 legendLabels = [
1102 "%s, %s %s" % (pu140Sample.name(), _stripRelease(self._newRelease), pu140Selection),
1103 "%s, %s %s" % (pu200Sample.name(), _stripRelease(self._newRelease), pu200Selection),
1104 ]
1105 plotterFolder.create(rootFiles, legendLabels, dqmSubFolder, isPileupSample=pu140Sample.pileupEnabled(), requireAllHistograms=True)
1106 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1107
1108
1109 for tableCreator in plotterFolder.getTableCreators():
1110 htmlReport.addTable(tableCreator.create(rootFiles, legendLabels, dqmSubFolder))
1111
1112 pu200ValFile.Close()
1113 pu140ValFile.Close()
1114
1115 if len(fileList) == 0:
1116 print("No object found in %s" % plotterFolder.getName())
1117 return []
1118
1119 dups = _findDuplicates(fileList)
1120 if len(dups) > 0:
1121 print("Plotter produced multiple files with names", ", ".join(dups))
1122 print("Typically this is a naming problem in the plotter configuration")
1123 sys.exit(1)
1124
1125
1126 print("Created plots in %s" % (newdir))
1127 return list(map(lambda n: n.replace(newdir, newsubdir), fileList))
1128
1129
1130 def _copySubDir(oldfile, newfile, basenames, dirname):
1131 """Copy a subdirectory from oldfile to newfile.
1132
1133 Arguments:
1134 oldfile -- String for source TFile
1135 newfile -- String for destination TFile
1136 basenames -- List of strings for base directories, first existing one is picked
1137 dirname -- String for directory name under the base directory
1138 """
1139 oldf = ROOT.TFile.Open(oldfile)
1140
1141 dirold = None
1142 for basename in basenames:
1143 dirold = oldf.GetDirectory(basename)
1144 if dirold:
1145 break
1146 if not dirold:
1147 raise Exception("Did not find any of %s directories from file %s" % (",".join(basenames), oldfile))
1148 if dirname:
1149 d = dirold.Get(dirname)
1150 if not d:
1151 raise Exception("Did not find directory %s under %s" % (dirname, dirold.GetPath()))
1152 dirold = d
1153
1154 newf = ROOT.TFile.Open(newfile, "RECREATE")
1155 dirnew = newf
1156 for d in basenames[0].split("/"):
1157 dirnew = dirnew.mkdir(d)
1158 if dirname:
1159 dirnew = dirnew.mkdir(dirname)
1160 _copyDir(dirold, dirnew)
1161
1162 oldf.Close()
1163 return newf
1164
1165 def _copyDir(src, dst):
1166 """Copy non-TTree objects from src TDirectory to dst TDirectory."""
1167 keys = src.GetListOfKeys()
1168 for key in keys:
1169 classname = key.GetClassName()
1170 cl = ROOT.TClass.GetClass(classname)
1171 if not cl:
1172 continue
1173 if not (cl.InheritsFrom("TTree") and cl.InheritsFrom("TDirectory")):
1174 dst.cd()
1175 obj = key.ReadObj()
1176 obj.Write()
1177 obj.Delete()
1178
1179 def _findDuplicates(lst):
1180 found = set()
1181 found2 = set()
1182 for x in lst:
1183 if x in found:
1184 found2.add(x)
1185 else:
1186 found.add(x)
1187 return list(found2)
1188
1189 def _doPlotsForPlotter(self, plotter, sample, limitSubFoldersOnlyTo=None):
1190 plotterInstance = plotter.readDirs(*self._openFiles)
1191 manager = multiprocessing.Manager()
1192 return_dict = manager.dict()
1193 proc = []
1194 iProc = 0
1195
1196 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
1197 if sample is not None and not _processPlotsForSample(plotterFolder, sample):
1198 continue
1199 newsubdir = self._subdirprefix+plotterFolder.getSelectionName(dqmSubFolder)
1200 newdir = os.path.join(self._newdir, newsubdir)
1201 if not os.path.exists(newdir):
1202 os.makedirs(newdir, exist_ok=True)
1203
1204 plotterFolder.create(self._openFiles, self._labels, dqmSubFolder)
1205 p = multiprocessing.Process(target=self._doPlots, args=(plotterFolder, dqmSubFolder, newsubdir, newdir, iProc, return_dict))
1206 proc.append((plotterFolder, dqmSubFolder, p))
1207 p.start()
1208 iProc += 1
1209
1210 for i in range(iProc):
1211 proc[i][2].join()
1212 if len(return_dict[i]) > 0:
1213 self._htmlReport.addPlots(proc[i][0], proc[i][1], return_dict[i])
1214
1215 class SimpleSample:
1216 def __init__(self, label, name, fileLegends, pileup=True, customPileupLabel=""):
1217 self._label = label
1218 self._name = name
1219 self._fileLegends = fileLegends
1220 self._pileup = pileup
1221 self._customPileupLabel = customPileupLabel
1222
1223 def digest(self):
1224
1225 return self._label
1226
1227 def label(self):
1228 return self._label
1229
1230 def name(self):
1231 return self._name
1232
1233 def files(self):
1234 return [t[0] for t in self._fileLegends]
1235
1236 def legendLabels(self):
1237 return [t[1] for t in self._fileLegends]
1238
1239 def fastsim(self):
1240
1241 return False
1242
1243 def pileupEnabled(self):
1244 return self._pileup
1245
1246 def customPileupLabel(self):
1247 return self._customPileupLabel
1248
1249 def doElectron(self):
1250 return True
1251
1252 def doConversion(self):
1253 return True
1254
1255 def doBHadron(self):
1256 return True
1257
1258 class SimpleValidation:
1259 def __init__(self, samples, newdir):
1260 self._samples = samples
1261 self._newdir = newdir
1262 if not os.path.exists(newdir):
1263 os.makedirs(newdir)
1264
1265 self._htmlReport = html.HtmlReportDummy()
1266
1267 def createHtmlReport(self, validationName=""):
1268 if hasattr(self._htmlReport, "write"):
1269 raise Exception("HTML report object already created. There is probably some logic error in the calling code.")
1270 self._htmlReport = html.HtmlReport(validationName, self._newdir)
1271 return self._htmlReport
1272
1273 def doPlots(self, plotters, plotterDrawArgs={}, **kwargs):
1274 self._plotterDrawArgs = plotterDrawArgs
1275
1276 for sample in self._samples:
1277 self._subdirprefix = sample.label()
1278 self._labels = sample.legendLabels()
1279 self._htmlReport.beginSample(sample)
1280
1281 self._openFiles = []
1282 for f in sample.files():
1283 if os.path.exists(f):
1284 self._openFiles.append(ROOT.TFile.Open(f))
1285 else:
1286 print("File %s not found (from sample %s), ignoring it" % (f, sample.name()))
1287 self._openFiles.append(None)
1288
1289 for plotter in plotters:
1290 _doPlotsForPlotter(self, plotter, sample, **kwargs)
1291
1292 for tf in self._openFiles:
1293 if tf is not None:
1294 tf.Close()
1295 self._openFiles = []
1296
1297 def _doPlots(self, plotterFolder, dqmSubFolder, newsubdir, newdir, iProc, return_dict):
1298 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1299
1300 if len(fileList) == 0:
1301 print("No object found in %s" % plotterFolder.getName())
1302
1303 for tableCreator in plotterFolder.getTableCreators():
1304 self._htmlReport.addTable(tableCreator.create(self._openFiles, self._labels, dqmSubFolder))
1305
1306 dups = _findDuplicates(fileList)
1307 if len(dups) > 0:
1308 print("Plotter produced multiple files with names", ", ".join(dups))
1309 print("Typically this is a naming problem in the plotter configuration")
1310 sys.exit(1)
1311
1312 if self._plotterDrawArgs.get("separate", False):
1313 if not os.path.exists("%s/res"%newdir):
1314 os.makedirs("%s/res"%newdir)
1315 downloadables = ["index.php", "res/jquery-ui.js", "res/jquery.js", "res/style.css", "res/style.js", "res/theme.css"]
1316 for d in downloadables:
1317 if not os.path.exists("%s/%s" % (newdir,d)):
1318 urllib.request.urlretrieve("https://raw.githubusercontent.com/musella/php-plots/master/%s"%d, "%s/%s"%(newdir,d))
1319
1320 print("Created plots in %s" % newdir)
1321 return_dict[iProc] = list(map(lambda n: n.replace(newdir, newsubdir), fileList))
1322
1323 class SeparateValidation:
1324
1325
1326 def __init__(self, samples, newdir):
1327 self._samples = samples
1328 self._newdir = newdir
1329 if not os.path.exists(newdir):
1330 os.makedirs(newdir)
1331
1332 self._htmlReport = html.HtmlReportDummy()
1333
1334 def createHtmlReport(self, validationName=""):
1335 if hasattr(self._htmlReport, "write"):
1336 raise Exception("HTML report object already created. There is probably some logic error in the calling code.")
1337 self._htmlReport = html.HtmlReport(validationName, self._newdir)
1338 return self._htmlReport
1339
1340 def doPlots(self, plotters, plotterDrawArgs={}, **kwargs):
1341 self._plotterDrawArgs = plotterDrawArgs
1342
1343 for sample in self._samples:
1344 self._subdirprefix = sample.label()
1345 self._labels = sample.legendLabels()
1346 self._htmlReport.beginSample(sample)
1347
1348 self._openFiles = []
1349 for f in sample.files():
1350 if os.path.exists(f):
1351 self._openFiles.append(ROOT.TFile.Open(f))
1352 else:
1353 print("File %s not found (from sample %s), ignoring it" % (f, sample.name()))
1354 self._openFiles.append(None)
1355
1356 for plotter in plotters:
1357 _doPlotsForPlotter(self, plotter, sample, **kwargs)
1358
1359 for tf in self._openFiles:
1360 if tf is not None:
1361 tf.Close()
1362 self._openFiles = []
1363
1364 def _doPlots(self, plotterFolder, dqmSubFolder, newsubdir, newdir, iProc, return_dict):
1365 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1366
1367
1368 if len(fileList) == 0:
1369 print("No object found in %s" % plotterFolder.getName())
1370
1371
1372 dups = _findDuplicates(fileList)
1373 if len(dups) > 0:
1374 print("Plotter produced multiple files with names", ", ".join(dups))
1375 print("Typically this is a naming problem in the plotter configuration")
1376 sys.exit(1)
1377
1378 linkList = []
1379 for f in fileList:
1380 if f[:f.rfind("/")] not in linkList :
1381 if str(f[:f.rfind("/")]) != str(newdir) :
1382 linkList.append(f[:f.rfind("/")])
1383
1384 for tableCreator in plotterFolder.getTableCreators():
1385 self._htmlReport.addTable(tableCreator.create(self._openFiles, self._labels, dqmSubFolder))
1386
1387 for link in linkList :
1388 if not os.path.exists("%s/res"%link):
1389 os.makedirs("%s/res"%link)
1390 downloadables = ["index.php", "res/jquery-ui.js", "res/jquery.js", "res/style.css", "res/style.js", "res/theme.css"]
1391 for d in downloadables:
1392 if not os.path.exists("%s/%s" % (link,d)):
1393 urllib.request.urlretrieve("https://raw.githubusercontent.com/rovere/php-plots/master/%s"%d, "%s/%s"%(link,d))
1394
1395 print("Created separated plots in %s" % newdir)
1396 return_dict[iProc] = list(map(lambda n: n.replace(newdir, newsubdir), linkList))
