File indexing completed on 2023-11-02 03:11:35
0001 from __future__ import print_function
0002 from __future__ import absolute_import
0003 import os
0004 import re
0005 import sys
0006 import time
0007 import shutil
0008 import subprocess
0009 import urllib.request
0010 import multiprocessing
0011
0012 import ROOT
0013 ROOT.gROOT.SetBatch(True)
0014 ROOT.PyConfig.IgnoreCommandLineOptions = True
0015
0016 from . import plotting
0017 from . import html
0018
0019
0020 _globalTags = {
0021 "CMSSW_6_2_0": {"default": "PRE_ST62_V8"},
0022 "CMSSW_6_2_0_SLHC15": {"UPG2019withGEM": "DES19_62_V8", "UPG2023SHNoTaper": "DES23_62_V1"},
0023 "CMSSW_6_2_0_SLHC17": {"UPG2019withGEM": "DES19_62_V8", "UPG2023SHNoTaper": "DES23_62_V1"},
0024 "CMSSW_6_2_0_SLHC20": {"UPG2019withGEM": "DES19_62_V8", "UPG2023SHNoTaper": "DES23_62_V1_UPG2023SHNoTaper"},
0025 "CMSSW_6_2_0_SLHC22": {"UPG2023SHNoTaper": "PH2_1K_FB_V6_UPG23SHNoTaper",
0026
0027 "2023GReco": "PH2_1K_FB_V6_UPG23SHNoTaper", "2023GRecoPU35": "", "2023GRecoPU140": "", "2023GRecoPU200": "",
0028 "2023tilted": "PH2_1K_FB_V6_UPG23SHNoTaper", "2023tiltedPU35": "", "2023tiltedPU140": "", "2023tiltedPU200": ""},
0029 "CMSSW_6_2_0_SLHC26": {"LHCCRefPU140": "DES23_62_V1_LHCCRefPU140", "LHCCRefPU200": "DES23_62_V1_LHCCRefPU200",
0030
0031 "2023GReco": "", "2023GRecoPU35": "", "2023GRecoPU140": "DES23_62_V1_LHCCRefPU140", "2023GRecoPU200": "DES23_62_V1_LHCCRefPU200",
0032 "2023tilted": "", "2023tiltedPU35": "", "2023tiltedPU140": "DES23_62_V1_LHCCRefPU140", "2023tiltedPU200": "DES23_62_V1_LHCCRefPU200"},
0033 "CMSSW_6_2_0_SLHC27_phase1": {"default": "DES17_62_V8_UPG17"},
0034 "CMSSW_7_0_0": {"default": "POSTLS170_V3", "fullsim_50ns": "POSTLS170_V4"},
0035 "CMSSW_7_0_0_AlcaCSA14": {"default": "POSTLS170_V5_AlcaCSA14", "fullsim_50ns": "POSTLS170_V6_AlcaCSA14"},
0036 "CMSSW_7_0_7_pmx": {"default": "PLS170_V7AN1", "fullsim_50ns": "PLS170_V6AN1"},
0037 "CMSSW_7_0_9_patch3": {"default": "PLS170_V7AN2", "fullsim_50ns": "PLS170_V6AN2"},
0038 "CMSSW_7_0_9_patch3_Premix": {"default": "PLS170_V7AN2", "fullsim_50ns": "PLS170_V6AN2"},
0039 "CMSSW_7_1_0": {"default": "POSTLS171_V15", "fullsim_50ns": "POSTLS171_V16"},
0040 "CMSSW_7_1_9": {"default": "POSTLS171_V17", "fullsim_50ns": "POSTLS171_V18"},
0041 "CMSSW_7_1_9_patch2": {"default": "POSTLS171_V17", "fullsim_50ns": "POSTLS171_V18"},
0042 "CMSSW_7_1_10_patch2": {"default": "MCRUN2_71_V1", "fullsim_50ns": "MCRUN2_71_V0"},
0043 "CMSSW_7_2_0_pre5": {"default": "POSTLS172_V3", "fullsim_50ns": "POSTLS172_V4"},
0044 "CMSSW_7_2_0_pre7": {"default": "PRE_LS172_V11", "fullsim_50ns": "PRE_LS172_V12"},
0045 "CMSSW_7_2_0_pre8": {"default": "PRE_LS172_V15", "fullsim_50ns": "PRE_LS172_V16"},
0046 "CMSSW_7_2_0": {"default": "PRE_LS172_V15", "fullsim_50ns": "PRE_LS172_V16"},
0047 "CMSSW_7_2_0_PHYS14": {"default": "PHYS14_25_V1_Phys14"},
0048 "CMSSW_7_2_2_patch1": {"default": "MCRUN2_72_V1", "fullsim_50ns": "MCRUN2_72_V0"},
0049 "CMSSW_7_2_2_patch1_Fall14DR": {"default": "MCRUN2_72_V3_71XGENSIM"},
0050
0051 "CMSSW_7_3_0_pre1": {"default": "PRE_LS172_V15", "fullsim_50ns": "PRE_LS172_V16"},
0052
0053 "CMSSW_7_3_0_pre2": {"default": "MCRUN2_73_V1", "fullsim_50ns": "MCRUN2_73_V0"},
0054 "CMSSW_7_3_0_pre3": {"default": "MCRUN2_73_V5", "fullsim_50ns": "MCRUN2_73_V4"},
0055 "CMSSW_7_3_0": {"default": "MCRUN2_73_V7", "fullsim_50ns": "MCRUN2_73_V6"},
0056 "CMSSW_7_3_0_71XGENSIM": {"default": "MCRUN2_73_V7_71XGENSIM"},
0057 "CMSSW_7_3_0_71XGENSIM_FIXGT": {"default": "MCRUN2_73_V9_71XGENSIM_FIXGT"},
0058 "CMSSW_7_3_1_patch1": {"default": "MCRUN2_73_V9", "fastsim": "MCRUN2_73_V7"},
0059 "CMSSW_7_3_1_patch1_GenSim_7113": {"default": "MCRUN2_73_V9_GenSim_7113"},
0060 "CMSSW_7_3_3": {"default": "MCRUN2_73_V11", "fullsim_50ns": "MCRUN2_73_V10", "fastsim": "MCRUN2_73_V13"},
0061 "CMSSW_7_4_0_pre1": {"default": "MCRUN2_73_V5", "fullsim_50ns": "MCRUN2_73_V4"},
0062 "CMSSW_7_4_0_pre2": {"default": "MCRUN2_73_V7", "fullsim_50ns": "MCRUN2_73_V6"},
0063 "CMSSW_7_4_0_pre2_73XGENSIM": {"default": "MCRUN2_73_V7_73XGENSIM_Pythia6", "fullsim_50ns": "MCRUN2_73_V6_73XGENSIM_Pythia6"},
0064 "CMSSW_7_4_0_pre5": {"default": "MCRUN2_73_V7", "fullsim_50ns": "MCRUN2_73_V6"},
0065 "CMSSW_7_4_0_pre5_BS": {"default": "MCRUN2_73_V9_postLS1beamspot", "fullsim_50ns": "MCRUN2_73_V8_postLS1beamspot"},
0066 "CMSSW_7_4_0_pre6": {"default": "MCRUN2_74_V1", "fullsim_50ns": "MCRUN2_74_V0"},
0067 "CMSSW_7_4_0_pre8": {"default": "MCRUN2_74_V7", "fullsim_25ns": "MCRUN2_74_V5_AsympMinGT", "fullsim_50ns": "MCRUN2_74_V4_StartupMinGT"},
0068 "CMSSW_7_4_0_pre8_minimal": {"default": "MCRUN2_74_V5_MinGT", "fullsim_25ns": "MCRUN2_74_V5_AsympMinGT", "fullsim_50ns": "MCRUN2_74_V4_StartupMinGT"},
0069 "CMSSW_7_4_0_pre8_25ns_asymptotic": {"default": "MCRUN2_74_V7"},
0070 "CMSSW_7_4_0_pre8_50ns_startup": {"default": "MCRUN2_74_V6"},
0071 "CMSSW_7_4_0_pre8_50ns_asympref": {"default": "MCRUN2_74_V5A_AsympMinGT"},
0072 "CMSSW_7_4_0_pre8_50ns_asymptotic": {"default": "MCRUN2_74_V7A_AsympGT"},
0073 "CMSSW_7_4_0_pre8_ROOT6": {"default": "MCRUN2_74_V7"},
0074 "CMSSW_7_4_0_pre8_pmx": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0075 "CMSSW_7_4_0_pre8_pmx_v2": {"default": "MCRUN2_74_V7_gs_pre7", "fullsim_50ns": "MCRUN2_74_V6_gs_pre7"},
0076 "CMSSW_7_4_0_pre8_pmx_v3": {"default": "MCRUN2_74_V7_bis", "fullsim_50ns": "MCRUN2_74_V6_bis"},
0077 "CMSSW_7_4_0_pre9": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0078 "CMSSW_7_4_0_pre9_ROOT6": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0079 "CMSSW_7_4_0_pre9_extended": {"default": "MCRUN2_74_V7_extended"},
0080 "CMSSW_7_4_0": {"default": "MCRUN2_74_V7_gensim_740pre7", "fullsim_50ns": "MCRUN2_74_V6_gensim_740pre7", "fastsim": "MCRUN2_74_V7"},
0081 "CMSSW_7_4_0_71XGENSIM": {"default": "MCRUN2_74_V7_GENSIM_7_1_15", "fullsim_50ns": "MCRUN2_74_V6_GENSIM_7_1_15"},
0082 "CMSSW_7_4_0_71XGENSIM_PU": {"default": "MCRUN2_74_V7_gs7115_puProd", "fullsim_50ns": "MCRUN2_74_V6_gs7115_puProd"},
0083 "CMSSW_7_4_0_71XGENSIM_PXworst": {"default": "MCRUN2_74_V7C_pxWorst_gs7115", "fullsim_50ns": "MCRUN2_74_V6A_pxWorst_gs7115"},
0084 "CMSSW_7_4_0_71XGENSIM_PXbest": {"default": "MCRUN2_74_V7D_pxBest_gs7115", "fullsim_50ns": "MCRUN2_74_V6B_pxBest_gs7115"},
0085 "CMSSW_7_4_0_pmx": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0086 "CMSSW_7_4_1": {"default": "MCRUN2_74_V9_gensim_740pre7", "fullsim_50ns": "MCRUN2_74_V8_gensim_740pre7", "fastsim": "MCRUN2_74_V9"},
0087 "CMSSW_7_4_1_71XGENSIM": {"default": "MCRUN2_74_V9_gensim71X", "fullsim_50ns": "MCRUN2_74_V8_gensim71X"},
0088 "CMSSW_7_4_1_extended": {"default": "MCRUN2_74_V9_extended"},
0089 "CMSSW_7_4_3": {"default": "MCRUN2_74_V9", "fullsim_50ns": "MCRUN2_74_V8", "fastsim": "MCRUN2_74_V9", "fastsim_25ns": "MCRUN2_74_V9_fixMem"},
0090 "CMSSW_7_4_3_extended": {"default": "MCRUN2_74_V9_ext","fastsim": "MCRUN2_74_V9_fixMem"},
0091 "CMSSW_7_4_3_pmx": {"default": "MCRUN2_74_V9_ext", "fullsim_50ns": "MCRUN2_74_V8", "fastsim": "MCRUN2_74_V9_fixMem"},
0092 "CMSSW_7_4_3_patch1_unsch": {"default": "MCRUN2_74_V9_unsch", "fullsim_50ns": "MCRUN2_74_V8_unsch"},
0093 "CMSSW_7_4_4": {"default": "MCRUN2_74_V9_38Tbis", "fullsim_50ns": "MCRUN2_74_V8_38Tbis"},
0094 "CMSSW_7_4_4_0T": {"default": "MCRUN2_740TV1_0Tv2", "fullsim_50ns": "MCRUN2_740TV0_0TV2", "fullsim_25ns": "MCRUN2_740TV1_0TV2"},
0095 "CMSSW_7_4_6_patch6": {"default": "MCRUN2_74_V9_scheduled", "fullsim_50ns": "MCRUN2_74_V8_scheduled"},
0096 "CMSSW_7_4_6_patch6_unsch": {"default": "MCRUN2_74_V9", "fullsim_50ns": "MCRUN2_74_V8"},
0097 "CMSSW_7_4_6_patch6_noCCC": {"default": "MCRUN2_74_V9_unsch_noCCC", "fullsim_50ns": "MCRUN2_74_V8_unsch_noCCC"},
0098 "CMSSW_7_4_6_patch6_noCCC_v3": {"default": "MCRUN2_74_V9_unsch_noCCC_v3", "fullsim_50ns": "MCRUN2_74_V8_unsch_noCCC_v3"},
0099 "CMSSW_7_4_6_patch6_BS": {"default": "74X_mcRun2_asymptotic_realisticBS_v0_2015Jul24", "fullsim_50ns": "74X_mcRun2_startup_realisticBS_v0_2015Jul24PU", "fullsim_25ns": "74X_mcRun2_asymptotic_realisticBS_v0_2015Jul24PU"},
0100 "CMSSW_7_4_8_patch1_MT": {"default": "MCRUN2_74_V11_mulTrh", "fullsim_50ns": "MCRUN2_74_V10_mulTrh",},
0101 "CMSSW_7_4_12": {"default": "74X_mcRun2_asymptotic_v2", "fullsim_25ns": "74X_mcRun2_asymptotic_v2_v2", "fullsim_50ns": "74X_mcRun2_startup_v2_v2"},
0102 "CMSSW_7_5_0_pre1": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0103 "CMSSW_7_5_0_pre2": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0104 "CMSSW_7_5_0_pre3": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0105 "CMSSW_7_5_0_pre4": {"default": "MCRUN2_75_V1", "fullsim_50ns": "MCRUN2_75_V0"},
0106 "CMSSW_7_5_0_pre5": {"default": "MCRUN2_75_V5", "fullsim_50ns": "MCRUN2_75_V4"},
0107 "CMSSW_7_5_0_pre6": {"default": "75X_mcRun2_asymptotic_v1", "fullsim_50ns": "75X_mcRun2_startup_v1"},
0108 "CMSSW_7_5_0": {"default": "75X_mcRun2_asymptotic_v1", "fullsim_50ns": "75X_mcRun2_startup_v1"},
0109 "CMSSW_7_5_0_71XGENSIM": {"default": "75X_mcRun2_asymptotic_v1_gs7115", "fullsim_50ns": "75X_mcRun2_startup_v1_gs7115"},
0110 "CMSSW_7_5_1": {"default": "75X_mcRun2_asymptotic_v3", "fullsim_50ns": "75X_mcRun2_startup_v3"},
0111 "CMSSW_7_5_1_71XGENSIM": {"default": "75X_mcRun2_asymptotic_v3_gs7118", "fullsim_50ns": "75X_mcRun2_startup_v3_gs7118"},
0112 "CMSSW_7_5_2": {"default": "75X_mcRun2_asymptotic_v5", "fullsim_50ns": "75X_mcRun2_startup_v4"},
0113 "CMSSW_7_6_0_pre1": {"default": "75X_mcRun2_asymptotic_v1", "fullsim_50ns": "75X_mcRun2_startup_v1"},
0114 "CMSSW_7_6_0_pre2": {"default": "75X_mcRun2_asymptotic_v2", "fullsim_50ns": "75X_mcRun2_startup_v2"},
0115 "CMSSW_7_6_0_pre3": {"default": "75X_mcRun2_asymptotic_v2", "fullsim_50ns": "75X_mcRun2_startup_v2"},
0116 "CMSSW_7_6_0_pre4": {"default": "76X_mcRun2_asymptotic_v1", "fullsim_50ns": "76X_mcRun2_startup_v1"},
0117 "CMSSW_7_6_0_pre5": {"default": "76X_mcRun2_asymptotic_v1", "fullsim_50ns": "76X_mcRun2_startup_v1"},
0118 "CMSSW_7_6_0_pre6": {"default": "76X_mcRun2_asymptotic_v4", "fullsim_50ns": "76X_mcRun2_startup_v4"},
0119 "CMSSW_7_6_0_pre7": {"default": "76X_mcRun2_asymptotic_v5", "fullsim_50ns": "76X_mcRun2_startup_v5", "fastsim": "76X_mcRun2_asymptotic_v5_resub"},
0120 "CMSSW_7_6_0": {"default": "76X_mcRun2_asymptotic_v11", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0121 "CMSSW_7_6_0_71XGENSIM": {"default": "76X_mcRun2_asymptotic_v11_gs7120p2rlBS", "fullsim_50ns": "76X_mcRun2_startup_v11_gs7120p2rlBS"},
0122 "CMSSW_7_6_2": {"default": "76X_mcRun2_asymptotic_v12", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0123 "CMSSW_7_6_3_patch2_0T": {"default": "76X_mcRun2_0T_v1_0Tv1GT"},
0124 "CMSSW_8_0_0_pre1": {"default": "76X_mcRun2_asymptotic_v11", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0125 "CMSSW_8_0_0_pre2": {"default": "76X_mcRun2_asymptotic_v12", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0126 "CMSSW_8_0_0_pre2_phase1": {"default": "76X_upgrade2017_design_v7"},
0127 "CMSSW_8_0_0_pre2_phase1_rereco": {"default": "76X_upgrade2017_design_v7_rereco"},
0128 "CMSSW_8_0_0_pre3_phase1": {"default": "76X_upgrade2017_design_v8"},
0129 "CMSSW_8_0_0_pre3_phase1_pythia8": {"default": "76X_upgrade2017_design_v8_pythia8"},
0130 "CMSSW_8_0_0_pre4": {"default": "76X_mcRun2_asymptotic_v13", "fullsim_50ns": "76X_mcRun2_startup_v12"},
0131 "CMSSW_8_0_0_pre4_phase1": {"default": "76X_upgrade2017_design_v8_UPG17"},
0132 "CMSSW_8_0_0_pre4_phase1_13TeV": {"default": "76X_upgrade2017_design_v8_UPG17"},
0133 "CMSSW_8_0_0_pre4_ecal15fb": {"default": "80X_mcRun2_asymptotic_2016EcalTune_15fb_v0_ecal15fbm1"},
0134 "CMSSW_8_0_0_pre4_ecal30fb": {"default": "80X_mcRun2_asymptotic_2016EcalTune_30fb_v0_ecal30fbm1"},
0135 "CMSSW_8_0_0_pre5": {"default": "80X_mcRun2_asymptotic_v1", "fullsim_50ns": "80X_mcRun2_startup_v1"},
0136 "CMSSW_8_0_0_pre5_phase1": {"default": "80X_upgrade2017_design_v1_UPG17"},
0137 "CMSSW_8_0_0_pre6": {"default": "80X_mcRun2_asymptotic_v4"},
0138 "CMSSW_8_0_0_pre6_phase1": {"default": "80X_upgrade2017_design_v3_UPG17"},
0139 "CMSSW_8_0_0_pre6_MT": {"default": "80X_mcRun2_asymptotic_v4_multiCoreResub"},
0140 "CMSSW_8_0_0": {"default": "80X_mcRun2_asymptotic_v4"},
0141 "CMSSW_8_0_0_patch1_phase1": {"default": "80X_upgrade2017_design_v4_UPG17"},
0142 "CMSSW_8_0_0_patch1_phase1_rereco": {"default": "80X_upgrade2017_design_v4_UPG17_rereco"},
0143 "CMSSW_8_0_0_patch2": {"default": "80X_mcRun2_asymptotic_v5_refGT"},
0144 "CMSSW_8_0_0_patch2_pixDynIneff": {"default": "80X_mcRun2_asymptotic_v5_2016PixDynIneff_targetGT"},
0145
0146
0147 "CMSSW_8_0_0_patch2": {"default": "80X_mcRun2_asymptotic_v5_refGT_resub"},
0148 "CMSSW_8_0_0_patch2_pixDynIneff": {"default": "80X_mcRun2_asymptotic_v5_2016PixDynIneff_targetGT_resub"},
0149 "CMSSW_8_0_1": {"default": "80X_mcRun2_asymptotic_v6"},
0150 "CMSSW_8_0_1_71XGENSIM": {"default": "80X_mcRun2_asymptotic_v6_gs7120p2"},
0151 "CMSSW_8_0_1_gcc530": {"default": "80X_mcRun2_asymptotic_v6_gcc530"},
0152 "CMSSW_8_0_3_71XGENSIM": {"default": "80X_mcRun2_asymptotic_2016_v3_gs7120p2NewGTv3"},
0153 "CMSSW_8_0_3_71XGENSIM_hcal": {"default": "80X_mcRun2_asymptotic_2016_v3_gs71xNewGtHcalCust"},
0154 "CMSSW_8_0_3_71XGENSIM_tec": {"default": "80X_mcRun2_asymptotic_SiStripBad_TEC_CL62_for2016_v1_mc_gs7120p2TrkCoolLoop"},
0155 "CMSSW_8_0_5": {"default": "80X_mcRun2_asymptotic_v12_gs7120p2", "fastsim": "80X_mcRun2_asymptotic_v12"},
0156 "CMSSW_8_0_5_pmx": {"default": "80X_mcRun2_asymptotic_v12_gs7120p2_resub", "fastsim": "80X_mcRun2_asymptotic_v12"},
0157 "CMSSW_8_0_10": {"default": "80X_mcRun2_asymptotic_v14"},
0158 "CMSSW_8_0_10_patch1_BS": {"default": "80X_mcRun2_asymptotic_RealisticBS_25ns_13TeV2016_v1_mc_realisticBS2016"},
0159 "CMSSW_8_0_11": {"default": "80X_mcRun2_asymptotic_v14"},
0160 "CMSSW_8_0_15": {"default": "80X_mcRun2_asymptotic_v16_gs7120p2", "fastsim": "80X_mcRun2_asymptotic_v16"},
0161 "CMSSW_8_0_16": {"default": "80X_mcRun2_asymptotic_v16_gs7120p2", "fastsim": "80X_mcRun2_asymptotic_v16"},
0162 "CMSSW_8_0_16_Tranche4GT": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_gs7120p2_Tranch4GT",
0163 "fastsim": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_Tranch4GT", "RelValTTbar": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_Tr4GT_resub"}, "fastsim_25ns": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_Tranch4GT"},
0164 "CMSSW_8_0_16_Tranche4GT_v2": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v2_Tr4GT_v2"},
0165 "CMSSW_8_0_16_Tranche4GT_pmx": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_gs7120p2_Tranch4GT", "fastsim": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_resub"},
0166 "CMSSW_8_0_19_Tranche4GT_v2": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v2_Tr4GT_v2"},
0167 "CMSSW_8_0_20_Tranche4GT": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v4_Tr4GT_v4"},
0168 "CMSSW_8_0_21_Tranche4GT": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v6_Tr4GT_v6"},
0169 "CMSSW_8_1_0_pre1": {"default": "80X_mcRun2_asymptotic_v6"},
0170 "CMSSW_8_1_0_pre1_phase1": {"default": "80X_upgrade2017_design_v4_UPG17", "fullsim_25ns": "80X_upgrade2017_design_v4_UPG17PU35"},
0171 "CMSSW_8_1_0_pre2": {"default": "80X_mcRun2_asymptotic_v10_gs810pre2", "fastsim": "80X_mcRun2_asymptotic_v10"},
0172 "CMSSW_8_1_0_pre2_phase1": {"default": "80X_upgrade2017_design_v9_UPG17designGT", "fullsim_25ns": "80X_upgrade2017_design_v9_UPG17PU35designGT"},
0173 "CMSSW_8_1_0_pre2_phase1_realGT": {"default": "80X_upgrade2017_realistic_v1_UPG17realGT", "fullsim_25ns": "80X_upgrade2017_realistic_v1_UPG17PU35realGT"},
0174 "CMSSW_8_1_0_pre3": {"default": "80X_mcRun2_asymptotic_v12"},
0175 "CMSSW_8_1_0_pre3_phase1": {"default": "80X_upgrade2017_realistic_v3_UPG17", "fullsim_25ns": "80X_upgrade2017_realistic_v3_UPG17PU35"},
0176 "CMSSW_8_1_0_pre4": {"default": "80X_mcRun2_asymptotic_v13"},
0177 "CMSSW_8_1_0_pre4_phase1": {"default": "80X_upgrade2017_realistic_v4_UPG17", "fullsim_25ns": "80X_upgrade2017_realistic_v4_UPG17PU35"},
0178 "CMSSW_8_1_0_pre5": {"default": "80X_mcRun2_asymptotic_v13"},
0179 "CMSSW_8_1_0_pre5_phase1": {"default": "80X_upgrade2017_realistic_v4_resubUPG17", "fullsim_25ns": "80X_upgrade2017_realistic_v4_resubUPG17PU35"},
0180 "CMSSW_8_1_0_pre6": {"default": "80X_mcRun2_asymptotic_v14"},
0181 "CMSSW_8_1_0_pre6_phase1": {"default": "81X_upgrade2017_realistic_v0_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v0_UPG17PU35"},
0182 "CMSSW_8_1_0_pre7": {"default": "81X_mcRun2_asymptotic_v0"},
0183 "CMSSW_8_1_0_pre7_phase1": {"default": "81X_upgrade2017_realistic_v2_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v2_UPG17PU35"},
0184 "CMSSW_8_1_0_pre7_phase1_newGT": {"default": "81X_upgrade2017_realistic_v3_UPG17newGT", "fullsim_25ns": "81X_upgrade2017_realistic_v3_UPG17PU35newGTresub"},
0185 "CMSSW_8_1_0_pre7_phase2": {"2023GReco": "81X_mcRun2_asymptotic_v0_2023GReco", "2023GRecoPU35": "", "2023GRecoPU140": "81X_mcRun2_asymptotic_v0_2023GRecoPU140resubmit2", "2023GRecoPU200": "81X_mcRun2_asymptotic_v0_2023GRecoPU200resubmit2",
0186 "2023tilted": "81X_mcRun2_asymptotic_v0_2023tilted", "2023tiltedPU35": "", "2023tiltedPU140": "81X_mcRun2_asymptotic_v0_2023tiltedPU140resubmit2", "2023tiltedPU200": "81X_mcRun2_asymptotic_v0_2023tiltedPU200resubmit2"},
0187 "CMSSW_8_1_0_pre8": {"default": "81X_mcRun2_asymptotic_v1"},
0188 "CMSSW_8_1_0_pre8_phase1": {"default": "81X_upgrade2017_realistic_v3_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v3_UPG17PU35"},
0189 "CMSSW_8_1_0_pre8_phase1_newGT": {"default": "81X_upgrade2017_realistic_v4_UPG17newGT", "fullsim_25ns": "81X_upgrade2017_realistic_v4_UPG17PU35newGT"},
0190 "CMSSW_8_1_0_pre8_phase1_newGT2": {"default": "81X_upgrade2017_realistic_v5_UPG17newGTset2", "fullsim_25ns": "81X_upgrade2017_realistic_v5_UPG17PU35newGTset2"},
0191 "CMSSW_8_1_0_pre8_phase2": {"2023GReco": "81X_mcRun2_asymptotic_v1_resub2023GReco", "2023GRecoPU35": "81X_mcRun2_asymptotic_v1_resub2023GRecoPU35", "2023GRecoPU140": "81X_mcRun2_asymptotic_v1_resub2023GRecoPU140", "2023GRecoPU200": "81X_mcRun2_asymptotic_v1_resub2023GRecoPU200",
0192 "2023tilted": "81X_mcRun2_asymptotic_v1_2023tilted", "2023tiltedPU35": "81X_mcRun2_asymptotic_v1_2023tiltedPU", "2023tiltedPU140": "81X_mcRun2_asymptotic_v1_2023tiltedPU140", "2023tiltedPU200": "81X_mcRun2_asymptotic_v1_2023tiltedPU200"},
0193 "CMSSW_8_1_0_pre9": {"default": "81X_mcRun2_asymptotic_v2"},
0194 "CMSSW_8_1_0_pre9_Geant4102": {"default": "81X_mcRun2_asymptotic_v2"},
0195 "CMSSW_8_1_0_pre9_phase1": {"default": "81X_upgrade2017_realistic_v5_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v5_UPG17PU35"},
0196 "CMSSW_8_1_0_pre9_phase1_newGT": {"default": "81X_upgrade2017_realistic_v6_UPG17newGT", "fullsim_25ns": "81X_upgrade2017_realistic_v6_UPG17PU35newGT"},
0197 "CMSSW_8_1_0_pre10": {"default": "81X_mcRun2_asymptotic_v5_recycle", "fullsim_25ns": "81X_mcRun2_asymptotic_v5_resub", "fastsim": "81X_mcRun2_asymptotic_v5"},
0198 "CMSSW_8_1_0_pre10_pmx": {"default": "81X_mcRun2_asymptotic_v5"},
0199 "CMSSW_8_1_0_pre10_phase1": {"default": "81X_upgrade2017_realistic_v9_UPG17resub", "fullsim_25ns": "81X_upgrade2017_realistic_v9_UPG17PU35resub"},
0200 "CMSSW_8_1_0_pre11": {"default": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55", "fullsim_25ns": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55_resub", "fastsim_25ns": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55_resub2"},
0201 "CMSSW_8_1_0_pre11_pmx": {"default": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55"},
0202 "CMSSW_8_1_0_pre11_phase1": {"default": "81X_upgrade2017_realistic_v9_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v9_UPG17PU35"},
0203 "CMSSW_8_1_0_pre12": {"default": "81X_mcRun2_asymptotic_v8", "fullsim_25ns": "81X_mcRun2_asymptotic_v8_resub", "fastsim": "81X_mcRun2_asymptotic_v8_resub", "fastsim_25ns": "81X_mcRun2_asymptotic_v8"},
0204 "CMSSW_8_1_0_pre12_pmx": {"default": "81X_mcRun2_asymptotic_v8_resub", "fastsim_25ns": "81X_mcRun2_asymptotic_v8_rsub"},
0205 "CMSSW_8_1_0_pre12_phase1": {"default": "81X_upgrade2017_realistic_v13"},
0206 "CMSSW_8_1_0_pre12_phase1_newBPix": {"default": "81X_upgrade2017_realistic_newBPix_wAlign_v1_BpixGeom"},
0207 "CMSSW_8_1_0_pre12_phase1_newBPixFPix": {"default": "81X_upgrade2017_realistic_v13_BpixFpixGeom"},
0208 "CMSSW_8_1_0_pre12_phase1_newBPixFPixHCAL": {"default": "81X_upgrade2017_HCALdev_v2_BpixFpixHcalGeom"},
0209 "CMSSW_8_1_0_pre12_phase1_newHCAL": {"default": "81X_upgrade2017_HCALdev_v2_HcalGeom"},
0210 "CMSSW_8_1_0_pre12_phase1_newBPixHCAL": {"default": "81X_upgrade2017_HCALdev_v2_NewBPix_BpixHcalGeom"},
0211 "CMSSW_8_1_0_pre15": {"default": "81X_mcRun2_asymptotic_v11"},
0212 "CMSSW_8_1_0_pre15_HIP": {"default": "81X_mcRun2_asymptotic_v11_hip"},
0213 "CMSSW_8_1_0_pre15_PU": {"default": "81X_mcRun2_asymptotic_v11_M17"},
0214 "CMSSW_8_1_0_pre15_PU_HIP": {"default": "81X_mcRun2_asymptotic_v11_hipM17"},
0215 "CMSSW_8_1_0_pre15_phase1": {"default": "81X_upgrade2017_realistic_v17_BpixFpixHcalGeom",
0216 "Design": "81X_upgrade2017_design_IdealBS_v1_2017design", "Design_fullsim_25ns": "81X_upgrade2017_design_IdealBS_v1_design"},
0217 "CMSSW_8_1_0_pre16": {"default": "81X_mcRun2_asymptotic_v11"},
0218 "CMSSW_8_1_0_pre16_phase1": {"default": "81X_upgrade2017_realistic_v22", "Design": "81X_upgrade2017_design_IdealBS_v6"},
0219 "CMSSW_8_1_0": {"default": "81X_mcRun2_asymptotic_v12"},
0220 "CMSSW_8_1_0_phase1": {"default": "81X_upgrade2017_realistic_v26_HLT2017"},
0221 "CMSSW_9_0_0_pre1": {"default": "90X_mcRun2_asymptotic_v0"},
0222 "CMSSW_9_0_0_pre1_phase1": {"default": "90X_upgrade2017_realistic_v0",
0223 "Design": "90X_upgrade2017_design_IdealBS_v0", "Design_fullsim_25ns": "90X_upgrade2017_design_IdealBS_v0_resub"},
0224 "CMSSW_9_0_0_pre2": {"default": "90X_mcRun2_asymptotic_v0"},
0225 "CMSSW_9_0_0_pre2_ROOT6": {"default": "90X_mcRun2_asymptotic_v0"},
0226 "CMSSW_9_0_0_pre2_phase1": {"default": "90X_upgrade2017_realistic_v0",
0227 "Design": "90X_upgrade2017_design_IdealBS_v0", "Design_fullsim_25ns": "90X_upgrade2017_design_IdealBS_v0_resub"},
0228 "CMSSW_9_0_0_pre4": {"default": "90X_mcRun2_asymptotic_v1", "fullsim_25ns": "90X_mcRun2_asymptotic_v1_resub"},
0229 "CMSSW_9_0_0_pre4_phase1": {"default": "90X_upgrade2017_realistic_v6", "fullsim_25ns_PU50": "Nonexistent",
0230 "Design": "90X_upgrade2017_design_IdealBS_v6"},
0231 "CMSSW_9_0_0_pre4_GS": {"default": "90X_mcRun2_asymptotic_v1_GSval", "fullsim_25ns": "90X_mcRun2_asymptotic_v1_GSval"},
0232 "CMSSW_9_0_0_pre4_phase1_ecalsrb5": {"default": "90X_upgrade2017_realistic_v6_B5"},
0233 "CMSSW_9_0_0_pre4_phase1_ecalsrc1": {"default": "90X_upgrade2017_realistic_v6_C1"},
0234 "CMSSW_9_0_0_pre4_phase1_ecalsrd7": {"default": "90X_upgrade2017_realistic_v6_D7"},
0235 "CMSSW_9_0_0_pre5": {"default": "90X_mcRun2_asymptotic_v4"},
0236 "CMSSW_9_0_0_pre5_pmx": {"default": "90X_mcRun2_asymptotic_v4", "fastsim_25ns": "90X_mcRun2_asymptotic_v4_resub"},
0237 "CMSSW_9_0_0_pre5_phase1": {"default": "90X_upgrade2017_realistic_v15",
0238 "fullsim_25ns_PU35": "90X_upgrade2017_realistic_v15_resub", "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v15_PU50",
0239 "Design": "90X_upgrade2017_design_IdealBS_v15"},
0240 "CMSSW_9_0_0_pre6": {"default": "90X_mcRun2_asymptotic_v4"},
0241 "CMSSW_9_0_0_pre6_phase1": {"default": "90X_upgrade2017_realistic_v15",
0242 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v15_PU50",
0243 "Design": "90X_upgrade2017_design_IdealBS_v15"},
0244 "CMSSW_9_0_0": {"default": "90X_mcRun2_asymptotic_v5"},
0245 "CMSSW_9_0_0_phase1": {"default": "90X_upgrade2017_realistic_v20_resub",
0246 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50_resub",
0247 "Design": "90X_upgrade2017_design_IdealBS_v19_resub"},
0248 "CMSSW_9_0_0_gcc630": {"default": "90X_mcRun2_asymptotic_v5_gcc630"},
0249 "CMSSW_9_0_0_phase1_gcc630": {"default": "90X_upgrade2017_realistic_v20_gcc630"},
0250 "CMSSW_9_0_0_cc7": {"default": "90X_mcRun2_asymptotic_v5_cc7"},
0251 "CMSSW_9_0_0_phase1_cc7": {"default": "90X_upgrade2017_realistic_v20_cc7_rsb"},
0252 "CMSSW_9_0_2_phase1": {"default": "90X_upgrade2017_realistic_v20",
0253 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50",
0254 "Design": "90X_upgrade2017_design_IdealBS_v19"},
0255 "CMSSW_9_1_0_pre1": {"default": "90X_mcRun2_asymptotic_v5"},
0256 "CMSSW_9_1_0_pre1_phase1": {"default": "90X_upgrade2017_realistic_v20",
0257 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50",
0258 "Design": "90X_upgrade2017_design_IdealBS_v19_resub"},
0259 "CMSSW_9_1_0_pre2": {"default": "90X_mcRun2_asymptotic_v5"},
0260 "CMSSW_9_1_0_pre2_phase1": {"default": "90X_upgrade2017_realistic_v20",
0261 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50",
0262 "Design": "90X_upgrade2017_design_IdealBS_v19"},
0263 "CMSSW_9_1_0_pre3": {"default": "91X_mcRun2_asymptotic_v2"},
0264 "CMSSW_9_1_0_pre3_phase1": {"default": "91X_upgrade2017_realistic_v3",
0265 "fullsim_25ns_PU50": "91X_upgrade2017_realistic_v3_PU50_resub",
0266 "Design": "91X_upgrade2017_design_IdealBS_v3"},
0267 "CMSSW_9_2_0": {"default": "91X_mcRun2_asymptotic_v3"},
0268 "CMSSW_9_2_0_phase1": {"default": "91X_upgrade2017_realistic_v5",
0269 "fullsim_25ns_PU50": "91X_upgrade2017_realistic_v5_PU50",
0270 "Design": "91X_upgrade2017_design_IdealBS_v5"},
0271 "CMSSW_9_2_0_phase1_PXmap": {"default": "91X_upgrade2017_realistic_v5_pixel_ideal_PXgeom"},
0272 "CMSSW_9_2_1_phase1": {"default": "92X_upgrade2017_realistic_v1",
0273 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v1_PU50",
0274 "Design": "92X_upgrade2017_design_IdealBS_v1"},
0275 "CMSSW_9_2_2": {"default": "91X_mcRun2_asymptotic_v3"},
0276 "CMSSW_9_2_2_phase1": {"default": "92X_upgrade2017_realistic_v1",
0277 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v1_highPU_AVE50",
0278 "Design": "92X_upgrade2017_design_IdealBS_v1"},
0279 "CMSSW_9_2_4_run1": {"default": "91X_mcRun1_realistic_v2"},
0280 "CMSSW_9_2_4": {"default": "91X_mcRun2_asymptotic_v3"},
0281 "CMSSW_9_2_7_phase1": {"default": "92X_upgrade2017_realistic_v7"},
0282 "CMSSW_9_3_0_pre1": {"default": "92X_mcRun2_asymptotic_v2"},
0283 "CMSSW_9_3_0_pre1_phase1": {"default": "92X_upgrade2017_realistic_v7",
0284 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v7_highPU_AVE50",
0285 "Design": "92X_upgrade2017_design_IdealBS_v7"},
0286 "CMSSW_9_3_0_pre1_run1": {"default": "92X_mcRun1_realistic_v2"},
0287 "CMSSW_9_3_0_pre2": {"default": "92X_mcRun2_asymptotic_v2"},
0288 "CMSSW_9_3_0_pre2_phase1": {"default": "92X_upgrade2017_realistic_v7",
0289 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v7_highPU_AVE50_resub",
0290 "Design": "92X_upgrade2017_design_IdealBS_v7"},
0291 "CMSSW_9_3_0_pre3": {"default": "92X_mcRun2_asymptotic_v2"},
0292 "CMSSW_9_3_0_pre3_phase1": {"default": "92X_upgrade2017_realistic_v10_resub",
0293 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v10_highPU_AVE50_resub",
0294 "Design": "92X_upgrade2017_design_IdealBS_v10_resub"},
0295 "CMSSW_9_3_0_pre3_phase1_pmx": {"default": "92X_upgrade2017_realistic_v10_resub2"},
0296 "CMSSW_9_3_0_pre4": {"default": "93X_mcRun2_asymptotic_v0"},
0297 "CMSSW_9_3_0_pre4_phase1": {"default": "93X_mc2017_realistic_v1",
0298 "fullsim_25ns_PU50": "93X_mc2017_realistic_v1_highPU_AVE50",
0299 "Design": "93X_mc2017_design_IdealBS_v1"},
0300 "CMSSW_9_3_0_pre5": {"default": "93X_mcRun2_asymptotic_v0"},
0301 "CMSSW_9_3_0_pre5_phase1": {"default": "93X_mc2017_realistic_v2",
0302 "fullsim_25ns_PU50": "93X_mc2017_realistic_v2_highPU_AVE50",
0303 "Design": "93X_mc2017_design_IdealBS_v2"},
0304 "CMSSW_9_4_0_pre1": {"default": "93X_mcRun2_asymptotic_v1"},
0305 "CMSSW_9_4_0_pre1_phase1": {"default": "93X_mc2017_realistic_v3",
0306 "fullsim_25ns_PU50": "93X_mc2017_realistic_v3_highPU_AVE50",
0307 "Design": "93X_mc2017_design_IdealBS_v3"},
0308 "CMSSW_9_4_0_pre2": {"default": "93X_mcRun2_asymptotic_v2"},
0309 "CMSSW_9_4_0_pre2_phase1": {"default": "94X_mc2017_realistic_v1",
0310 "fullsim_25ns_PU50": "94X_mc2017_realistic_v1_highPU_AVE50",
0311 "Design": "94X_mc2017_design_IdealBS_v0"},
0312 "CMSSW_9_4_0_pre3": {"default": "94X_mcRun2_asymptotic_v0"},
0313 "CMSSW_9_4_0_pre3_phase1": {"default": "94X_mc2017_realistic_v4",
0314 "fullsim_25ns_PU50": "94X_mc2017_realistic_v4_highPU_AVE50",
0315 "Design": "94X_mc2017_design_IdealBS_v4"},
0316 "CMSSW_9_4_0": {"default": "94X_mcRun2_asymptotic_v0"},
0317 "CMSSW_9_4_0_phase1": {"default": "94X_mc2017_realistic_v10",
0318 "fullsim_25ns_PU50": "94X_mc2017_realistic_v10_highPU_AVE50",
0319 "Design": "94X_mc2017_design_IdealBS_v5"},
0320 "CMSSW_10_0_0_pre1": {"default": "94X_mcRun2_asymptotic_v0"},
0321 "CMSSW_10_0_0_pre1_phase1": {"default": "94X_mc2017_realistic_v10",
0322 "fullsim_25ns_PU50": "94X_mc2017_realistic_v10_highPU_AVE50",
0323 "Design": "94X_mc2017_design_IdealBS_v5"},
0324 "CMSSW_10_0_0_pre2": {"default": "100X_mcRun2_asymptotic_v2"},
0325 "CMSSW_10_0_0_pre2_2017": {"default": "100X_mc2017_realistic_v1",
0326 "fullsim_25ns": "100X_mc2017_realistic_v1_resub",
0327 "fullsim_25ns_PU50": "100X_mc2017_realistic_v1_highPU_AVE50",
0328 "Design": "100X_mc2017_design_IdealBS_v1",
0329 "Design_fullsim_25ns_PU50": "Does_not_exist"},
0330 "CMSSW_10_0_0_pre2_2017_pmx": {"default": "100X_mc2017_realistic_v1"},
0331 "CMSSW_10_0_0_pre2_2018": {"default": "100X_upgrade2018_realistic_v1",
0332 "fullsim_25ns": "100X_upgrade2018_realistic_v1_resub",
0333 "Design": "100X_upgrade2018_design_IdealBS_v1",
0334 "Design_fullsim_25ns": "100X_upgrade2018_design_IdealBS_v1_resub"},
0335 "CMSSW_10_0_0_pre3": {"default": "100X_mcRun2_asymptotic_v2"},
0336 "CMSSW_10_0_0_pre3_2017": {"default": "100X_mc2017_realistic_v1_mahiOFF",
0337 "fullsim_25ns_PU50": "100X_mc2017_realistic_v1_highPU_AVE50_mahiOFF",
0338 "Design": "100X_mc2017_design_IdealBS_v1_mahiOFF"},
0339 "CMSSW_10_0_0_pre3_2018": {"default": "100X_upgrade2018_realistic_v4_mahiOFF",
0340 "Design": "100X_upgrade2018_design_IdealBS_v3_mahiOFF"},
0341 "CMSSW_10_0_0_pre3_2018_pmx": {"default": "100X_upgrade2018_realistic_v4",
0342 "Design": "100X_upgrade2018_design_IdealBS_v3"},
0343 "CMSSW_10_0_0_pre3_2017_mahi": {"default": "100X_mc2017_realistic_v1_mahiON",
0344 "fullsim_25ns_PU50": "100X_mc2017_realistic_v1_highPU_AVE50_mahiON",
0345 "Design": "100X_mc2017_design_IdealBS_v1_mahiON"},
0346 "CMSSW_10_0_0_pre3_2018_mahi": {"default": "100X_upgrade2018_realistic_v4_mahiON",
0347 "Design": "100X_upgrade2018_design_IdealBS_v3_mahiON"},
0348 "CMSSW_10_0_0_pre3_GEANT4_2018_mahi": {"default": "100X_upgrade2018_realistic_v4_mahiON"},
0349 "CMSSW_10_0_0_pre3_G4VecGeom2_2018": {"default": "100X_upgrade2018_realistic_v4"},
0350 }
0351
0352 _releasePostfixes = ["_AlcaCSA14", "_PHYS14", "_TEST", "_v2", "_v3", "_pmx", "_Fall14DR", "_FIXGT", "_PU", "_PXbest", "_PXworst", "_hcal", "_tec", "_71XGENSIM", "_73XGENSIM", "_BS", "_GenSim_7113", "_extended",
0353 "_25ns_asymptotic", "_50ns_startup", "_50ns_asympref", "_50ns_asymptotic", "_minimal", "_0T", "_unsch", "_noCCC", "_MT", "_GS", "_rereco", "_pythia8", "_13TeV", "_realGT", "_newGT2", "_newGT", "_phase1", "_phase2", "_2017", "_2018", "_ecal15fb", "_ecal30fb", "_ecalsrb5", "_ecalsrc1", "_ecalsrd7", "_pixDynIneff", "_PXmap", "_gcc530", "_gcc630", "_cc7", "_Tranche4GT", "_newBPixFPixHCAL", "_newBPixFPix", "_newBPixHCAL", "_newBPix", "_newHCAL", "_HIP", "_run1", "_mahi"]
0354 def _stripRelease(release):
0355 for pf in _releasePostfixes:
0356 if pf in release:
0357 return _stripRelease(release.replace(pf, ""))
0358 return release
0359
0360
0361 def _getGlobalTag(sample, release):
0362 """Get a GlobalTag.
0363
0364 Arguments:
0365 sample -- Sample object
0366 release -- CMSSW release string
0367 """
0368 if not release in _globalTags:
0369 print("Release %s not found from globaltag map in validation.py" % release)
0370 sys.exit(1)
0371 gtmap = _globalTags[release]
0372 selectedGT = None
0373 if sample.hasOverrideGlobalTag():
0374 ogt = sample.overrideGlobalTag()
0375 if release in ogt:
0376 gtmap = _globalTags[ogt[release]]
0377 scenario = ""
0378 if sample.hasScenario():
0379 scenario = sample.scenario()
0380 sims = []
0381 if sample.fullsim():
0382 if sample.pileupEnabled():
0383 sim = "fullsim_"+sample.pileupType()
0384 sims.extend([
0385 sim+"_PU%d"%sample.pileupNumber(),
0386 sim
0387 ])
0388 elif sample.fastsim():
0389 sim = "fastsim"
0390 if sample.pileupEnabled():
0391 sim += "_"+sample.pileupType()
0392 sims.append(sim+"_PU%d"%sample.pileupNumber())
0393 sims.append(sim)
0394
0395 selectedGT = None
0396
0397 if scenario != "":
0398 for sim in sims:
0399 selectedGT = gtmap.get(scenario+"_"+sim, None)
0400 if selectedGT is not None:
0401 break
0402
0403 if selectedGT is None:
0404 selectedGT = gtmap.get(scenario, None)
0405
0406 if selectedGT is None:
0407 for sim in sims:
0408 selectedGT = gtmap.get(sim, None)
0409 if selectedGT is not None:
0410 break
0411
0412 if selectedGT is None:
0413 selectedGT = gtmap["default"]
0414
0415 if isinstance(selectedGT, dict):
0416 return selectedGT.get(sample.name(), selectedGT["default"])
0417 else:
0418 return selectedGT
0419
0420
0421 _relvalUrls = {
0422 "6_2_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_6_2_x/",
0423 "7_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_0_x/",
0424 "7_1_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_1_x/",
0425 "7_2_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_2_x/",
0426 "7_3_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_3_x/",
0427 "7_4_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_4_x/",
0428 "7_5_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_5_x/",
0429 "7_6_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_6_x/",
0430 "8_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_8_0_x/",
0431 "8_1_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_8_1_x/",
0432 "9_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_0_x/",
0433 "9_1_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_1_x/",
0434 "9_2_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_2_x/",
0435 "9_3_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_3_x/",
0436 "9_4_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_4_x/",
0437 "10_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_10_0_x/",
0438 }
0439
0440 _doElectronSamples = [
0441 "RelValTTbar",
0442 "RelValSingleElectronPt35",
0443 "RelValSingleElectronPt10",
0444 ]
0445 _doConversionSamples = [
0446 "RelValTTbar",
0447 "RelValH125GGgluonfusion",
0448 ]
0449 _doBHadronSamples = [
0450 "RelValTTbar"
0451 ]
0452
0453 def _getRelValUrl(release):
0454 """Get RelVal download URL for a given release."""
0455 version_re = re.compile("CMSSW_(?P<X>\d+)_(?P<Y>\d+)")
0456 m = version_re.search(release)
0457 if not m:
0458 raise Exception("Regex %s does not match to release version %s" % (version_re.pattern, release))
0459 version = "%s_%s_X" % (m.group("X"), m.group("Y"))
0460 if not version in _relvalUrls:
0461 print("No RelVal URL for version %s, please update _relvalUrls" % version)
0462 sys.exit(1)
0463 return _relvalUrls[version]
0464
0465 def _processPlotsForSample(plotterFolder, sample):
0466 if plotterFolder.onlyForPileup() and not sample.pileupEnabled():
0467 return False
0468 if plotterFolder.onlyForElectron() and not sample.doElectron():
0469 return False
0470 if plotterFolder.onlyForConversion() and not sample.doConversion():
0471 return False
0472 if plotterFolder.onlyForBHadron() and not sample.doBHadron():
0473 return False
0474 return True
0475
0476 class Sample:
0477 """Represents a RelVal sample."""
0478 def __init__(self, sample, append=None, midfix=None, putype=None, punum=0,
0479 fastsim=False, fastsimCorrespondingFullsimPileup=None,
0480 doElectron=None, doConversion=None, doBHadron=None,
0481 version="v1", dqmVersion="0001", scenario=None, overrideGlobalTag=None, appendGlobalTag=""):
0482 """Constructor.
0483
0484 Arguments:
0485 sample -- String for name of the sample
0486
0487 Keyword arguments
0488 append -- String for a variable name within the DWM file names, to be directly appended to sample name (e.g. "HS"; default None)
0489 midfix -- String for a variable name within the DQM file names, to be appended after underscore to "sample name+append" (e.g. "13", "UP15"; default None)
0490 putype -- String for pileup type (e.g. "25ns"/"50ns" for FullSim, "AVE20" for FastSim; default None)
0491 punum -- String for amount of pileup (default None)
0492 fastsim -- Bool indicating the FastSim status (default False)
0493 fastsimCorrespondingFullSimPileup -- String indicating what is the FullSim pileup sample corresponding this FastSim sample. Must be set if fastsim=True and putype!=None (default None)
0494 doElectron -- Bool specifying if electron-specific plots should be produced (default depends on sample)
0495 doConversion -- Bool specifying if conversion-specific plots should be produced (default depends on sample)
0496 doBHadron -- Bool specifying if B-hadron-specific plots should be produced (default depends on sample)
0497 version -- String for dataset/DQM file version (default "v1")
0498 scenario -- Geometry scenario for upgrade samples (default None)
0499 overrideGlobalTag -- GlobalTag obtained from release information (in the form of {"release": "actualRelease"}; default None)
0500 appendGlobalTag -- String to append to GlobalTag (intended for one-time hacks; default "")
0501 """
0502 self._sample = sample
0503 self._append = append
0504 self._midfix = midfix
0505 self._putype = putype
0506 self._punum = punum
0507 self._fastsim = fastsim
0508 self._fastsimCorrespondingFullsimPileup = fastsimCorrespondingFullsimPileup
0509 self._version = version
0510 self._dqmVersion = dqmVersion
0511 self._scenario = scenario
0512 self._overrideGlobalTag = overrideGlobalTag
0513 self._appendGlobalTag = appendGlobalTag
0514
0515 if doElectron is not None:
0516 self._doElectron = doElectron
0517 else:
0518 self._doElectron = (sample in _doElectronSamples)
0519 if doConversion is not None:
0520 self._doConversion = doConversion
0521 else:
0522 self._doConversion = (sample in _doConversionSamples)
0523 if doBHadron is not None:
0524 self._doBHadron = doBHadron
0525 else:
0526 self._doBHadron = (sample in _doBHadronSamples)
0527
0528 if self._fastsim and self.hasPileup() and self._fastsimCorrespondingFullsimPileup is None:
0529 self._fastsimCorrespondingFullsimPileup = self._putype
0530
0531 def digest(self):
0532 """Return a tuple uniquely identifying the sample, to be used e.g. as a key to dict"""
0533 return (self.name(), self.pileupNumber(), self.pileupType(), self.scenario(), self.fastsim())
0534
0535 def sample(self):
0536 """Get the sample name"""
0537 return self._sample
0538
0539 def name(self):
0540 """Get the sample name"""
0541 return self._sample
0542
0543 def label(self):
0544 return self._sample
0545
0546 def hasPileup(self):
0547 """Return True if sample has pileup (for HTML generation)"""
0548 return self._putype is not None
0549
0550 def pileupEnabled(self):
0551 """Return True if pileup plots are enabled (for plot generation)"""
0552 return self.hasPileup()
0553
0554 def pileup(self):
0555 """Return "PU"/"noPU" corresponding the pileup status"""
0556 if self.hasPileup():
0557 return "PU"
0558 else:
0559 return "noPU"
0560
0561 def pileupType(self, release=None):
0562 """Return the pileup type"""
0563 if isinstance(self._putype, dict):
0564 return self._putype.get(release, self._putype["default"])
0565 else:
0566 return self._putype
0567
0568 def pileupNumber(self):
0569 return self._punum
0570
0571 def doElectron(self):
0572 return self._doElectron
0573
0574 def doConversion(self):
0575 return self._doConversion
0576
0577 def doBHadron(self):
0578 return self._doBHadron
0579
0580 def version(self, release=None):
0581 if isinstance(self._version, dict):
0582 return self._version.get(release, self._version["default"])
0583 else:
0584 return self._version
0585
0586 def hasScenario(self):
0587 return self._scenario is not None
0588
0589 def scenario(self):
0590 return self._scenario
0591
0592 def hasOverrideGlobalTag(self):
0593 return self._overrideGlobalTag is not None
0594
0595 def overrideGlobalTag(self):
0596 return self._overrideGlobalTag
0597
0598 def fastsim(self):
0599 """Return True for FastSim sample"""
0600 return self._fastsim
0601
0602 def fullsim(self):
0603 """Return True for FullSim sample"""
0604 return not self._fastsim
0605
0606 def fastsimCorrespondingFullsimPileup(self):
0607 return self._fastsimCorrespondingFullsimPileup
0608
0609 def dirname(self, newRepository, newRelease, newSelection):
0610 """Return the output directory name
0611
0612 Arguments:
0613 newRepository -- String for base directory for output files
0614 newRelease -- String for CMSSW release
0615 newSelection -- String for histogram selection
0616 """
0617 pileup = ""
0618 if self.hasPileup() and not self._fastsim:
0619 pileup = "_"+self._putype
0620 return "{newRepository}/{newRelease}/{newSelection}{pileup}/{sample}".format(
0621 newRepository=newRepository, newRelease=newRelease, newSelection=newSelection,
0622 pileup=pileup, sample=sample)
0623
0624 def filename(self, newRelease):
0625 """Return the DQM file name
0626
0627 Arguments:
0628 newRelease -- String for CMSSW release
0629 """
0630 pileup = ""
0631 fastsim = ""
0632 midfix = ""
0633 sample = self._sample
0634 if self._append is not None:
0635 midfix += self._append
0636 if self._midfix is not None:
0637 midfix += "_"+self._midfix
0638 if self.hasPileup():
0639 if self._fastsim:
0640
0641
0642
0643
0644
0645 pileup = "PU"+self.pileupType(newRelease)+"_"
0646 else:
0647 pileup = "PU"+self.pileupType(newRelease)+"_"
0648 if self._fastsim:
0649 fastsim = "_FastSim"
0650
0651 globalTag = _getGlobalTag(self, newRelease)
0652
0653 fname = 'DQM_V{dqmVersion}_R000000001__{sample}{midfix}__{newrelease}-{pileup}{globaltag}{appendGlobalTag}{fastsim}-{version}__DQMIO.root'.format(
0654 sample=sample, midfix=midfix, newrelease=_stripRelease(newRelease),
0655 pileup=pileup, globaltag=globalTag, appendGlobalTag=self._appendGlobalTag, fastsim=fastsim,
0656 version=self.version(newRelease), dqmVersion=self._dqmVersion
0657 )
0658
0659 return fname
0660
0661 def datasetpattern(self, newRelease):
0662 """Return the dataset pattern
0663
0664 Arguments:
0665 newRelease -- String for CMSSW release
0666 """
0667 pileup = ""
0668 fastsim = ""
0669 digi = ""
0670 if self.hasPileup():
0671 pileup = "-PU_"
0672 if self._fastsim:
0673 fastsim = "_FastSim-"
0674 digi = "DIGI-"
0675 else:
0676 fastsim = "*"
0677 globalTag = _getGlobalTag(self, newRelease)
0678 return "{sample}/{newrelease}-{pileup}{globaltag}{fastsim}{version}/GEN-SIM-{digi}RECO".format(
0679 sample=self._sample, newrelease=newRelease,
0680 pileup=pileup, globaltag=globalTag, fastsim=fastsim, digi=digi,
0681 version=self.version(newRelease)
0682 )
0683
0684 class Validation:
0685 """Base class for Tracking/Vertex validation."""
0686 def __init__(self, fullsimSamples, fastsimSamples, refRelease, refRepository, newRelease, newRepository, newFileModifier=None, selectionName=""):
0687 """Constructor.
0688
0689 Arguments:
0690 fullsimSamples -- List of Sample objects for FullSim samples (may be empty)
0691 fastsimSamples -- List of Sample objects for FastSim samples (may be empty)
0692 refRelease -- String for reference CMSSW release (can be None for no reference release)
0693 newRepository -- String for directory whete to put new files
0694 newRelease -- CMSSW release to be validated
0695 refRepository -- String for directory where reference root files are
0696 newFileModifier -- If given, a function to modify the names of the new files (function takes a string and returns a string)
0697 selectionName -- If given, use this string as the selection name (appended to GlobalTag for directory names)
0698 """
0699 try:
0700 self._newRelease = os.environ["CMSSW_VERSION"]
0701 except KeyError:
0702 print('Error: CMSSW environment variables are not available.', file=sys.stderr)
0703 print(' Please run cmsenv', file=sys.stderr)
0704 sys.exit()
0705
0706 self._fullsimSamples = fullsimSamples
0707 self._fastsimSamples = fastsimSamples
0708 self._refRelease = refRelease
0709 self._refRepository = refRepository
0710 self._newRelease = newRelease
0711 self._newBaseDir = os.path.join(newRepository, self._newRelease)
0712 self._newFileModifier = newFileModifier
0713 self._selectionName = selectionName
0714
0715 def _getDirectoryName(self, *args, **kwargs):
0716 return None
0717
0718 def _getSelectionName(self, *args, **kwargs):
0719 return self._selectionName
0720
0721 def download(self):
0722 """Download DQM files. Requires grid certificate and asks your password for it."""
0723 filenames = [s.filename(self._newRelease) for s in self._fullsimSamples+self._fastsimSamples]
0724 if self._newFileModifier is not None:
0725 filenames = list(map(self._newFileModifier, filenames))
0726 filenames = [f for f in filenames if not os.path.exists(f)]
0727 if len(filenames) == 0:
0728 print("All files already downloaded")
0729 return
0730
0731 relvalUrl = _getRelValUrl(self._newRelease)
0732 urls = [relvalUrl+f for f in filenames]
0733 certfile = os.path.join(os.environ["HOME"], ".globus", "usercert.pem")
0734 if not os.path.exists(certfile):
0735 print("Certificate file {certfile} does not exist, unable to download RelVal files from {url}".format(certfile=certfile, url=relvalUrl))
0736 sys.exit(1)
0737 keyfile = os.path.join(os.environ["HOME"], ".globus", "userkey.pem")
0738 if not os.path.exists(certfile):
0739 print("Private key file {keyfile} does not exist, unable to download RelVal files from {url}".format(keyfile=keyfile, url=relvalUrl))
0740 sys.exit(1)
0741
0742
0743 cmd = ["curl", "--cert-type", "PEM", "--cert", certfile, "--key", keyfile, "-k"]
0744 for u in urls:
0745 cmd.extend(["-O", u])
0746 print("Downloading %d files from RelVal URL %s:" % (len(filenames), relvalUrl))
0747 print(" "+"\n ".join(filenames))
0748 print("Please provide your private key pass phrase when curl asks it")
0749 ret = subprocess.call(cmd)
0750 if ret != 0:
0751 print("Downloading failed with exit code %d" % ret)
0752 sys.exit(1)
0753
0754
0755 allFine = True
0756 for f in filenames:
0757 p = subprocess.Popen(["file", f], stdout=subprocess.PIPE)
0758 stdout = p.communicate()[0]
0759 if p.returncode != 0:
0760 print("file command failed with exit code %d" % p.returncode)
0761 sys.exit(1)
0762 if not "ROOT" in stdout:
0763 print("File {f} is not ROOT, please check the correct version, GlobalTag etc. from {url}".format(f=f, url=relvalUrl))
0764 allFine = False
0765 if os.path.exists(f):
0766 os.remove(f)
0767 if not allFine:
0768 sys.exit(1)
0769
0770 def createHtmlReport(self):
0771 return html.HtmlReport(self._newRelease, self._newBaseDir)
0772
0773 def doPlots(self, plotter, plotterDrawArgs={}, limitSubFoldersOnlyTo=None, htmlReport=html.HtmlReportDummy(), doFastVsFull=True, doPhase2PU=False):
0774 """Create validation plots.
0775
0776 Arguments:
0777 plotter -- plotting.Plotter object that does the plotting
0778
0779 Keyword arguments:
0780 plotterDrawArgs -- Dictionary for additional arguments to Plotter.draw() (default: {})
0781 limitSubFoldersOnlyTo -- If not None, should be a dictionary from string to an object. The string is the name of a PlotFolder, and the object is PlotFolder-type specific to limit the subfolders to be processed. In general case the object is a list of strings, but e.g. for track iteration plots it is a function taking the algo and quality as parameters.
0782 htmlReport -- Object returned by createHtmlReport(), in case HTML report generation is desired
0783 doFastVsFull -- Do FastSim vs. FullSim comparison? (default: True)
0784 doPhase2PU -- Do Phase2 PU 200 vs. 140 comparison (default: False)
0785 """
0786 self._plotter = plotter
0787 self._plotterDrawArgs = plotterDrawArgs
0788
0789
0790 for sample in self._fullsimSamples+self._fastsimSamples:
0791
0792 harvestedFile = sample.filename(self._newRelease)
0793 if not os.path.exists(harvestedFile):
0794 print("Harvested file %s does not exist!" % harvestedFile)
0795 sys.exit(1)
0796
0797 plotterInstance = plotter.readDirs(harvestedFile)
0798 htmlReport.beginSample(sample)
0799 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
0800 if not _processPlotsForSample(plotterFolder, sample):
0801 continue
0802 plotFiles = self._doPlots(sample, harvestedFile, plotterFolder, dqmSubFolder, htmlReport)
0803 htmlReport.addPlots(plotterFolder, dqmSubFolder, plotFiles)
0804
0805
0806 if doFastVsFull:
0807 self._doFastsimFastVsFullPlots(limitSubFoldersOnlyTo, htmlReport)
0808
0809
0810 if doPhase2PU:
0811 self._doPhase2PileupPlots(limitSubFoldersOnlyTo, htmlReport)
0812
0813 def _doFastsimFastVsFullPlots(self, limitSubFoldersOnlyTo, htmlReport):
0814 for fast in self._fastsimSamples:
0815 correspondingFull = None
0816 for full in self._fullsimSamples:
0817 if fast.name() != full.name():
0818 continue
0819 if fast.pileupEnabled():
0820 if not full.pileupEnabled():
0821 continue
0822 if fast.fastsimCorrespondingFullsimPileup() != full.pileupType():
0823 continue
0824 else:
0825 if full.pileupEnabled():
0826 continue
0827
0828 if correspondingFull is None:
0829 correspondingFull = full
0830 else:
0831 raise Exception("Got multiple compatible FullSim samples for FastSim sample %s %s" % (fast.name(), fast.pileup()))
0832 if correspondingFull is None:
0833 print("WARNING: Did not find compatible FullSim sample for FastSim sample %s %s, omitting FastSim vs. FullSim comparison" % (fast.name(), fast.pileup()))
0834 continue
0835
0836
0837 harvestedFile = fast.filename(self._newRelease)
0838 plotterInstance = self._plotter.readDirs(harvestedFile)
0839 htmlReport.beginSample(fast, fastVsFull=True)
0840 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
0841 if not _processPlotsForSample(plotterFolder, fast):
0842 continue
0843 plotFiles = self._doPlotsFastFull(fast, correspondingFull, plotterFolder, dqmSubFolder, htmlReport)
0844 htmlReport.addPlots(plotterFolder, dqmSubFolder, plotFiles)
0845
0846 def _doPhase2PileupPlots(self, limitSubFoldersOnlyTo, htmlReport):
0847 def _stripScenario(name):
0848 puindex = name.find("PU")
0849 if puindex < 0:
0850 return name
0851 return name[:puindex]
0852
0853 pu140samples = {}
0854 for sample in self._fullsimSamples:
0855 if sample.pileupNumber() == 140:
0856 key = (sample.name(), _stripScenario(sample.scenario()))
0857 if key in pu140samples:
0858 raise Exception("Duplicate entry for sample %s in scenario %s" % (sample.name(), sample.scenar()))
0859 pu140samples[key] = sample
0860
0861 for sample in self._fullsimSamples:
0862 if sample.pileupNumber() != 200:
0863 continue
0864 key = (sample.name(), _stripScenario(sample.scenario()))
0865 if not key in pu140samples:
0866 continue
0867
0868 sample_pu140 = pu140samples[key]
0869
0870
0871 harvestedFile = sample.filename(self._newRelease)
0872 plotterInstance = self._plotter.readDirs(harvestedFile)
0873 htmlReport.beginSample(sample, pileupComparison="vs. PU140")
0874 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
0875 if not _processPlotsForSample(plotterFolder, sample):
0876 continue
0877 plotFiles = self._doPlotsPileup(sample_pu140, sample, plotterFolder, dqmSubFolder, htmlReport)
0878 htmlReport.addPlots(plotterFolder, dqmSubFolder, plotFiles)
0879
0880
0881 def _getRefFileAndSelection(self, sample, plotterFolder, dqmSubFolder, selectionNameBase, valname):
0882 if self._refRelease is None:
0883 return (None, "")
0884
0885 refGlobalTag = _getGlobalTag(sample, self._refRelease)
0886 def _createRefSelection(selectionName):
0887 sel = refGlobalTag+selectionNameBase+selectionName
0888 if sample.pileupEnabled():
0889 refPu = sample.pileupType(self._refRelease)
0890 if refPu != "":
0891 sel += "_"+refPu
0892 return sel
0893 refSelection = _createRefSelection(plotterFolder.getSelectionName(dqmSubFolder))
0894
0895
0896 refValFile = None
0897 triedRefValFiles = []
0898 tmp = [self._refRepository, self._refRelease]
0899 if sample.fastsim():
0900 tmp.extend(["fastsim", self._refRelease])
0901 for selName in plotterFolder.getSelectionNameIterator(dqmSubFolder):
0902 refSel = _createRefSelection(selName)
0903 refdir = os.path.join(*(tmp+[refSel, sample.name()]))
0904
0905
0906 refValFilePath = os.path.join(refdir, valname)
0907 if os.path.exists(refValFilePath):
0908 refSelection = refSel
0909 refValFile = ROOT.TFile.Open(refValFilePath)
0910 break
0911 else:
0912 triedRefValFiles.append(refValFilePath)
0913 if refValFile is None:
0914 if len(triedRefValFiles) == 1:
0915 if plotting.verbose:
0916 print("Reference file %s not found" % triedRefValFiles[0])
0917 else:
0918 if plotting.verbose:
0919 print("None of the possible reference files %s not found" % ",".join(triedRefValFiles))
0920
0921 return (refValFile, refSelection)
0922
0923 def _doPlots(self, sample, harvestedFile, plotterFolder, dqmSubFolder, htmlReport):
0924 """Do the real plotting work for a given sample and DQM subfolder"""
0925
0926 newGlobalTag = _getGlobalTag(sample, self._newRelease)
0927
0928
0929 selectionNameBase = "_"+sample.pileup()
0930 newSelection = newGlobalTag+selectionNameBase+plotterFolder.getSelectionName(dqmSubFolder)
0931 if sample.pileupEnabled():
0932 newPu = sample.pileupType(self._newRelease)
0933 if newPu != "":
0934 newSelection += "_"+newPu
0935
0936 valname = "val.{sample}.root".format(sample=sample.name())
0937
0938
0939 (refValFile, refSelection) = self._getRefFileAndSelection(sample, plotterFolder, dqmSubFolder, selectionNameBase, valname)
0940
0941
0942 tmp = []
0943 if sample.fastsim():
0944 tmp.extend(["fastsim", self._newRelease])
0945 tmp.extend([newSelection, sample.name()])
0946 newsubdir = os.path.join(*tmp)
0947 newdir = os.path.join(self._newBaseDir, newsubdir)
0948 if not os.path.exists(newdir):
0949 os.makedirs(newdir)
0950 valnameFullPath = os.path.join(newdir, valname)
0951
0952
0953
0954 newValFile = _copySubDir(harvestedFile, valnameFullPath, plotterFolder.getPossibleDQMFolders(), dqmSubFolder.subfolder if dqmSubFolder is not None else None)
0955 fileList = []
0956
0957
0958 if plotting.verbose:
0959 print("Comparing ref and new {sim} {sample} {translatedFolder}".format(
0960 sim="FullSim" if not sample.fastsim() else "FastSim",
0961 sample=sample.name(), translatedFolder=str(dqmSubFolder.translated) if dqmSubFolder is not None else ""))
0962 rootFiles = [refValFile, newValFile]
0963 legendLabels = [
0964 "%s, %s %s" % (sample.name(), _stripRelease(self._refRelease), refSelection) if self._refRelease is not None else "dummy",
0965 "%s, %s %s" % (sample.name(), _stripRelease(self._newRelease), newSelection)
0966 ]
0967 plotterFolder.create(rootFiles, legendLabels, dqmSubFolder, isPileupSample=sample.pileupEnabled())
0968 fileList.extend(plotterFolder.draw(directory=newdir, **self._plotterDrawArgs))
0969
0970 if len(fileList) > 0:
0971 fileList.append(valnameFullPath)
0972
0973
0974 for tableCreator in plotterFolder.getTableCreators():
0975 htmlReport.addTable(tableCreator.create(rootFiles, legendLabels, dqmSubFolder))
0976
0977 newValFile.Close()
0978 if refValFile is not None:
0979 refValFile.Close()
0980
0981 if len(fileList) == 0:
0982 print("No object found in %s" % plotterFolder.getName())
0983 return []
0984
0985 dups = _findDuplicates(fileList)
0986 if len(dups) > 0:
0987 print("Plotter produced multiple files with names", ", ".join(dups))
0988 print("Typically this is a naming problem in the plotter configuration")
0989 sys.exit(1)
0990
0991
0992 print("Created plots and %s in %s" % (valname, newdir))
0993 return list(map(lambda n: n.replace(newdir, newsubdir), fileList))
0994
0995 def _doPlotsFastFull(self, fastSample, fullSample, plotterFolder, dqmSubFolder, htmlReport):
0996 """Do the real plotting work for FastSim vs. FullSim for a given algorithm, quality flag, and sample."""
0997
0998 fastGlobalTag = _getGlobalTag(fastSample, self._newRelease)
0999 fullGlobalTag = _getGlobalTag(fullSample, self._newRelease)
1000
1001
1002 tmp = plotterFolder.getSelectionName(dqmSubFolder)
1003 fastSelection = fastGlobalTag+"_"+fastSample.pileup()+tmp
1004 fullSelection = fullGlobalTag+"_"+fullSample.pileup()+tmp
1005 if fullSample.pileupEnabled():
1006 fullSelection += "_"+fullSample.pileupType(self._newRelease)
1007 fastSelection += "_"+fastSample.pileupType(self._newRelease)
1008
1009
1010 fastdir = os.path.join(self._newBaseDir, "fastsim", self._newRelease, fastSelection, fastSample.name())
1011 fulldir = os.path.join(self._newBaseDir, fullSelection, fullSample.name())
1012 newsubdir = os.path.join("fastfull", self._newRelease, fastSelection, fastSample.name())
1013 newdir = os.path.join(self._newBaseDir, newsubdir)
1014 if not os.path.exists(newdir):
1015 os.makedirs(newdir)
1016
1017
1018 valname = "val.{sample}.root".format(sample=fastSample.name())
1019 fastValFilePath = os.path.join(fastdir, valname)
1020 if not os.path.exists(fastValFilePath) and plotting.verbose:
1021 print("FastSim file %s not found" % fastValFilePath)
1022 fullValFilePath = os.path.join(fulldir, valname)
1023 if not os.path.exists(fullValFilePath) and plotting.verbose:
1024 print("FullSim file %s not found" % fullValFilePath)
1025
1026 fastValFile = ROOT.TFile.Open(fastValFilePath)
1027 fullValFile = ROOT.TFile.Open(fullValFilePath)
1028
1029
1030 if plotting.verbose:
1031 print("Comparing FullSim and FastSim {sample} {translatedFolder}".format(
1032 sample=fastSample.name(), translatedFolder=str(dqmSubFolder.translated) if dqmSubFolder is not None else ""))
1033 rootFiles = [fullValFile, fastValFile]
1034 legendLabels = [
1035 "FullSim %s, %s %s" % (fullSample.name(), _stripRelease(self._newRelease), fullSelection),
1036 "FastSim %s, %s %s" % (fastSample.name(), _stripRelease(self._newRelease), fastSelection),
1037 ]
1038 plotterFolder.create(rootFiles, legendLabels, dqmSubFolder, isPileupSample=fastSample.pileupEnabled(), requireAllHistograms=True)
1039 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1040
1041
1042 for tableCreator in plotterFolder.getTableCreators():
1043 htmlReport.addTable(tableCreator.create(rootFiles, legendLabels, dqmSubFolder))
1044
1045 fullValFile.Close()
1046 fastValFile.Close()
1047
1048 if len(fileList) == 0:
1049 print("No object found in %s" % plotterFolder.getName())
1050 return []
1051
1052 dups = _findDuplicates(fileList)
1053 if len(dups) > 0:
1054 print("Plotter produced multiple files with names", ", ".join(dups))
1055 print("Typically this is a naming problem in the plotter configuration")
1056 sys.exit(1)
1057
1058
1059 print("Created plots in %s" % (newdir))
1060 return list(map(lambda n: n.replace(newdir, newsubdir), fileList))
1061
1062 def _doPlotsPileup(self, pu140Sample, pu200Sample, plotterFolder, dqmSubFolder, htmlReport):
1063 """Do the real plotting work for two pileup scenarios for a given algorithm, quality flag, and sample."""
1064
1065 pu140GlobalTag = _getGlobalTag(pu140Sample, self._newRelease)
1066 pu200GlobalTag = _getGlobalTag(pu200Sample, self._newRelease)
1067
1068
1069 tmp = plotterFolder.getSelectionName(dqmSubFolder)
1070 pu140Selection = pu140GlobalTag+"_"+pu140Sample.pileup()+tmp+"_"+pu140Sample.pileupType(self._newRelease)
1071 pu200Selection = pu200GlobalTag+"_"+pu200Sample.pileup()+tmp+"_"+pu200Sample.pileupType(self._newRelease)
1072
1073
1074 pu140dir = os.path.join(self._newBaseDir, pu140Selection, pu140Sample.name())
1075 pu200dir = os.path.join(self._newBaseDir, pu200Selection, pu200Sample.name())
1076 newsubdir = os.path.join("pileup", self._newRelease, pu200Selection, pu200Sample.name())
1077 newdir = os.path.join(self._newBaseDir, newsubdir)
1078 if not os.path.exists(newdir):
1079 os.makedirs(newdir)
1080
1081
1082 valname = "val.{sample}.root".format(sample=pu140Sample.name())
1083 pu140ValFilePath = os.path.join(pu140dir, valname)
1084 if not os.path.exists(pu140ValFilePath):
1085 if plotting.verbose:
1086 print("PU140 file %s not found" % pu140ValFilePath)
1087 return []
1088 pu200ValFilePath = os.path.join(pu200dir, valname)
1089 if not os.path.exists(pu200ValFilePath):
1090 if plotting.verbose:
1091 print("PU200 file %s not found" % pu200ValFilePath)
1092 return []
1093
1094 pu140ValFile = ROOT.TFile.Open(pu140ValFilePath)
1095 pu200ValFile = ROOT.TFile.Open(pu200ValFilePath)
1096
1097
1098 if plotting.verbose:
1099 print("Comparing PU140 and PU200 {sample} {translatedFolder}".format(
1100 sample=pu200Sample.name(), translatedFolder=str(dqmSubFolder.translated) if dqmSubFolder is not None else ""))
1101 rootFiles = [pu140ValFile, pu200ValFile]
1102 legendLabels = [
1103 "%s, %s %s" % (pu140Sample.name(), _stripRelease(self._newRelease), pu140Selection),
1104 "%s, %s %s" % (pu200Sample.name(), _stripRelease(self._newRelease), pu200Selection),
1105 ]
1106 plotterFolder.create(rootFiles, legendLabels, dqmSubFolder, isPileupSample=pu140Sample.pileupEnabled(), requireAllHistograms=True)
1107 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1108
1109
1110 for tableCreator in plotterFolder.getTableCreators():
1111 htmlReport.addTable(tableCreator.create(rootFiles, legendLabels, dqmSubFolder))
1112
1113 pu200ValFile.Close()
1114 pu140ValFile.Close()
1115
1116 if len(fileList) == 0:
1117 print("No object found in %s" % plotterFolder.getName())
1118 return []
1119
1120 dups = _findDuplicates(fileList)
1121 if len(dups) > 0:
1122 print("Plotter produced multiple files with names", ", ".join(dups))
1123 print("Typically this is a naming problem in the plotter configuration")
1124 sys.exit(1)
1125
1126
1127 print("Created plots in %s" % (newdir))
1128 return list(map(lambda n: n.replace(newdir, newsubdir), fileList))
1129
1130
1131 def _copySubDir(oldfile, newfile, basenames, dirname):
1132 """Copy a subdirectory from oldfile to newfile.
1133
1134 Arguments:
1135 oldfile -- String for source TFile
1136 newfile -- String for destination TFile
1137 basenames -- List of strings for base directories, first existing one is picked
1138 dirname -- String for directory name under the base directory
1139 """
1140 oldf = ROOT.TFile.Open(oldfile)
1141
1142 dirold = None
1143 for basename in basenames:
1144 dirold = oldf.GetDirectory(basename)
1145 if dirold:
1146 break
1147 if not dirold:
1148 raise Exception("Did not find any of %s directories from file %s" % (",".join(basenames), oldfile))
1149 if dirname:
1150 d = dirold.Get(dirname)
1151 if not d:
1152 raise Exception("Did not find directory %s under %s" % (dirname, dirold.GetPath()))
1153 dirold = d
1154
1155 newf = ROOT.TFile.Open(newfile, "RECREATE")
1156 dirnew = newf
1157 for d in basenames[0].split("/"):
1158 dirnew = dirnew.mkdir(d)
1159 if dirname:
1160 dirnew = dirnew.mkdir(dirname)
1161 _copyDir(dirold, dirnew)
1162
1163 oldf.Close()
1164 return newf
1165
1166 def _copyDir(src, dst):
1167 """Copy non-TTree objects from src TDirectory to dst TDirectory."""
1168 keys = src.GetListOfKeys()
1169 for key in keys:
1170 classname = key.GetClassName()
1171 cl = ROOT.TClass.GetClass(classname)
1172 if not cl:
1173 continue
1174 if not (cl.InheritsFrom("TTree") and cl.InheritsFrom("TDirectory")):
1175 dst.cd()
1176 obj = key.ReadObj()
1177 obj.Write()
1178 obj.Delete()
1179
1180 def _findDuplicates(lst):
1181 found = set()
1182 found2 = set()
1183 for x in lst:
1184 if x in found:
1185 found2.add(x)
1186 else:
1187 found.add(x)
1188 return list(found2)
1189
1190 def _doPlotsForPlotter(self, plotter, sample, limitSubFoldersOnlyTo=None):
1191 plottingProcess = self._nProc
1192 if plottingProcess<=0: plottingProcess = os.cpu_count()
1193 plotterInstance = plotter.readDirs(*self._openFiles)
1194 manager = multiprocessing.Manager()
1195 return_dict = manager.dict()
1196 proc = []
1197 iProc = 0
1198 active_proc = []
1199
1200 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
1201 if sample is not None and not _processPlotsForSample(plotterFolder, sample):
1202 continue
1203 newsubdir = self._subdirprefix+plotterFolder.getSelectionName(dqmSubFolder)
1204 newdir = os.path.join(self._newdir, newsubdir)
1205 if not os.path.exists(newdir):
1206 os.makedirs(newdir, exist_ok=True)
1207
1208 plotterFolder.create(self._openFiles, self._labels, dqmSubFolder)
1209 while len(active_proc)>=plottingProcess:
1210 time.sleep(0.1)
1211 active_proc = [p for p in active_proc if p.is_alive()]
1212 p = multiprocessing.Process(target=self._doPlots, args=(plotterFolder, dqmSubFolder, newsubdir, newdir, iProc, return_dict))
1213 proc.append((plotterFolder, dqmSubFolder, p))
1214 active_proc.append(p)
1215 p.start()
1216 iProc += 1
1217
1218 for i in range(iProc):
1219 proc[i][2].join()
1220 if len(return_dict[i]) > 0:
1221 self._htmlReport.addPlots(proc[i][0], proc[i][1], return_dict[i])
1222
1223 class SimpleSample:
1224 def __init__(self, label, name, fileLegends, pileup=True, customPileupLabel=""):
1225 self._label = label
1226 self._name = name
1227 self._fileLegends = fileLegends
1228 self._pileup = pileup
1229 self._customPileupLabel = customPileupLabel
1230
1231 def digest(self):
1232
1233 return self._label
1234
1235 def label(self):
1236 return self._label
1237
1238 def name(self):
1239 return self._name
1240
1241 def files(self):
1242 return [t[0] for t in self._fileLegends]
1243
1244 def legendLabels(self):
1245 return [t[1] for t in self._fileLegends]
1246
1247 def fastsim(self):
1248
1249 return False
1250
1251 def pileupEnabled(self):
1252 return self._pileup
1253
1254 def customPileupLabel(self):
1255 return self._customPileupLabel
1256
1257 def doElectron(self):
1258 return True
1259
1260 def doConversion(self):
1261 return True
1262
1263 def doBHadron(self):
1264 return True
1265
1266 class SimpleValidation:
1267 def __init__(self, samples, newdir, nProc=0):
1268 self._samples = samples
1269 self._newdir = newdir
1270 self._nProc = nProc
1271 if not os.path.exists(newdir):
1272 os.makedirs(newdir)
1273
1274 self._htmlReport = html.HtmlReportDummy()
1275
1276 def createHtmlReport(self, validationName=""):
1277 if hasattr(self._htmlReport, "write"):
1278 raise Exception("HTML report object already created. There is probably some logic error in the calling code.")
1279 self._htmlReport = html.HtmlReport(validationName, self._newdir)
1280 return self._htmlReport
1281
1282 def doPlots(self, plotters, plotterDrawArgs={}, **kwargs):
1283 self._plotterDrawArgs = plotterDrawArgs
1284
1285 for sample in self._samples:
1286 self._subdirprefix = sample.label()
1287 self._labels = sample.legendLabels()
1288 self._htmlReport.beginSample(sample)
1289
1290 self._openFiles = []
1291 for f in sample.files():
1292 if os.path.exists(f):
1293 self._openFiles.append(ROOT.TFile.Open(f))
1294 else:
1295 print("File %s not found (from sample %s), ignoring it" % (f, sample.name()))
1296 self._openFiles.append(None)
1297
1298 for plotter in plotters:
1299 _doPlotsForPlotter(self, plotter, sample, **kwargs)
1300
1301 for tf in self._openFiles:
1302 if tf is not None:
1303 tf.Close()
1304 self._openFiles = []
1305
1306 def _doPlots(self, plotterFolder, dqmSubFolder, newsubdir, newdir, iProc, return_dict):
1307 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1308
1309 if len(fileList) == 0:
1310 print("No object found in %s" % plotterFolder.getName())
1311
1312 for tableCreator in plotterFolder.getTableCreators():
1313 self._htmlReport.addTable(tableCreator.create(self._openFiles, self._labels, dqmSubFolder))
1314
1315 dups = _findDuplicates(fileList)
1316 if len(dups) > 0:
1317 print("Plotter produced multiple files with names", ", ".join(dups))
1318 print("Typically this is a naming problem in the plotter configuration")
1319 sys.exit(1)
1320
1321 if self._plotterDrawArgs.get("separate", False):
1322 if not os.path.exists("%s/res"%newdir):
1323 os.makedirs("%s/res"%newdir)
1324 downloadables = ["index.php", "res/jquery-ui.js", "res/jquery.js", "res/style.css", "res/style.js", "res/theme.css"]
1325 for d in downloadables:
1326 if not os.path.exists("%s/%s" % (newdir,d)):
1327 urllib.request.urlretrieve("https://raw.githubusercontent.com/musella/php-plots/master/%s"%d, "%s/%s"%(newdir,d))
1328
1329 print("Created plots in %s" % newdir)
1330 return_dict[iProc] = list(map(lambda n: n.replace(newdir, newsubdir), fileList))
1331
1332 class SeparateValidation:
1333
1334
1335 def __init__(self, samples, newdir, nProc=0):
1336 self._samples = samples
1337 self._newdir = newdir
1338 self._nProc = nProc
1339 if not os.path.exists(newdir):
1340 os.makedirs(newdir)
1341
1342 self._htmlReport = html.HtmlReportDummy()
1343
1344 def createHtmlReport(self, validationName=""):
1345 if hasattr(self._htmlReport, "write"):
1346 raise Exception("HTML report object already created. There is probably some logic error in the calling code.")
1347 self._htmlReport = html.HtmlReport(validationName, self._newdir)
1348 return self._htmlReport
1349
1350 def doPlots(self, plotters, plotterDrawArgs={}, **kwargs):
1351 self._plotterDrawArgs = plotterDrawArgs
1352
1353 for sample in self._samples:
1354 self._subdirprefix = sample.label()
1355 self._labels = sample.legendLabels()
1356 self._htmlReport.beginSample(sample)
1357
1358 self._openFiles = []
1359 for f in sample.files():
1360 if os.path.exists(f):
1361 self._openFiles.append(ROOT.TFile.Open(f))
1362 else:
1363 print("File %s not found (from sample %s), ignoring it" % (f, sample.name()))
1364 self._openFiles.append(None)
1365
1366 for plotter in plotters:
1367 _doPlotsForPlotter(self, plotter, sample, **kwargs)
1368
1369 for tf in self._openFiles:
1370 if tf is not None:
1371 tf.Close()
1372 self._openFiles = []
1373
1374 def _doPlots(self, plotterFolder, dqmSubFolder, newsubdir, newdir, iProc, return_dict):
1375 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1376
1377
1378 if len(fileList) == 0:
1379 print("No object found in %s" % plotterFolder.getName())
1380
1381
1382 dups = _findDuplicates(fileList)
1383 if len(dups) > 0:
1384 print("Plotter produced multiple files with names", ", ".join(dups))
1385 print("Typically this is a naming problem in the plotter configuration")
1386 sys.exit(1)
1387
1388 linkList = []
1389 for f in fileList:
1390 if f[:f.rfind("/")] not in linkList :
1391 if str(f[:f.rfind("/")]) != str(newdir) :
1392 linkList.append(f[:f.rfind("/")])
1393
1394 for tableCreator in plotterFolder.getTableCreators():
1395 self._htmlReport.addTable(tableCreator.create(self._openFiles, self._labels, dqmSubFolder))
1396
1397 for link in linkList :
1398 if not os.path.exists("%s/res"%link):
1399 os.makedirs("%s/res"%link)
1400 downloadables = ["index.php", "res/jquery-ui.js", "res/jquery.js", "res/style.css", "res/style.js", "res/theme.css"]
1401 for d in downloadables:
1402 if not os.path.exists("%s/%s" % (link,d)):
1403 urllib.request.urlretrieve("https://raw.githubusercontent.com/rovere/php-plots/master/%s"%d, "%s/%s"%(link,d))
1404
1405 print("Created separated plots in %s" % newdir)
1406 return_dict[iProc] = list(map(lambda n: n.replace(newdir, newsubdir), linkList))
