File indexing completed on 2024-11-26 02:34:39
0001 import os
0002 import re
0003 import sys
0004 import time
0005 import shutil
0006 import subprocess
0007 import urllib.request
0008 import multiprocessing
0009
0010 import ROOT
0011 ROOT.gROOT.SetBatch(True)
0012 ROOT.PyConfig.IgnoreCommandLineOptions = True
0013
0014 from . import plotting
0015 from . import html
0016
0017
0018 _globalTags = {
0019 "CMSSW_6_2_0": {"default": "PRE_ST62_V8"},
0020 "CMSSW_6_2_0_SLHC15": {"UPG2019withGEM": "DES19_62_V8", "UPG2023SHNoTaper": "DES23_62_V1"},
0021 "CMSSW_6_2_0_SLHC17": {"UPG2019withGEM": "DES19_62_V8", "UPG2023SHNoTaper": "DES23_62_V1"},
0022 "CMSSW_6_2_0_SLHC20": {"UPG2019withGEM": "DES19_62_V8", "UPG2023SHNoTaper": "DES23_62_V1_UPG2023SHNoTaper"},
0023 "CMSSW_6_2_0_SLHC22": {"UPG2023SHNoTaper": "PH2_1K_FB_V6_UPG23SHNoTaper",
0024
0025 "2023GReco": "PH2_1K_FB_V6_UPG23SHNoTaper", "2023GRecoPU35": "", "2023GRecoPU140": "", "2023GRecoPU200": "",
0026 "2023tilted": "PH2_1K_FB_V6_UPG23SHNoTaper", "2023tiltedPU35": "", "2023tiltedPU140": "", "2023tiltedPU200": ""},
0027 "CMSSW_6_2_0_SLHC26": {"LHCCRefPU140": "DES23_62_V1_LHCCRefPU140", "LHCCRefPU200": "DES23_62_V1_LHCCRefPU200",
0028
0029 "2023GReco": "", "2023GRecoPU35": "", "2023GRecoPU140": "DES23_62_V1_LHCCRefPU140", "2023GRecoPU200": "DES23_62_V1_LHCCRefPU200",
0030 "2023tilted": "", "2023tiltedPU35": "", "2023tiltedPU140": "DES23_62_V1_LHCCRefPU140", "2023tiltedPU200": "DES23_62_V1_LHCCRefPU200"},
0031 "CMSSW_6_2_0_SLHC27_phase1": {"default": "DES17_62_V8_UPG17"},
0032 "CMSSW_7_0_0": {"default": "POSTLS170_V3", "fullsim_50ns": "POSTLS170_V4"},
0033 "CMSSW_7_0_0_AlcaCSA14": {"default": "POSTLS170_V5_AlcaCSA14", "fullsim_50ns": "POSTLS170_V6_AlcaCSA14"},
0034 "CMSSW_7_0_7_pmx": {"default": "PLS170_V7AN1", "fullsim_50ns": "PLS170_V6AN1"},
0035 "CMSSW_7_0_9_patch3": {"default": "PLS170_V7AN2", "fullsim_50ns": "PLS170_V6AN2"},
0036 "CMSSW_7_0_9_patch3_Premix": {"default": "PLS170_V7AN2", "fullsim_50ns": "PLS170_V6AN2"},
0037 "CMSSW_7_1_0": {"default": "POSTLS171_V15", "fullsim_50ns": "POSTLS171_V16"},
0038 "CMSSW_7_1_9": {"default": "POSTLS171_V17", "fullsim_50ns": "POSTLS171_V18"},
0039 "CMSSW_7_1_9_patch2": {"default": "POSTLS171_V17", "fullsim_50ns": "POSTLS171_V18"},
0040 "CMSSW_7_1_10_patch2": {"default": "MCRUN2_71_V1", "fullsim_50ns": "MCRUN2_71_V0"},
0041 "CMSSW_7_2_0_pre5": {"default": "POSTLS172_V3", "fullsim_50ns": "POSTLS172_V4"},
0042 "CMSSW_7_2_0_pre7": {"default": "PRE_LS172_V11", "fullsim_50ns": "PRE_LS172_V12"},
0043 "CMSSW_7_2_0_pre8": {"default": "PRE_LS172_V15", "fullsim_50ns": "PRE_LS172_V16"},
0044 "CMSSW_7_2_0": {"default": "PRE_LS172_V15", "fullsim_50ns": "PRE_LS172_V16"},
0045 "CMSSW_7_2_0_PHYS14": {"default": "PHYS14_25_V1_Phys14"},
0046 "CMSSW_7_2_2_patch1": {"default": "MCRUN2_72_V1", "fullsim_50ns": "MCRUN2_72_V0"},
0047 "CMSSW_7_2_2_patch1_Fall14DR": {"default": "MCRUN2_72_V3_71XGENSIM"},
0048
0049 "CMSSW_7_3_0_pre1": {"default": "PRE_LS172_V15", "fullsim_50ns": "PRE_LS172_V16"},
0050
0051 "CMSSW_7_3_0_pre2": {"default": "MCRUN2_73_V1", "fullsim_50ns": "MCRUN2_73_V0"},
0052 "CMSSW_7_3_0_pre3": {"default": "MCRUN2_73_V5", "fullsim_50ns": "MCRUN2_73_V4"},
0053 "CMSSW_7_3_0": {"default": "MCRUN2_73_V7", "fullsim_50ns": "MCRUN2_73_V6"},
0054 "CMSSW_7_3_0_71XGENSIM": {"default": "MCRUN2_73_V7_71XGENSIM"},
0055 "CMSSW_7_3_0_71XGENSIM_FIXGT": {"default": "MCRUN2_73_V9_71XGENSIM_FIXGT"},
0056 "CMSSW_7_3_1_patch1": {"default": "MCRUN2_73_V9", "fastsim": "MCRUN2_73_V7"},
0057 "CMSSW_7_3_1_patch1_GenSim_7113": {"default": "MCRUN2_73_V9_GenSim_7113"},
0058 "CMSSW_7_3_3": {"default": "MCRUN2_73_V11", "fullsim_50ns": "MCRUN2_73_V10", "fastsim": "MCRUN2_73_V13"},
0059 "CMSSW_7_4_0_pre1": {"default": "MCRUN2_73_V5", "fullsim_50ns": "MCRUN2_73_V4"},
0060 "CMSSW_7_4_0_pre2": {"default": "MCRUN2_73_V7", "fullsim_50ns": "MCRUN2_73_V6"},
0061 "CMSSW_7_4_0_pre2_73XGENSIM": {"default": "MCRUN2_73_V7_73XGENSIM_Pythia6", "fullsim_50ns": "MCRUN2_73_V6_73XGENSIM_Pythia6"},
0062 "CMSSW_7_4_0_pre5": {"default": "MCRUN2_73_V7", "fullsim_50ns": "MCRUN2_73_V6"},
0063 "CMSSW_7_4_0_pre5_BS": {"default": "MCRUN2_73_V9_postLS1beamspot", "fullsim_50ns": "MCRUN2_73_V8_postLS1beamspot"},
0064 "CMSSW_7_4_0_pre6": {"default": "MCRUN2_74_V1", "fullsim_50ns": "MCRUN2_74_V0"},
0065 "CMSSW_7_4_0_pre8": {"default": "MCRUN2_74_V7", "fullsim_25ns": "MCRUN2_74_V5_AsympMinGT", "fullsim_50ns": "MCRUN2_74_V4_StartupMinGT"},
0066 "CMSSW_7_4_0_pre8_minimal": {"default": "MCRUN2_74_V5_MinGT", "fullsim_25ns": "MCRUN2_74_V5_AsympMinGT", "fullsim_50ns": "MCRUN2_74_V4_StartupMinGT"},
0067 "CMSSW_7_4_0_pre8_25ns_asymptotic": {"default": "MCRUN2_74_V7"},
0068 "CMSSW_7_4_0_pre8_50ns_startup": {"default": "MCRUN2_74_V6"},
0069 "CMSSW_7_4_0_pre8_50ns_asympref": {"default": "MCRUN2_74_V5A_AsympMinGT"},
0070 "CMSSW_7_4_0_pre8_50ns_asymptotic": {"default": "MCRUN2_74_V7A_AsympGT"},
0071 "CMSSW_7_4_0_pre8_ROOT6": {"default": "MCRUN2_74_V7"},
0072 "CMSSW_7_4_0_pre8_pmx": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0073 "CMSSW_7_4_0_pre8_pmx_v2": {"default": "MCRUN2_74_V7_gs_pre7", "fullsim_50ns": "MCRUN2_74_V6_gs_pre7"},
0074 "CMSSW_7_4_0_pre8_pmx_v3": {"default": "MCRUN2_74_V7_bis", "fullsim_50ns": "MCRUN2_74_V6_bis"},
0075 "CMSSW_7_4_0_pre9": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0076 "CMSSW_7_4_0_pre9_ROOT6": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0077 "CMSSW_7_4_0_pre9_extended": {"default": "MCRUN2_74_V7_extended"},
0078 "CMSSW_7_4_0": {"default": "MCRUN2_74_V7_gensim_740pre7", "fullsim_50ns": "MCRUN2_74_V6_gensim_740pre7", "fastsim": "MCRUN2_74_V7"},
0079 "CMSSW_7_4_0_71XGENSIM": {"default": "MCRUN2_74_V7_GENSIM_7_1_15", "fullsim_50ns": "MCRUN2_74_V6_GENSIM_7_1_15"},
0080 "CMSSW_7_4_0_71XGENSIM_PU": {"default": "MCRUN2_74_V7_gs7115_puProd", "fullsim_50ns": "MCRUN2_74_V6_gs7115_puProd"},
0081 "CMSSW_7_4_0_71XGENSIM_PXworst": {"default": "MCRUN2_74_V7C_pxWorst_gs7115", "fullsim_50ns": "MCRUN2_74_V6A_pxWorst_gs7115"},
0082 "CMSSW_7_4_0_71XGENSIM_PXbest": {"default": "MCRUN2_74_V7D_pxBest_gs7115", "fullsim_50ns": "MCRUN2_74_V6B_pxBest_gs7115"},
0083 "CMSSW_7_4_0_pmx": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0084 "CMSSW_7_4_1": {"default": "MCRUN2_74_V9_gensim_740pre7", "fullsim_50ns": "MCRUN2_74_V8_gensim_740pre7", "fastsim": "MCRUN2_74_V9"},
0085 "CMSSW_7_4_1_71XGENSIM": {"default": "MCRUN2_74_V9_gensim71X", "fullsim_50ns": "MCRUN2_74_V8_gensim71X"},
0086 "CMSSW_7_4_1_extended": {"default": "MCRUN2_74_V9_extended"},
0087 "CMSSW_7_4_3": {"default": "MCRUN2_74_V9", "fullsim_50ns": "MCRUN2_74_V8", "fastsim": "MCRUN2_74_V9", "fastsim_25ns": "MCRUN2_74_V9_fixMem"},
0088 "CMSSW_7_4_3_extended": {"default": "MCRUN2_74_V9_ext","fastsim": "MCRUN2_74_V9_fixMem"},
0089 "CMSSW_7_4_3_pmx": {"default": "MCRUN2_74_V9_ext", "fullsim_50ns": "MCRUN2_74_V8", "fastsim": "MCRUN2_74_V9_fixMem"},
0090 "CMSSW_7_4_3_patch1_unsch": {"default": "MCRUN2_74_V9_unsch", "fullsim_50ns": "MCRUN2_74_V8_unsch"},
0091 "CMSSW_7_4_4": {"default": "MCRUN2_74_V9_38Tbis", "fullsim_50ns": "MCRUN2_74_V8_38Tbis"},
0092 "CMSSW_7_4_4_0T": {"default": "MCRUN2_740TV1_0Tv2", "fullsim_50ns": "MCRUN2_740TV0_0TV2", "fullsim_25ns": "MCRUN2_740TV1_0TV2"},
0093 "CMSSW_7_4_6_patch6": {"default": "MCRUN2_74_V9_scheduled", "fullsim_50ns": "MCRUN2_74_V8_scheduled"},
0094 "CMSSW_7_4_6_patch6_unsch": {"default": "MCRUN2_74_V9", "fullsim_50ns": "MCRUN2_74_V8"},
0095 "CMSSW_7_4_6_patch6_noCCC": {"default": "MCRUN2_74_V9_unsch_noCCC", "fullsim_50ns": "MCRUN2_74_V8_unsch_noCCC"},
0096 "CMSSW_7_4_6_patch6_noCCC_v3": {"default": "MCRUN2_74_V9_unsch_noCCC_v3", "fullsim_50ns": "MCRUN2_74_V8_unsch_noCCC_v3"},
0097 "CMSSW_7_4_6_patch6_BS": {"default": "74X_mcRun2_asymptotic_realisticBS_v0_2015Jul24", "fullsim_50ns": "74X_mcRun2_startup_realisticBS_v0_2015Jul24PU", "fullsim_25ns": "74X_mcRun2_asymptotic_realisticBS_v0_2015Jul24PU"},
0098 "CMSSW_7_4_8_patch1_MT": {"default": "MCRUN2_74_V11_mulTrh", "fullsim_50ns": "MCRUN2_74_V10_mulTrh",},
0099 "CMSSW_7_4_12": {"default": "74X_mcRun2_asymptotic_v2", "fullsim_25ns": "74X_mcRun2_asymptotic_v2_v2", "fullsim_50ns": "74X_mcRun2_startup_v2_v2"},
0100 "CMSSW_7_5_0_pre1": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0101 "CMSSW_7_5_0_pre2": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0102 "CMSSW_7_5_0_pre3": {"default": "MCRUN2_74_V7", "fullsim_50ns": "MCRUN2_74_V6"},
0103 "CMSSW_7_5_0_pre4": {"default": "MCRUN2_75_V1", "fullsim_50ns": "MCRUN2_75_V0"},
0104 "CMSSW_7_5_0_pre5": {"default": "MCRUN2_75_V5", "fullsim_50ns": "MCRUN2_75_V4"},
0105 "CMSSW_7_5_0_pre6": {"default": "75X_mcRun2_asymptotic_v1", "fullsim_50ns": "75X_mcRun2_startup_v1"},
0106 "CMSSW_7_5_0": {"default": "75X_mcRun2_asymptotic_v1", "fullsim_50ns": "75X_mcRun2_startup_v1"},
0107 "CMSSW_7_5_0_71XGENSIM": {"default": "75X_mcRun2_asymptotic_v1_gs7115", "fullsim_50ns": "75X_mcRun2_startup_v1_gs7115"},
0108 "CMSSW_7_5_1": {"default": "75X_mcRun2_asymptotic_v3", "fullsim_50ns": "75X_mcRun2_startup_v3"},
0109 "CMSSW_7_5_1_71XGENSIM": {"default": "75X_mcRun2_asymptotic_v3_gs7118", "fullsim_50ns": "75X_mcRun2_startup_v3_gs7118"},
0110 "CMSSW_7_5_2": {"default": "75X_mcRun2_asymptotic_v5", "fullsim_50ns": "75X_mcRun2_startup_v4"},
0111 "CMSSW_7_6_0_pre1": {"default": "75X_mcRun2_asymptotic_v1", "fullsim_50ns": "75X_mcRun2_startup_v1"},
0112 "CMSSW_7_6_0_pre2": {"default": "75X_mcRun2_asymptotic_v2", "fullsim_50ns": "75X_mcRun2_startup_v2"},
0113 "CMSSW_7_6_0_pre3": {"default": "75X_mcRun2_asymptotic_v2", "fullsim_50ns": "75X_mcRun2_startup_v2"},
0114 "CMSSW_7_6_0_pre4": {"default": "76X_mcRun2_asymptotic_v1", "fullsim_50ns": "76X_mcRun2_startup_v1"},
0115 "CMSSW_7_6_0_pre5": {"default": "76X_mcRun2_asymptotic_v1", "fullsim_50ns": "76X_mcRun2_startup_v1"},
0116 "CMSSW_7_6_0_pre6": {"default": "76X_mcRun2_asymptotic_v4", "fullsim_50ns": "76X_mcRun2_startup_v4"},
0117 "CMSSW_7_6_0_pre7": {"default": "76X_mcRun2_asymptotic_v5", "fullsim_50ns": "76X_mcRun2_startup_v5", "fastsim": "76X_mcRun2_asymptotic_v5_resub"},
0118 "CMSSW_7_6_0": {"default": "76X_mcRun2_asymptotic_v11", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0119 "CMSSW_7_6_0_71XGENSIM": {"default": "76X_mcRun2_asymptotic_v11_gs7120p2rlBS", "fullsim_50ns": "76X_mcRun2_startup_v11_gs7120p2rlBS"},
0120 "CMSSW_7_6_2": {"default": "76X_mcRun2_asymptotic_v12", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0121 "CMSSW_7_6_3_patch2_0T": {"default": "76X_mcRun2_0T_v1_0Tv1GT"},
0122 "CMSSW_8_0_0_pre1": {"default": "76X_mcRun2_asymptotic_v11", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0123 "CMSSW_8_0_0_pre2": {"default": "76X_mcRun2_asymptotic_v12", "fullsim_50ns": "76X_mcRun2_startup_v11"},
0124 "CMSSW_8_0_0_pre2_phase1": {"default": "76X_upgrade2017_design_v7"},
0125 "CMSSW_8_0_0_pre2_phase1_rereco": {"default": "76X_upgrade2017_design_v7_rereco"},
0126 "CMSSW_8_0_0_pre3_phase1": {"default": "76X_upgrade2017_design_v8"},
0127 "CMSSW_8_0_0_pre3_phase1_pythia8": {"default": "76X_upgrade2017_design_v8_pythia8"},
0128 "CMSSW_8_0_0_pre4": {"default": "76X_mcRun2_asymptotic_v13", "fullsim_50ns": "76X_mcRun2_startup_v12"},
0129 "CMSSW_8_0_0_pre4_phase1": {"default": "76X_upgrade2017_design_v8_UPG17"},
0130 "CMSSW_8_0_0_pre4_phase1_13TeV": {"default": "76X_upgrade2017_design_v8_UPG17"},
0131 "CMSSW_8_0_0_pre4_ecal15fb": {"default": "80X_mcRun2_asymptotic_2016EcalTune_15fb_v0_ecal15fbm1"},
0132 "CMSSW_8_0_0_pre4_ecal30fb": {"default": "80X_mcRun2_asymptotic_2016EcalTune_30fb_v0_ecal30fbm1"},
0133 "CMSSW_8_0_0_pre5": {"default": "80X_mcRun2_asymptotic_v1", "fullsim_50ns": "80X_mcRun2_startup_v1"},
0134 "CMSSW_8_0_0_pre5_phase1": {"default": "80X_upgrade2017_design_v1_UPG17"},
0135 "CMSSW_8_0_0_pre6": {"default": "80X_mcRun2_asymptotic_v4"},
0136 "CMSSW_8_0_0_pre6_phase1": {"default": "80X_upgrade2017_design_v3_UPG17"},
0137 "CMSSW_8_0_0_pre6_MT": {"default": "80X_mcRun2_asymptotic_v4_multiCoreResub"},
0138 "CMSSW_8_0_0": {"default": "80X_mcRun2_asymptotic_v4"},
0139 "CMSSW_8_0_0_patch1_phase1": {"default": "80X_upgrade2017_design_v4_UPG17"},
0140 "CMSSW_8_0_0_patch1_phase1_rereco": {"default": "80X_upgrade2017_design_v4_UPG17_rereco"},
0141 "CMSSW_8_0_0_patch2": {"default": "80X_mcRun2_asymptotic_v5_refGT"},
0142 "CMSSW_8_0_0_patch2_pixDynIneff": {"default": "80X_mcRun2_asymptotic_v5_2016PixDynIneff_targetGT"},
0143
0144
0145 "CMSSW_8_0_0_patch2": {"default": "80X_mcRun2_asymptotic_v5_refGT_resub"},
0146 "CMSSW_8_0_0_patch2_pixDynIneff": {"default": "80X_mcRun2_asymptotic_v5_2016PixDynIneff_targetGT_resub"},
0147 "CMSSW_8_0_1": {"default": "80X_mcRun2_asymptotic_v6"},
0148 "CMSSW_8_0_1_71XGENSIM": {"default": "80X_mcRun2_asymptotic_v6_gs7120p2"},
0149 "CMSSW_8_0_1_gcc530": {"default": "80X_mcRun2_asymptotic_v6_gcc530"},
0150 "CMSSW_8_0_3_71XGENSIM": {"default": "80X_mcRun2_asymptotic_2016_v3_gs7120p2NewGTv3"},
0151 "CMSSW_8_0_3_71XGENSIM_hcal": {"default": "80X_mcRun2_asymptotic_2016_v3_gs71xNewGtHcalCust"},
0152 "CMSSW_8_0_3_71XGENSIM_tec": {"default": "80X_mcRun2_asymptotic_SiStripBad_TEC_CL62_for2016_v1_mc_gs7120p2TrkCoolLoop"},
0153 "CMSSW_8_0_5": {"default": "80X_mcRun2_asymptotic_v12_gs7120p2", "fastsim": "80X_mcRun2_asymptotic_v12"},
0154 "CMSSW_8_0_5_pmx": {"default": "80X_mcRun2_asymptotic_v12_gs7120p2_resub", "fastsim": "80X_mcRun2_asymptotic_v12"},
0155 "CMSSW_8_0_10": {"default": "80X_mcRun2_asymptotic_v14"},
0156 "CMSSW_8_0_10_patch1_BS": {"default": "80X_mcRun2_asymptotic_RealisticBS_25ns_13TeV2016_v1_mc_realisticBS2016"},
0157 "CMSSW_8_0_11": {"default": "80X_mcRun2_asymptotic_v14"},
0158 "CMSSW_8_0_15": {"default": "80X_mcRun2_asymptotic_v16_gs7120p2", "fastsim": "80X_mcRun2_asymptotic_v16"},
0159 "CMSSW_8_0_16": {"default": "80X_mcRun2_asymptotic_v16_gs7120p2", "fastsim": "80X_mcRun2_asymptotic_v16"},
0160 "CMSSW_8_0_16_Tranche4GT": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_gs7120p2_Tranch4GT",
0161 "fastsim": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_Tranch4GT", "RelValTTbar": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_Tr4GT_resub"}, "fastsim_25ns": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_Tranch4GT"},
0162 "CMSSW_8_0_16_Tranche4GT_v2": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v2_Tr4GT_v2"},
0163 "CMSSW_8_0_16_Tranche4GT_pmx": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_gs7120p2_Tranch4GT", "fastsim": "80X_mcRun2_asymptotic_2016_TrancheIV_v0_resub"},
0164 "CMSSW_8_0_19_Tranche4GT_v2": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v2_Tr4GT_v2"},
0165 "CMSSW_8_0_20_Tranche4GT": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v4_Tr4GT_v4"},
0166 "CMSSW_8_0_21_Tranche4GT": {"default": "80X_mcRun2_asymptotic_2016_TrancheIV_v6_Tr4GT_v6"},
0167 "CMSSW_8_1_0_pre1": {"default": "80X_mcRun2_asymptotic_v6"},
0168 "CMSSW_8_1_0_pre1_phase1": {"default": "80X_upgrade2017_design_v4_UPG17", "fullsim_25ns": "80X_upgrade2017_design_v4_UPG17PU35"},
0169 "CMSSW_8_1_0_pre2": {"default": "80X_mcRun2_asymptotic_v10_gs810pre2", "fastsim": "80X_mcRun2_asymptotic_v10"},
0170 "CMSSW_8_1_0_pre2_phase1": {"default": "80X_upgrade2017_design_v9_UPG17designGT", "fullsim_25ns": "80X_upgrade2017_design_v9_UPG17PU35designGT"},
0171 "CMSSW_8_1_0_pre2_phase1_realGT": {"default": "80X_upgrade2017_realistic_v1_UPG17realGT", "fullsim_25ns": "80X_upgrade2017_realistic_v1_UPG17PU35realGT"},
0172 "CMSSW_8_1_0_pre3": {"default": "80X_mcRun2_asymptotic_v12"},
0173 "CMSSW_8_1_0_pre3_phase1": {"default": "80X_upgrade2017_realistic_v3_UPG17", "fullsim_25ns": "80X_upgrade2017_realistic_v3_UPG17PU35"},
0174 "CMSSW_8_1_0_pre4": {"default": "80X_mcRun2_asymptotic_v13"},
0175 "CMSSW_8_1_0_pre4_phase1": {"default": "80X_upgrade2017_realistic_v4_UPG17", "fullsim_25ns": "80X_upgrade2017_realistic_v4_UPG17PU35"},
0176 "CMSSW_8_1_0_pre5": {"default": "80X_mcRun2_asymptotic_v13"},
0177 "CMSSW_8_1_0_pre5_phase1": {"default": "80X_upgrade2017_realistic_v4_resubUPG17", "fullsim_25ns": "80X_upgrade2017_realistic_v4_resubUPG17PU35"},
0178 "CMSSW_8_1_0_pre6": {"default": "80X_mcRun2_asymptotic_v14"},
0179 "CMSSW_8_1_0_pre6_phase1": {"default": "81X_upgrade2017_realistic_v0_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v0_UPG17PU35"},
0180 "CMSSW_8_1_0_pre7": {"default": "81X_mcRun2_asymptotic_v0"},
0181 "CMSSW_8_1_0_pre7_phase1": {"default": "81X_upgrade2017_realistic_v2_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v2_UPG17PU35"},
0182 "CMSSW_8_1_0_pre7_phase1_newGT": {"default": "81X_upgrade2017_realistic_v3_UPG17newGT", "fullsim_25ns": "81X_upgrade2017_realistic_v3_UPG17PU35newGTresub"},
0183 "CMSSW_8_1_0_pre7_phase2": {"2023GReco": "81X_mcRun2_asymptotic_v0_2023GReco", "2023GRecoPU35": "", "2023GRecoPU140": "81X_mcRun2_asymptotic_v0_2023GRecoPU140resubmit2", "2023GRecoPU200": "81X_mcRun2_asymptotic_v0_2023GRecoPU200resubmit2",
0184 "2023tilted": "81X_mcRun2_asymptotic_v0_2023tilted", "2023tiltedPU35": "", "2023tiltedPU140": "81X_mcRun2_asymptotic_v0_2023tiltedPU140resubmit2", "2023tiltedPU200": "81X_mcRun2_asymptotic_v0_2023tiltedPU200resubmit2"},
0185 "CMSSW_8_1_0_pre8": {"default": "81X_mcRun2_asymptotic_v1"},
0186 "CMSSW_8_1_0_pre8_phase1": {"default": "81X_upgrade2017_realistic_v3_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v3_UPG17PU35"},
0187 "CMSSW_8_1_0_pre8_phase1_newGT": {"default": "81X_upgrade2017_realistic_v4_UPG17newGT", "fullsim_25ns": "81X_upgrade2017_realistic_v4_UPG17PU35newGT"},
0188 "CMSSW_8_1_0_pre8_phase1_newGT2": {"default": "81X_upgrade2017_realistic_v5_UPG17newGTset2", "fullsim_25ns": "81X_upgrade2017_realistic_v5_UPG17PU35newGTset2"},
0189 "CMSSW_8_1_0_pre8_phase2": {"2023GReco": "81X_mcRun2_asymptotic_v1_resub2023GReco", "2023GRecoPU35": "81X_mcRun2_asymptotic_v1_resub2023GRecoPU35", "2023GRecoPU140": "81X_mcRun2_asymptotic_v1_resub2023GRecoPU140", "2023GRecoPU200": "81X_mcRun2_asymptotic_v1_resub2023GRecoPU200",
0190 "2023tilted": "81X_mcRun2_asymptotic_v1_2023tilted", "2023tiltedPU35": "81X_mcRun2_asymptotic_v1_2023tiltedPU", "2023tiltedPU140": "81X_mcRun2_asymptotic_v1_2023tiltedPU140", "2023tiltedPU200": "81X_mcRun2_asymptotic_v1_2023tiltedPU200"},
0191 "CMSSW_8_1_0_pre9": {"default": "81X_mcRun2_asymptotic_v2"},
0192 "CMSSW_8_1_0_pre9_Geant4102": {"default": "81X_mcRun2_asymptotic_v2"},
0193 "CMSSW_8_1_0_pre9_phase1": {"default": "81X_upgrade2017_realistic_v5_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v5_UPG17PU35"},
0194 "CMSSW_8_1_0_pre9_phase1_newGT": {"default": "81X_upgrade2017_realistic_v6_UPG17newGT", "fullsim_25ns": "81X_upgrade2017_realistic_v6_UPG17PU35newGT"},
0195 "CMSSW_8_1_0_pre10": {"default": "81X_mcRun2_asymptotic_v5_recycle", "fullsim_25ns": "81X_mcRun2_asymptotic_v5_resub", "fastsim": "81X_mcRun2_asymptotic_v5"},
0196 "CMSSW_8_1_0_pre10_pmx": {"default": "81X_mcRun2_asymptotic_v5"},
0197 "CMSSW_8_1_0_pre10_phase1": {"default": "81X_upgrade2017_realistic_v9_UPG17resub", "fullsim_25ns": "81X_upgrade2017_realistic_v9_UPG17PU35resub"},
0198 "CMSSW_8_1_0_pre11": {"default": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55", "fullsim_25ns": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55_resub", "fastsim_25ns": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55_resub2"},
0199 "CMSSW_8_1_0_pre11_pmx": {"default": "81X_mcRun2_asymptotic_Candidate_2016_08_30_11_31_55"},
0200 "CMSSW_8_1_0_pre11_phase1": {"default": "81X_upgrade2017_realistic_v9_UPG17", "fullsim_25ns": "81X_upgrade2017_realistic_v9_UPG17PU35"},
0201 "CMSSW_8_1_0_pre12": {"default": "81X_mcRun2_asymptotic_v8", "fullsim_25ns": "81X_mcRun2_asymptotic_v8_resub", "fastsim": "81X_mcRun2_asymptotic_v8_resub", "fastsim_25ns": "81X_mcRun2_asymptotic_v8"},
0202 "CMSSW_8_1_0_pre12_pmx": {"default": "81X_mcRun2_asymptotic_v8_resub", "fastsim_25ns": "81X_mcRun2_asymptotic_v8_rsub"},
0203 "CMSSW_8_1_0_pre12_phase1": {"default": "81X_upgrade2017_realistic_v13"},
0204 "CMSSW_8_1_0_pre12_phase1_newBPix": {"default": "81X_upgrade2017_realistic_newBPix_wAlign_v1_BpixGeom"},
0205 "CMSSW_8_1_0_pre12_phase1_newBPixFPix": {"default": "81X_upgrade2017_realistic_v13_BpixFpixGeom"},
0206 "CMSSW_8_1_0_pre12_phase1_newBPixFPixHCAL": {"default": "81X_upgrade2017_HCALdev_v2_BpixFpixHcalGeom"},
0207 "CMSSW_8_1_0_pre12_phase1_newHCAL": {"default": "81X_upgrade2017_HCALdev_v2_HcalGeom"},
0208 "CMSSW_8_1_0_pre12_phase1_newBPixHCAL": {"default": "81X_upgrade2017_HCALdev_v2_NewBPix_BpixHcalGeom"},
0209 "CMSSW_8_1_0_pre15": {"default": "81X_mcRun2_asymptotic_v11"},
0210 "CMSSW_8_1_0_pre15_HIP": {"default": "81X_mcRun2_asymptotic_v11_hip"},
0211 "CMSSW_8_1_0_pre15_PU": {"default": "81X_mcRun2_asymptotic_v11_M17"},
0212 "CMSSW_8_1_0_pre15_PU_HIP": {"default": "81X_mcRun2_asymptotic_v11_hipM17"},
0213 "CMSSW_8_1_0_pre15_phase1": {"default": "81X_upgrade2017_realistic_v17_BpixFpixHcalGeom",
0214 "Design": "81X_upgrade2017_design_IdealBS_v1_2017design", "Design_fullsim_25ns": "81X_upgrade2017_design_IdealBS_v1_design"},
0215 "CMSSW_8_1_0_pre16": {"default": "81X_mcRun2_asymptotic_v11"},
0216 "CMSSW_8_1_0_pre16_phase1": {"default": "81X_upgrade2017_realistic_v22", "Design": "81X_upgrade2017_design_IdealBS_v6"},
0217 "CMSSW_8_1_0": {"default": "81X_mcRun2_asymptotic_v12"},
0218 "CMSSW_8_1_0_phase1": {"default": "81X_upgrade2017_realistic_v26_HLT2017"},
0219 "CMSSW_9_0_0_pre1": {"default": "90X_mcRun2_asymptotic_v0"},
0220 "CMSSW_9_0_0_pre1_phase1": {"default": "90X_upgrade2017_realistic_v0",
0221 "Design": "90X_upgrade2017_design_IdealBS_v0", "Design_fullsim_25ns": "90X_upgrade2017_design_IdealBS_v0_resub"},
0222 "CMSSW_9_0_0_pre2": {"default": "90X_mcRun2_asymptotic_v0"},
0223 "CMSSW_9_0_0_pre2_ROOT6": {"default": "90X_mcRun2_asymptotic_v0"},
0224 "CMSSW_9_0_0_pre2_phase1": {"default": "90X_upgrade2017_realistic_v0",
0225 "Design": "90X_upgrade2017_design_IdealBS_v0", "Design_fullsim_25ns": "90X_upgrade2017_design_IdealBS_v0_resub"},
0226 "CMSSW_9_0_0_pre4": {"default": "90X_mcRun2_asymptotic_v1", "fullsim_25ns": "90X_mcRun2_asymptotic_v1_resub"},
0227 "CMSSW_9_0_0_pre4_phase1": {"default": "90X_upgrade2017_realistic_v6", "fullsim_25ns_PU50": "Nonexistent",
0228 "Design": "90X_upgrade2017_design_IdealBS_v6"},
0229 "CMSSW_9_0_0_pre4_GS": {"default": "90X_mcRun2_asymptotic_v1_GSval", "fullsim_25ns": "90X_mcRun2_asymptotic_v1_GSval"},
0230 "CMSSW_9_0_0_pre4_phase1_ecalsrb5": {"default": "90X_upgrade2017_realistic_v6_B5"},
0231 "CMSSW_9_0_0_pre4_phase1_ecalsrc1": {"default": "90X_upgrade2017_realistic_v6_C1"},
0232 "CMSSW_9_0_0_pre4_phase1_ecalsrd7": {"default": "90X_upgrade2017_realistic_v6_D7"},
0233 "CMSSW_9_0_0_pre5": {"default": "90X_mcRun2_asymptotic_v4"},
0234 "CMSSW_9_0_0_pre5_pmx": {"default": "90X_mcRun2_asymptotic_v4", "fastsim_25ns": "90X_mcRun2_asymptotic_v4_resub"},
0235 "CMSSW_9_0_0_pre5_phase1": {"default": "90X_upgrade2017_realistic_v15",
0236 "fullsim_25ns_PU35": "90X_upgrade2017_realistic_v15_resub", "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v15_PU50",
0237 "Design": "90X_upgrade2017_design_IdealBS_v15"},
0238 "CMSSW_9_0_0_pre6": {"default": "90X_mcRun2_asymptotic_v4"},
0239 "CMSSW_9_0_0_pre6_phase1": {"default": "90X_upgrade2017_realistic_v15",
0240 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v15_PU50",
0241 "Design": "90X_upgrade2017_design_IdealBS_v15"},
0242 "CMSSW_9_0_0": {"default": "90X_mcRun2_asymptotic_v5"},
0243 "CMSSW_9_0_0_phase1": {"default": "90X_upgrade2017_realistic_v20_resub",
0244 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50_resub",
0245 "Design": "90X_upgrade2017_design_IdealBS_v19_resub"},
0246 "CMSSW_9_0_0_gcc630": {"default": "90X_mcRun2_asymptotic_v5_gcc630"},
0247 "CMSSW_9_0_0_phase1_gcc630": {"default": "90X_upgrade2017_realistic_v20_gcc630"},
0248 "CMSSW_9_0_0_cc7": {"default": "90X_mcRun2_asymptotic_v5_cc7"},
0249 "CMSSW_9_0_0_phase1_cc7": {"default": "90X_upgrade2017_realistic_v20_cc7_rsb"},
0250 "CMSSW_9_0_2_phase1": {"default": "90X_upgrade2017_realistic_v20",
0251 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50",
0252 "Design": "90X_upgrade2017_design_IdealBS_v19"},
0253 "CMSSW_9_1_0_pre1": {"default": "90X_mcRun2_asymptotic_v5"},
0254 "CMSSW_9_1_0_pre1_phase1": {"default": "90X_upgrade2017_realistic_v20",
0255 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50",
0256 "Design": "90X_upgrade2017_design_IdealBS_v19_resub"},
0257 "CMSSW_9_1_0_pre2": {"default": "90X_mcRun2_asymptotic_v5"},
0258 "CMSSW_9_1_0_pre2_phase1": {"default": "90X_upgrade2017_realistic_v20",
0259 "fullsim_25ns_PU50": "90X_upgrade2017_realistic_v20_PU50",
0260 "Design": "90X_upgrade2017_design_IdealBS_v19"},
0261 "CMSSW_9_1_0_pre3": {"default": "91X_mcRun2_asymptotic_v2"},
0262 "CMSSW_9_1_0_pre3_phase1": {"default": "91X_upgrade2017_realistic_v3",
0263 "fullsim_25ns_PU50": "91X_upgrade2017_realistic_v3_PU50_resub",
0264 "Design": "91X_upgrade2017_design_IdealBS_v3"},
0265 "CMSSW_9_2_0": {"default": "91X_mcRun2_asymptotic_v3"},
0266 "CMSSW_9_2_0_phase1": {"default": "91X_upgrade2017_realistic_v5",
0267 "fullsim_25ns_PU50": "91X_upgrade2017_realistic_v5_PU50",
0268 "Design": "91X_upgrade2017_design_IdealBS_v5"},
0269 "CMSSW_9_2_0_phase1_PXmap": {"default": "91X_upgrade2017_realistic_v5_pixel_ideal_PXgeom"},
0270 "CMSSW_9_2_1_phase1": {"default": "92X_upgrade2017_realistic_v1",
0271 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v1_PU50",
0272 "Design": "92X_upgrade2017_design_IdealBS_v1"},
0273 "CMSSW_9_2_2": {"default": "91X_mcRun2_asymptotic_v3"},
0274 "CMSSW_9_2_2_phase1": {"default": "92X_upgrade2017_realistic_v1",
0275 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v1_highPU_AVE50",
0276 "Design": "92X_upgrade2017_design_IdealBS_v1"},
0277 "CMSSW_9_2_4_run1": {"default": "91X_mcRun1_realistic_v2"},
0278 "CMSSW_9_2_4": {"default": "91X_mcRun2_asymptotic_v3"},
0279 "CMSSW_9_2_7_phase1": {"default": "92X_upgrade2017_realistic_v7"},
0280 "CMSSW_9_3_0_pre1": {"default": "92X_mcRun2_asymptotic_v2"},
0281 "CMSSW_9_3_0_pre1_phase1": {"default": "92X_upgrade2017_realistic_v7",
0282 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v7_highPU_AVE50",
0283 "Design": "92X_upgrade2017_design_IdealBS_v7"},
0284 "CMSSW_9_3_0_pre1_run1": {"default": "92X_mcRun1_realistic_v2"},
0285 "CMSSW_9_3_0_pre2": {"default": "92X_mcRun2_asymptotic_v2"},
0286 "CMSSW_9_3_0_pre2_phase1": {"default": "92X_upgrade2017_realistic_v7",
0287 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v7_highPU_AVE50_resub",
0288 "Design": "92X_upgrade2017_design_IdealBS_v7"},
0289 "CMSSW_9_3_0_pre3": {"default": "92X_mcRun2_asymptotic_v2"},
0290 "CMSSW_9_3_0_pre3_phase1": {"default": "92X_upgrade2017_realistic_v10_resub",
0291 "fullsim_25ns_PU50": "92X_upgrade2017_realistic_v10_highPU_AVE50_resub",
0292 "Design": "92X_upgrade2017_design_IdealBS_v10_resub"},
0293 "CMSSW_9_3_0_pre3_phase1_pmx": {"default": "92X_upgrade2017_realistic_v10_resub2"},
0294 "CMSSW_9_3_0_pre4": {"default": "93X_mcRun2_asymptotic_v0"},
0295 "CMSSW_9_3_0_pre4_phase1": {"default": "93X_mc2017_realistic_v1",
0296 "fullsim_25ns_PU50": "93X_mc2017_realistic_v1_highPU_AVE50",
0297 "Design": "93X_mc2017_design_IdealBS_v1"},
0298 "CMSSW_9_3_0_pre5": {"default": "93X_mcRun2_asymptotic_v0"},
0299 "CMSSW_9_3_0_pre5_phase1": {"default": "93X_mc2017_realistic_v2",
0300 "fullsim_25ns_PU50": "93X_mc2017_realistic_v2_highPU_AVE50",
0301 "Design": "93X_mc2017_design_IdealBS_v2"},
0302 "CMSSW_9_4_0_pre1": {"default": "93X_mcRun2_asymptotic_v1"},
0303 "CMSSW_9_4_0_pre1_phase1": {"default": "93X_mc2017_realistic_v3",
0304 "fullsim_25ns_PU50": "93X_mc2017_realistic_v3_highPU_AVE50",
0305 "Design": "93X_mc2017_design_IdealBS_v3"},
0306 "CMSSW_9_4_0_pre2": {"default": "93X_mcRun2_asymptotic_v2"},
0307 "CMSSW_9_4_0_pre2_phase1": {"default": "94X_mc2017_realistic_v1",
0308 "fullsim_25ns_PU50": "94X_mc2017_realistic_v1_highPU_AVE50",
0309 "Design": "94X_mc2017_design_IdealBS_v0"},
0310 "CMSSW_9_4_0_pre3": {"default": "94X_mcRun2_asymptotic_v0"},
0311 "CMSSW_9_4_0_pre3_phase1": {"default": "94X_mc2017_realistic_v4",
0312 "fullsim_25ns_PU50": "94X_mc2017_realistic_v4_highPU_AVE50",
0313 "Design": "94X_mc2017_design_IdealBS_v4"},
0314 "CMSSW_9_4_0": {"default": "94X_mcRun2_asymptotic_v0"},
0315 "CMSSW_9_4_0_phase1": {"default": "94X_mc2017_realistic_v10",
0316 "fullsim_25ns_PU50": "94X_mc2017_realistic_v10_highPU_AVE50",
0317 "Design": "94X_mc2017_design_IdealBS_v5"},
0318 "CMSSW_10_0_0_pre1": {"default": "94X_mcRun2_asymptotic_v0"},
0319 "CMSSW_10_0_0_pre1_phase1": {"default": "94X_mc2017_realistic_v10",
0320 "fullsim_25ns_PU50": "94X_mc2017_realistic_v10_highPU_AVE50",
0321 "Design": "94X_mc2017_design_IdealBS_v5"},
0322 "CMSSW_10_0_0_pre2": {"default": "100X_mcRun2_asymptotic_v2"},
0323 "CMSSW_10_0_0_pre2_2017": {"default": "100X_mc2017_realistic_v1",
0324 "fullsim_25ns": "100X_mc2017_realistic_v1_resub",
0325 "fullsim_25ns_PU50": "100X_mc2017_realistic_v1_highPU_AVE50",
0326 "Design": "100X_mc2017_design_IdealBS_v1",
0327 "Design_fullsim_25ns_PU50": "Does_not_exist"},
0328 "CMSSW_10_0_0_pre2_2017_pmx": {"default": "100X_mc2017_realistic_v1"},
0329 "CMSSW_10_0_0_pre2_2018": {"default": "100X_upgrade2018_realistic_v1",
0330 "fullsim_25ns": "100X_upgrade2018_realistic_v1_resub",
0331 "Design": "100X_upgrade2018_design_IdealBS_v1",
0332 "Design_fullsim_25ns": "100X_upgrade2018_design_IdealBS_v1_resub"},
0333 "CMSSW_10_0_0_pre3": {"default": "100X_mcRun2_asymptotic_v2"},
0334 "CMSSW_10_0_0_pre3_2017": {"default": "100X_mc2017_realistic_v1_mahiOFF",
0335 "fullsim_25ns_PU50": "100X_mc2017_realistic_v1_highPU_AVE50_mahiOFF",
0336 "Design": "100X_mc2017_design_IdealBS_v1_mahiOFF"},
0337 "CMSSW_10_0_0_pre3_2018": {"default": "100X_upgrade2018_realistic_v4_mahiOFF",
0338 "Design": "100X_upgrade2018_design_IdealBS_v3_mahiOFF"},
0339 "CMSSW_10_0_0_pre3_2018_pmx": {"default": "100X_upgrade2018_realistic_v4",
0340 "Design": "100X_upgrade2018_design_IdealBS_v3"},
0341 "CMSSW_10_0_0_pre3_2017_mahi": {"default": "100X_mc2017_realistic_v1_mahiON",
0342 "fullsim_25ns_PU50": "100X_mc2017_realistic_v1_highPU_AVE50_mahiON",
0343 "Design": "100X_mc2017_design_IdealBS_v1_mahiON"},
0344 "CMSSW_10_0_0_pre3_2018_mahi": {"default": "100X_upgrade2018_realistic_v4_mahiON",
0345 "Design": "100X_upgrade2018_design_IdealBS_v3_mahiON"},
0346 "CMSSW_10_0_0_pre3_GEANT4_2018_mahi": {"default": "100X_upgrade2018_realistic_v4_mahiON"},
0347 "CMSSW_10_0_0_pre3_G4VecGeom2_2018": {"default": "100X_upgrade2018_realistic_v4"},
0348 }
0349
0350 _releasePostfixes = ["_AlcaCSA14", "_PHYS14", "_TEST", "_v2", "_v3", "_pmx", "_Fall14DR", "_FIXGT", "_PU", "_PXbest", "_PXworst", "_hcal", "_tec", "_71XGENSIM", "_73XGENSIM", "_BS", "_GenSim_7113", "_extended",
0351 "_25ns_asymptotic", "_50ns_startup", "_50ns_asympref", "_50ns_asymptotic", "_minimal", "_0T", "_unsch", "_noCCC", "_MT", "_GS", "_rereco", "_pythia8", "_13TeV", "_realGT", "_newGT2", "_newGT", "_phase1", "_phase2", "_2017", "_2018", "_ecal15fb", "_ecal30fb", "_ecalsrb5", "_ecalsrc1", "_ecalsrd7", "_pixDynIneff", "_PXmap", "_gcc530", "_gcc630", "_cc7", "_Tranche4GT", "_newBPixFPixHCAL", "_newBPixFPix", "_newBPixHCAL", "_newBPix", "_newHCAL", "_HIP", "_run1", "_mahi"]
0352 def _stripRelease(release):
0353 for pf in _releasePostfixes:
0354 if pf in release:
0355 return _stripRelease(release.replace(pf, ""))
0356 return release
0357
0358
0359 def _getGlobalTag(sample, release):
0360 """Get a GlobalTag.
0361
0362 Arguments:
0363 sample -- Sample object
0364 release -- CMSSW release string
0365 """
0366 if not release in _globalTags:
0367 print("Release %s not found from globaltag map in validation.py" % release)
0368 sys.exit(1)
0369 gtmap = _globalTags[release]
0370 selectedGT = None
0371 if sample.hasOverrideGlobalTag():
0372 ogt = sample.overrideGlobalTag()
0373 if release in ogt:
0374 gtmap = _globalTags[ogt[release]]
0375 scenario = ""
0376 if sample.hasScenario():
0377 scenario = sample.scenario()
0378 sims = []
0379 if sample.fullsim():
0380 if sample.pileupEnabled():
0381 sim = "fullsim_"+sample.pileupType()
0382 sims.extend([
0383 sim+"_PU%d"%sample.pileupNumber(),
0384 sim
0385 ])
0386 elif sample.fastsim():
0387 sim = "fastsim"
0388 if sample.pileupEnabled():
0389 sim += "_"+sample.pileupType()
0390 sims.append(sim+"_PU%d"%sample.pileupNumber())
0391 sims.append(sim)
0392
0393 selectedGT = None
0394
0395 if scenario != "":
0396 for sim in sims:
0397 selectedGT = gtmap.get(scenario+"_"+sim, None)
0398 if selectedGT is not None:
0399 break
0400
0401 if selectedGT is None:
0402 selectedGT = gtmap.get(scenario, None)
0403
0404 if selectedGT is None:
0405 for sim in sims:
0406 selectedGT = gtmap.get(sim, None)
0407 if selectedGT is not None:
0408 break
0409
0410 if selectedGT is None:
0411 selectedGT = gtmap["default"]
0412
0413 if isinstance(selectedGT, dict):
0414 return selectedGT.get(sample.name(), selectedGT["default"])
0415 else:
0416 return selectedGT
0417
0418
0419 _relvalUrls = {
0420 "6_2_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_6_2_x/",
0421 "7_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_0_x/",
0422 "7_1_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_1_x/",
0423 "7_2_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_2_x/",
0424 "7_3_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_3_x/",
0425 "7_4_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_4_x/",
0426 "7_5_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_5_x/",
0427 "7_6_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_7_6_x/",
0428 "8_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_8_0_x/",
0429 "8_1_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_8_1_x/",
0430 "9_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_0_x/",
0431 "9_1_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_1_x/",
0432 "9_2_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_2_x/",
0433 "9_3_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_3_x/",
0434 "9_4_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_9_4_x/",
0435 "10_0_X": "https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/CMSSW_10_0_x/",
0436 }
0437
0438 _doElectronSamples = [
0439 "RelValTTbar",
0440 "RelValSingleElectronPt35",
0441 "RelValSingleElectronPt10",
0442 ]
0443 _doConversionSamples = [
0444 "RelValTTbar",
0445 "RelValH125GGgluonfusion",
0446 ]
0447 _doBHadronSamples = [
0448 "RelValTTbar"
0449 ]
0450
0451 def _getRelValUrl(release):
0452 """Get RelVal download URL for a given release."""
0453 version_re = re.compile("CMSSW_(?P<X>\\d+)_(?P<Y>\\d+)")
0454 m = version_re.search(release)
0455 if not m:
0456 raise Exception("Regex %s does not match to release version %s" % (version_re.pattern, release))
0457 version = "%s_%s_X" % (m.group("X"), m.group("Y"))
0458 if not version in _relvalUrls:
0459 print("No RelVal URL for version %s, please update _relvalUrls" % version)
0460 sys.exit(1)
0461 return _relvalUrls[version]
0462
0463 def _processPlotsForSample(plotterFolder, sample):
0464 if plotterFolder.onlyForPileup() and not sample.pileupEnabled():
0465 return False
0466 if plotterFolder.onlyForElectron() and not sample.doElectron():
0467 return False
0468 if plotterFolder.onlyForConversion() and not sample.doConversion():
0469 return False
0470 if plotterFolder.onlyForBHadron() and not sample.doBHadron():
0471 return False
0472 return True
0473
0474 class Sample:
0475 """Represents a RelVal sample."""
0476 def __init__(self, sample, append=None, midfix=None, putype=None, punum=0,
0477 fastsim=False, fastsimCorrespondingFullsimPileup=None,
0478 doElectron=None, doConversion=None, doBHadron=None,
0479 version="v1", dqmVersion="0001", scenario=None, overrideGlobalTag=None, appendGlobalTag=""):
0480 """Constructor.
0481
0482 Arguments:
0483 sample -- String for name of the sample
0484
0485 Keyword arguments
0486 append -- String for a variable name within the DWM file names, to be directly appended to sample name (e.g. "HS"; default None)
0487 midfix -- String for a variable name within the DQM file names, to be appended after underscore to "sample name+append" (e.g. "13", "UP15"; default None)
0488 putype -- String for pileup type (e.g. "25ns"/"50ns" for FullSim, "AVE20" for FastSim; default None)
0489 punum -- String for amount of pileup (default None)
0490 fastsim -- Bool indicating the FastSim status (default False)
0491 fastsimCorrespondingFullSimPileup -- String indicating what is the FullSim pileup sample corresponding this FastSim sample. Must be set if fastsim=True and putype!=None (default None)
0492 doElectron -- Bool specifying if electron-specific plots should be produced (default depends on sample)
0493 doConversion -- Bool specifying if conversion-specific plots should be produced (default depends on sample)
0494 doBHadron -- Bool specifying if B-hadron-specific plots should be produced (default depends on sample)
0495 version -- String for dataset/DQM file version (default "v1")
0496 scenario -- Geometry scenario for upgrade samples (default None)
0497 overrideGlobalTag -- GlobalTag obtained from release information (in the form of {"release": "actualRelease"}; default None)
0498 appendGlobalTag -- String to append to GlobalTag (intended for one-time hacks; default "")
0499 """
0500 self._sample = sample
0501 self._append = append
0502 self._midfix = midfix
0503 self._putype = putype
0504 self._punum = punum
0505 self._fastsim = fastsim
0506 self._fastsimCorrespondingFullsimPileup = fastsimCorrespondingFullsimPileup
0507 self._version = version
0508 self._dqmVersion = dqmVersion
0509 self._scenario = scenario
0510 self._overrideGlobalTag = overrideGlobalTag
0511 self._appendGlobalTag = appendGlobalTag
0512
0513 if doElectron is not None:
0514 self._doElectron = doElectron
0515 else:
0516 self._doElectron = (sample in _doElectronSamples)
0517 if doConversion is not None:
0518 self._doConversion = doConversion
0519 else:
0520 self._doConversion = (sample in _doConversionSamples)
0521 if doBHadron is not None:
0522 self._doBHadron = doBHadron
0523 else:
0524 self._doBHadron = (sample in _doBHadronSamples)
0525
0526 if self._fastsim and self.hasPileup() and self._fastsimCorrespondingFullsimPileup is None:
0527 self._fastsimCorrespondingFullsimPileup = self._putype
0528
0529 def digest(self):
0530 """Return a tuple uniquely identifying the sample, to be used e.g. as a key to dict"""
0531 return (self.name(), self.pileupNumber(), self.pileupType(), self.scenario(), self.fastsim())
0532
0533 def sample(self):
0534 """Get the sample name"""
0535 return self._sample
0536
0537 def name(self):
0538 """Get the sample name"""
0539 return self._sample
0540
0541 def label(self):
0542 return self._sample
0543
0544 def hasPileup(self):
0545 """Return True if sample has pileup (for HTML generation)"""
0546 return self._putype is not None
0547
0548 def pileupEnabled(self):
0549 """Return True if pileup plots are enabled (for plot generation)"""
0550 return self.hasPileup()
0551
0552 def pileup(self):
0553 """Return "PU"/"noPU" corresponding the pileup status"""
0554 if self.hasPileup():
0555 return "PU"
0556 else:
0557 return "noPU"
0558
0559 def pileupType(self, release=None):
0560 """Return the pileup type"""
0561 if isinstance(self._putype, dict):
0562 return self._putype.get(release, self._putype["default"])
0563 else:
0564 return self._putype
0565
0566 def pileupNumber(self):
0567 return self._punum
0568
0569 def doElectron(self):
0570 return self._doElectron
0571
0572 def doConversion(self):
0573 return self._doConversion
0574
0575 def doBHadron(self):
0576 return self._doBHadron
0577
0578 def version(self, release=None):
0579 if isinstance(self._version, dict):
0580 return self._version.get(release, self._version["default"])
0581 else:
0582 return self._version
0583
0584 def hasScenario(self):
0585 return self._scenario is not None
0586
0587 def scenario(self):
0588 return self._scenario
0589
0590 def hasOverrideGlobalTag(self):
0591 return self._overrideGlobalTag is not None
0592
0593 def overrideGlobalTag(self):
0594 return self._overrideGlobalTag
0595
0596 def fastsim(self):
0597 """Return True for FastSim sample"""
0598 return self._fastsim
0599
0600 def fullsim(self):
0601 """Return True for FullSim sample"""
0602 return not self._fastsim
0603
0604 def fastsimCorrespondingFullsimPileup(self):
0605 return self._fastsimCorrespondingFullsimPileup
0606
0607 def dirname(self, newRepository, newRelease, newSelection):
0608 """Return the output directory name
0609
0610 Arguments:
0611 newRepository -- String for base directory for output files
0612 newRelease -- String for CMSSW release
0613 newSelection -- String for histogram selection
0614 """
0615 pileup = ""
0616 if self.hasPileup() and not self._fastsim:
0617 pileup = "_"+self._putype
0618 return "{newRepository}/{newRelease}/{newSelection}{pileup}/{sample}".format(
0619 newRepository=newRepository, newRelease=newRelease, newSelection=newSelection,
0620 pileup=pileup, sample=sample)
0621
0622 def filename(self, newRelease):
0623 """Return the DQM file name
0624
0625 Arguments:
0626 newRelease -- String for CMSSW release
0627 """
0628 pileup = ""
0629 fastsim = ""
0630 midfix = ""
0631 sample = self._sample
0632 if self._append is not None:
0633 midfix += self._append
0634 if self._midfix is not None:
0635 midfix += "_"+self._midfix
0636 if self.hasPileup():
0637 if self._fastsim:
0638
0639
0640
0641
0642
0643 pileup = "PU"+self.pileupType(newRelease)+"_"
0644 else:
0645 pileup = "PU"+self.pileupType(newRelease)+"_"
0646 if self._fastsim:
0647 fastsim = "_FastSim"
0648
0649 globalTag = _getGlobalTag(self, newRelease)
0650
0651 fname = 'DQM_V{dqmVersion}_R000000001__{sample}{midfix}__{newrelease}-{pileup}{globaltag}{appendGlobalTag}{fastsim}-{version}__DQMIO.root'.format(
0652 sample=sample, midfix=midfix, newrelease=_stripRelease(newRelease),
0653 pileup=pileup, globaltag=globalTag, appendGlobalTag=self._appendGlobalTag, fastsim=fastsim,
0654 version=self.version(newRelease), dqmVersion=self._dqmVersion
0655 )
0656
0657 return fname
0658
0659 def datasetpattern(self, newRelease):
0660 """Return the dataset pattern
0661
0662 Arguments:
0663 newRelease -- String for CMSSW release
0664 """
0665 pileup = ""
0666 fastsim = ""
0667 digi = ""
0668 if self.hasPileup():
0669 pileup = "-PU_"
0670 if self._fastsim:
0671 fastsim = "_FastSim-"
0672 digi = "DIGI-"
0673 else:
0674 fastsim = "*"
0675 globalTag = _getGlobalTag(self, newRelease)
0676 return "{sample}/{newrelease}-{pileup}{globaltag}{fastsim}{version}/GEN-SIM-{digi}RECO".format(
0677 sample=self._sample, newrelease=newRelease,
0678 pileup=pileup, globaltag=globalTag, fastsim=fastsim, digi=digi,
0679 version=self.version(newRelease)
0680 )
0681
0682 class Validation:
0683 """Base class for Tracking/Vertex validation."""
0684 def __init__(self, fullsimSamples, fastsimSamples, refRelease, refRepository, newRelease, newRepository, newFileModifier=None, selectionName=""):
0685 """Constructor.
0686
0687 Arguments:
0688 fullsimSamples -- List of Sample objects for FullSim samples (may be empty)
0689 fastsimSamples -- List of Sample objects for FastSim samples (may be empty)
0690 refRelease -- String for reference CMSSW release (can be None for no reference release)
0691 newRepository -- String for directory whete to put new files
0692 newRelease -- CMSSW release to be validated
0693 refRepository -- String for directory where reference root files are
0694 newFileModifier -- If given, a function to modify the names of the new files (function takes a string and returns a string)
0695 selectionName -- If given, use this string as the selection name (appended to GlobalTag for directory names)
0696 """
0697 try:
0698 self._newRelease = os.environ["CMSSW_VERSION"]
0699 except KeyError:
0700 print('Error: CMSSW environment variables are not available.', file=sys.stderr)
0701 print(' Please run cmsenv', file=sys.stderr)
0702 sys.exit()
0703
0704 self._fullsimSamples = fullsimSamples
0705 self._fastsimSamples = fastsimSamples
0706 self._refRelease = refRelease
0707 self._refRepository = refRepository
0708 self._newRelease = newRelease
0709 self._newBaseDir = os.path.join(newRepository, self._newRelease)
0710 self._newFileModifier = newFileModifier
0711 self._selectionName = selectionName
0712
0713 def _getDirectoryName(self, *args, **kwargs):
0714 return None
0715
0716 def _getSelectionName(self, *args, **kwargs):
0717 return self._selectionName
0718
0719 def download(self):
0720 """Download DQM files. Requires grid certificate and asks your password for it."""
0721 filenames = [s.filename(self._newRelease) for s in self._fullsimSamples+self._fastsimSamples]
0722 if self._newFileModifier is not None:
0723 filenames = list(map(self._newFileModifier, filenames))
0724 filenames = [f for f in filenames if not os.path.exists(f)]
0725 if len(filenames) == 0:
0726 print("All files already downloaded")
0727 return
0728
0729 relvalUrl = _getRelValUrl(self._newRelease)
0730 urls = [relvalUrl+f for f in filenames]
0731 certfile = os.path.join(os.environ["HOME"], ".globus", "usercert.pem")
0732 if not os.path.exists(certfile):
0733 print("Certificate file {certfile} does not exist, unable to download RelVal files from {url}".format(certfile=certfile, url=relvalUrl))
0734 sys.exit(1)
0735 keyfile = os.path.join(os.environ["HOME"], ".globus", "userkey.pem")
0736 if not os.path.exists(certfile):
0737 print("Private key file {keyfile} does not exist, unable to download RelVal files from {url}".format(keyfile=keyfile, url=relvalUrl))
0738 sys.exit(1)
0739
0740
0741 cmd = ["curl", "--cert-type", "PEM", "--cert", certfile, "--key", keyfile, "-k"]
0742 for u in urls:
0743 cmd.extend(["-O", u])
0744 print("Downloading %d files from RelVal URL %s:" % (len(filenames), relvalUrl))
0745 print(" "+"\n ".join(filenames))
0746 print("Please provide your private key pass phrase when curl asks it")
0747 ret = subprocess.call(cmd)
0748 if ret != 0:
0749 print("Downloading failed with exit code %d" % ret)
0750 sys.exit(1)
0751
0752
0753 allFine = True
0754 for f in filenames:
0755 p = subprocess.Popen(["file", f], stdout=subprocess.PIPE)
0756 stdout = p.communicate()[0]
0757 if p.returncode != 0:
0758 print("file command failed with exit code %d" % p.returncode)
0759 sys.exit(1)
0760 if not "ROOT" in stdout:
0761 print("File {f} is not ROOT, please check the correct version, GlobalTag etc. from {url}".format(f=f, url=relvalUrl))
0762 allFine = False
0763 if os.path.exists(f):
0764 os.remove(f)
0765 if not allFine:
0766 sys.exit(1)
0767
0768 def createHtmlReport(self):
0769 return html.HtmlReport(self._newRelease, self._newBaseDir)
0770
0771 def doPlots(self, plotter, plotterDrawArgs={}, limitSubFoldersOnlyTo=None, htmlReport=html.HtmlReportDummy(), doFastVsFull=True, doPhase2PU=False):
0772 """Create validation plots.
0773
0774 Arguments:
0775 plotter -- plotting.Plotter object that does the plotting
0776
0777 Keyword arguments:
0778 plotterDrawArgs -- Dictionary for additional arguments to Plotter.draw() (default: {})
0779 limitSubFoldersOnlyTo -- If not None, should be a dictionary from string to an object. The string is the name of a PlotFolder, and the object is PlotFolder-type specific to limit the subfolders to be processed. In general case the object is a list of strings, but e.g. for track iteration plots it is a function taking the algo and quality as parameters.
0780 htmlReport -- Object returned by createHtmlReport(), in case HTML report generation is desired
0781 doFastVsFull -- Do FastSim vs. FullSim comparison? (default: True)
0782 doPhase2PU -- Do Phase2 PU 200 vs. 140 comparison (default: False)
0783 """
0784 self._plotter = plotter
0785 self._plotterDrawArgs = plotterDrawArgs
0786
0787
0788 for sample in self._fullsimSamples+self._fastsimSamples:
0789
0790 harvestedFile = sample.filename(self._newRelease)
0791 if not os.path.exists(harvestedFile):
0792 print("Harvested file %s does not exist!" % harvestedFile)
0793 sys.exit(1)
0794
0795 plotterInstance = plotter.readDirs(harvestedFile)
0796 htmlReport.beginSample(sample)
0797 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
0798 if not _processPlotsForSample(plotterFolder, sample):
0799 continue
0800 plotFiles = self._doPlots(sample, harvestedFile, plotterFolder, dqmSubFolder, htmlReport)
0801 htmlReport.addPlots(plotterFolder, dqmSubFolder, plotFiles)
0802
0803
0804 if doFastVsFull:
0805 self._doFastsimFastVsFullPlots(limitSubFoldersOnlyTo, htmlReport)
0806
0807
0808 if doPhase2PU:
0809 self._doPhase2PileupPlots(limitSubFoldersOnlyTo, htmlReport)
0810
0811 def _doFastsimFastVsFullPlots(self, limitSubFoldersOnlyTo, htmlReport):
0812 for fast in self._fastsimSamples:
0813 correspondingFull = None
0814 for full in self._fullsimSamples:
0815 if fast.name() != full.name():
0816 continue
0817 if fast.pileupEnabled():
0818 if not full.pileupEnabled():
0819 continue
0820 if fast.fastsimCorrespondingFullsimPileup() != full.pileupType():
0821 continue
0822 else:
0823 if full.pileupEnabled():
0824 continue
0825
0826 if correspondingFull is None:
0827 correspondingFull = full
0828 else:
0829 raise Exception("Got multiple compatible FullSim samples for FastSim sample %s %s" % (fast.name(), fast.pileup()))
0830 if correspondingFull is None:
0831 print("WARNING: Did not find compatible FullSim sample for FastSim sample %s %s, omitting FastSim vs. FullSim comparison" % (fast.name(), fast.pileup()))
0832 continue
0833
0834
0835 harvestedFile = fast.filename(self._newRelease)
0836 plotterInstance = self._plotter.readDirs(harvestedFile)
0837 htmlReport.beginSample(fast, fastVsFull=True)
0838 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
0839 if not _processPlotsForSample(plotterFolder, fast):
0840 continue
0841 plotFiles = self._doPlotsFastFull(fast, correspondingFull, plotterFolder, dqmSubFolder, htmlReport)
0842 htmlReport.addPlots(plotterFolder, dqmSubFolder, plotFiles)
0843
0844 def _doPhase2PileupPlots(self, limitSubFoldersOnlyTo, htmlReport):
0845 def _stripScenario(name):
0846 puindex = name.find("PU")
0847 if puindex < 0:
0848 return name
0849 return name[:puindex]
0850
0851 pu140samples = {}
0852 for sample in self._fullsimSamples:
0853 if sample.pileupNumber() == 140:
0854 key = (sample.name(), _stripScenario(sample.scenario()))
0855 if key in pu140samples:
0856 raise Exception("Duplicate entry for sample %s in scenario %s" % (sample.name(), sample.scenar()))
0857 pu140samples[key] = sample
0858
0859 for sample in self._fullsimSamples:
0860 if sample.pileupNumber() != 200:
0861 continue
0862 key = (sample.name(), _stripScenario(sample.scenario()))
0863 if not key in pu140samples:
0864 continue
0865
0866 sample_pu140 = pu140samples[key]
0867
0868
0869 harvestedFile = sample.filename(self._newRelease)
0870 plotterInstance = self._plotter.readDirs(harvestedFile)
0871 htmlReport.beginSample(sample, pileupComparison="vs. PU140")
0872 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
0873 if not _processPlotsForSample(plotterFolder, sample):
0874 continue
0875 plotFiles = self._doPlotsPileup(sample_pu140, sample, plotterFolder, dqmSubFolder, htmlReport)
0876 htmlReport.addPlots(plotterFolder, dqmSubFolder, plotFiles)
0877
0878
0879 def _getRefFileAndSelection(self, sample, plotterFolder, dqmSubFolder, selectionNameBase, valname):
0880 if self._refRelease is None:
0881 return (None, "")
0882
0883 refGlobalTag = _getGlobalTag(sample, self._refRelease)
0884 def _createRefSelection(selectionName):
0885 sel = refGlobalTag+selectionNameBase+selectionName
0886 if sample.pileupEnabled():
0887 refPu = sample.pileupType(self._refRelease)
0888 if refPu != "":
0889 sel += "_"+refPu
0890 return sel
0891 refSelection = _createRefSelection(plotterFolder.getSelectionName(dqmSubFolder))
0892
0893
0894 refValFile = None
0895 triedRefValFiles = []
0896 tmp = [self._refRepository, self._refRelease]
0897 if sample.fastsim():
0898 tmp.extend(["fastsim", self._refRelease])
0899 for selName in plotterFolder.getSelectionNameIterator(dqmSubFolder):
0900 refSel = _createRefSelection(selName)
0901 refdir = os.path.join(*(tmp+[refSel, sample.name()]))
0902
0903
0904 refValFilePath = os.path.join(refdir, valname)
0905 if os.path.exists(refValFilePath):
0906 refSelection = refSel
0907 refValFile = ROOT.TFile.Open(refValFilePath)
0908 break
0909 else:
0910 triedRefValFiles.append(refValFilePath)
0911 if refValFile is None:
0912 if len(triedRefValFiles) == 1:
0913 if plotting.verbose:
0914 print("Reference file %s not found" % triedRefValFiles[0])
0915 else:
0916 if plotting.verbose:
0917 print("None of the possible reference files %s not found" % ",".join(triedRefValFiles))
0918
0919 return (refValFile, refSelection)
0920
0921 def _doPlots(self, sample, harvestedFile, plotterFolder, dqmSubFolder, htmlReport):
0922 """Do the real plotting work for a given sample and DQM subfolder"""
0923
0924 newGlobalTag = _getGlobalTag(sample, self._newRelease)
0925
0926
0927 selectionNameBase = "_"+sample.pileup()
0928 newSelection = newGlobalTag+selectionNameBase+plotterFolder.getSelectionName(dqmSubFolder)
0929 if sample.pileupEnabled():
0930 newPu = sample.pileupType(self._newRelease)
0931 if newPu != "":
0932 newSelection += "_"+newPu
0933
0934 valname = "val.{sample}.root".format(sample=sample.name())
0935
0936
0937 (refValFile, refSelection) = self._getRefFileAndSelection(sample, plotterFolder, dqmSubFolder, selectionNameBase, valname)
0938
0939
0940 tmp = []
0941 if sample.fastsim():
0942 tmp.extend(["fastsim", self._newRelease])
0943 tmp.extend([newSelection, sample.name()])
0944 newsubdir = os.path.join(*tmp)
0945 newdir = os.path.join(self._newBaseDir, newsubdir)
0946 if not os.path.exists(newdir):
0947 os.makedirs(newdir)
0948 valnameFullPath = os.path.join(newdir, valname)
0949
0950
0951
0952 newValFile = _copySubDir(harvestedFile, valnameFullPath, plotterFolder.getPossibleDQMFolders(), dqmSubFolder.subfolder if dqmSubFolder is not None else None)
0953 fileList = []
0954
0955
0956 if plotting.verbose:
0957 print("Comparing ref and new {sim} {sample} {translatedFolder}".format(
0958 sim="FullSim" if not sample.fastsim() else "FastSim",
0959 sample=sample.name(), translatedFolder=str(dqmSubFolder.translated) if dqmSubFolder is not None else ""))
0960 rootFiles = [refValFile, newValFile]
0961 legendLabels = [
0962 "%s, %s %s" % (sample.name(), _stripRelease(self._refRelease), refSelection) if self._refRelease is not None else "dummy",
0963 "%s, %s %s" % (sample.name(), _stripRelease(self._newRelease), newSelection)
0964 ]
0965 plotterFolder.create(rootFiles, legendLabels, dqmSubFolder, isPileupSample=sample.pileupEnabled())
0966 fileList.extend(plotterFolder.draw(directory=newdir, **self._plotterDrawArgs))
0967
0968 if len(fileList) > 0:
0969 fileList.append(valnameFullPath)
0970
0971
0972 for tableCreator in plotterFolder.getTableCreators():
0973 htmlReport.addTable(tableCreator.create(rootFiles, legendLabels, dqmSubFolder))
0974
0975 newValFile.Close()
0976 if refValFile is not None:
0977 refValFile.Close()
0978
0979 if len(fileList) == 0:
0980 print("No object found in %s" % plotterFolder.getName())
0981 return []
0982
0983 dups = _findDuplicates(fileList)
0984 if len(dups) > 0:
0985 print("Plotter produced multiple files with names", ", ".join(dups))
0986 print("Typically this is a naming problem in the plotter configuration")
0987 sys.exit(1)
0988
0989
0990 print("Created plots and %s in %s" % (valname, newdir))
0991 return list(map(lambda n: n.replace(newdir, newsubdir), fileList))
0992
0993 def _doPlotsFastFull(self, fastSample, fullSample, plotterFolder, dqmSubFolder, htmlReport):
0994 """Do the real plotting work for FastSim vs. FullSim for a given algorithm, quality flag, and sample."""
0995
0996 fastGlobalTag = _getGlobalTag(fastSample, self._newRelease)
0997 fullGlobalTag = _getGlobalTag(fullSample, self._newRelease)
0998
0999
1000 tmp = plotterFolder.getSelectionName(dqmSubFolder)
1001 fastSelection = fastGlobalTag+"_"+fastSample.pileup()+tmp
1002 fullSelection = fullGlobalTag+"_"+fullSample.pileup()+tmp
1003 if fullSample.pileupEnabled():
1004 fullSelection += "_"+fullSample.pileupType(self._newRelease)
1005 fastSelection += "_"+fastSample.pileupType(self._newRelease)
1006
1007
1008 fastdir = os.path.join(self._newBaseDir, "fastsim", self._newRelease, fastSelection, fastSample.name())
1009 fulldir = os.path.join(self._newBaseDir, fullSelection, fullSample.name())
1010 newsubdir = os.path.join("fastfull", self._newRelease, fastSelection, fastSample.name())
1011 newdir = os.path.join(self._newBaseDir, newsubdir)
1012 if not os.path.exists(newdir):
1013 os.makedirs(newdir)
1014
1015
1016 valname = "val.{sample}.root".format(sample=fastSample.name())
1017 fastValFilePath = os.path.join(fastdir, valname)
1018 if not os.path.exists(fastValFilePath) and plotting.verbose:
1019 print("FastSim file %s not found" % fastValFilePath)
1020 fullValFilePath = os.path.join(fulldir, valname)
1021 if not os.path.exists(fullValFilePath) and plotting.verbose:
1022 print("FullSim file %s not found" % fullValFilePath)
1023
1024 fastValFile = ROOT.TFile.Open(fastValFilePath)
1025 fullValFile = ROOT.TFile.Open(fullValFilePath)
1026
1027
1028 if plotting.verbose:
1029 print("Comparing FullSim and FastSim {sample} {translatedFolder}".format(
1030 sample=fastSample.name(), translatedFolder=str(dqmSubFolder.translated) if dqmSubFolder is not None else ""))
1031 rootFiles = [fullValFile, fastValFile]
1032 legendLabels = [
1033 "FullSim %s, %s %s" % (fullSample.name(), _stripRelease(self._newRelease), fullSelection),
1034 "FastSim %s, %s %s" % (fastSample.name(), _stripRelease(self._newRelease), fastSelection),
1035 ]
1036 plotterFolder.create(rootFiles, legendLabels, dqmSubFolder, isPileupSample=fastSample.pileupEnabled(), requireAllHistograms=True)
1037 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1038
1039
1040 for tableCreator in plotterFolder.getTableCreators():
1041 htmlReport.addTable(tableCreator.create(rootFiles, legendLabels, dqmSubFolder))
1042
1043 fullValFile.Close()
1044 fastValFile.Close()
1045
1046 if len(fileList) == 0:
1047 print("No object found in %s" % plotterFolder.getName())
1048 return []
1049
1050 dups = _findDuplicates(fileList)
1051 if len(dups) > 0:
1052 print("Plotter produced multiple files with names", ", ".join(dups))
1053 print("Typically this is a naming problem in the plotter configuration")
1054 sys.exit(1)
1055
1056
1057 print("Created plots in %s" % (newdir))
1058 return list(map(lambda n: n.replace(newdir, newsubdir), fileList))
1059
1060 def _doPlotsPileup(self, pu140Sample, pu200Sample, plotterFolder, dqmSubFolder, htmlReport):
1061 """Do the real plotting work for two pileup scenarios for a given algorithm, quality flag, and sample."""
1062
1063 pu140GlobalTag = _getGlobalTag(pu140Sample, self._newRelease)
1064 pu200GlobalTag = _getGlobalTag(pu200Sample, self._newRelease)
1065
1066
1067 tmp = plotterFolder.getSelectionName(dqmSubFolder)
1068 pu140Selection = pu140GlobalTag+"_"+pu140Sample.pileup()+tmp+"_"+pu140Sample.pileupType(self._newRelease)
1069 pu200Selection = pu200GlobalTag+"_"+pu200Sample.pileup()+tmp+"_"+pu200Sample.pileupType(self._newRelease)
1070
1071
1072 pu140dir = os.path.join(self._newBaseDir, pu140Selection, pu140Sample.name())
1073 pu200dir = os.path.join(self._newBaseDir, pu200Selection, pu200Sample.name())
1074 newsubdir = os.path.join("pileup", self._newRelease, pu200Selection, pu200Sample.name())
1075 newdir = os.path.join(self._newBaseDir, newsubdir)
1076 if not os.path.exists(newdir):
1077 os.makedirs(newdir)
1078
1079
1080 valname = "val.{sample}.root".format(sample=pu140Sample.name())
1081 pu140ValFilePath = os.path.join(pu140dir, valname)
1082 if not os.path.exists(pu140ValFilePath):
1083 if plotting.verbose:
1084 print("PU140 file %s not found" % pu140ValFilePath)
1085 return []
1086 pu200ValFilePath = os.path.join(pu200dir, valname)
1087 if not os.path.exists(pu200ValFilePath):
1088 if plotting.verbose:
1089 print("PU200 file %s not found" % pu200ValFilePath)
1090 return []
1091
1092 pu140ValFile = ROOT.TFile.Open(pu140ValFilePath)
1093 pu200ValFile = ROOT.TFile.Open(pu200ValFilePath)
1094
1095
1096 if plotting.verbose:
1097 print("Comparing PU140 and PU200 {sample} {translatedFolder}".format(
1098 sample=pu200Sample.name(), translatedFolder=str(dqmSubFolder.translated) if dqmSubFolder is not None else ""))
1099 rootFiles = [pu140ValFile, pu200ValFile]
1100 legendLabels = [
1101 "%s, %s %s" % (pu140Sample.name(), _stripRelease(self._newRelease), pu140Selection),
1102 "%s, %s %s" % (pu200Sample.name(), _stripRelease(self._newRelease), pu200Selection),
1103 ]
1104 plotterFolder.create(rootFiles, legendLabels, dqmSubFolder, isPileupSample=pu140Sample.pileupEnabled(), requireAllHistograms=True)
1105 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1106
1107
1108 for tableCreator in plotterFolder.getTableCreators():
1109 htmlReport.addTable(tableCreator.create(rootFiles, legendLabels, dqmSubFolder))
1110
1111 pu200ValFile.Close()
1112 pu140ValFile.Close()
1113
1114 if len(fileList) == 0:
1115 print("No object found in %s" % plotterFolder.getName())
1116 return []
1117
1118 dups = _findDuplicates(fileList)
1119 if len(dups) > 0:
1120 print("Plotter produced multiple files with names", ", ".join(dups))
1121 print("Typically this is a naming problem in the plotter configuration")
1122 sys.exit(1)
1123
1124
1125 print("Created plots in %s" % (newdir))
1126 return list(map(lambda n: n.replace(newdir, newsubdir), fileList))
1127
1128
1129 def _copySubDir(oldfile, newfile, basenames, dirname):
1130 """Copy a subdirectory from oldfile to newfile.
1131
1132 Arguments:
1133 oldfile -- String for source TFile
1134 newfile -- String for destination TFile
1135 basenames -- List of strings for base directories, first existing one is picked
1136 dirname -- String for directory name under the base directory
1137 """
1138 oldf = ROOT.TFile.Open(oldfile)
1139
1140 dirold = None
1141 for basename in basenames:
1142 dirold = oldf.GetDirectory(basename)
1143 if dirold:
1144 break
1145 if not dirold:
1146 raise Exception("Did not find any of %s directories from file %s" % (",".join(basenames), oldfile))
1147 if dirname:
1148 d = dirold.Get(dirname)
1149 if not d:
1150 raise Exception("Did not find directory %s under %s" % (dirname, dirold.GetPath()))
1151 dirold = d
1152
1153 newf = ROOT.TFile.Open(newfile, "RECREATE")
1154 dirnew = newf
1155 for d in basenames[0].split("/"):
1156 dirnew = dirnew.mkdir(d)
1157 if dirname:
1158 dirnew = dirnew.mkdir(dirname)
1159 _copyDir(dirold, dirnew)
1160
1161 oldf.Close()
1162 return newf
1163
1164 def _copyDir(src, dst):
1165 """Copy non-TTree objects from src TDirectory to dst TDirectory."""
1166 keys = src.GetListOfKeys()
1167 for key in keys:
1168 classname = key.GetClassName()
1169 cl = ROOT.TClass.GetClass(classname)
1170 if not cl:
1171 continue
1172 if not (cl.InheritsFrom("TTree") and cl.InheritsFrom("TDirectory")):
1173 dst.cd()
1174 obj = key.ReadObj()
1175 obj.Write()
1176 obj.Delete()
1177
1178 def _findDuplicates(lst):
1179 found = set()
1180 found2 = set()
1181 for x in lst:
1182 if x in found:
1183 found2.add(x)
1184 else:
1185 found.add(x)
1186 return list(found2)
1187
1188 def _doPlotsForPlotter(self, plotter, sample, limitSubFoldersOnlyTo=None):
1189 plottingProcess = self._nProc
1190 if plottingProcess<=0: plottingProcess = os.cpu_count()
1191 plotterInstance = plotter.readDirs(*self._openFiles)
1192 manager = multiprocessing.Manager()
1193 return_dict = manager.dict()
1194 proc = []
1195 iProc = 0
1196 active_proc = []
1197
1198 for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
1199 if sample is not None and not _processPlotsForSample(plotterFolder, sample):
1200 continue
1201 newsubdir = self._subdirprefix+plotterFolder.getSelectionName(dqmSubFolder)
1202 newdir = os.path.join(self._newdir, newsubdir)
1203 if not os.path.exists(newdir):
1204 os.makedirs(newdir, exist_ok=True)
1205
1206 plotterFolder.create(self._openFiles, self._labels, dqmSubFolder)
1207 while len(active_proc)>=plottingProcess:
1208 time.sleep(0.1)
1209 active_proc = [p for p in active_proc if p.is_alive()]
1210 p = multiprocessing.Process(target=self._doPlots, args=(plotterFolder, dqmSubFolder, newsubdir, newdir, iProc, return_dict))
1211 proc.append((plotterFolder, dqmSubFolder, p))
1212 active_proc.append(p)
1213 p.start()
1214 iProc += 1
1215
1216 for i in range(iProc):
1217 proc[i][2].join()
1218 if len(return_dict[i]) > 0:
1219 self._htmlReport.addPlots(proc[i][0], proc[i][1], return_dict[i])
1220
1221 class SimpleSample:
1222 def __init__(self, label, name, fileLegends, pileup=True, customPileupLabel=""):
1223 self._label = label
1224 self._name = name
1225 self._fileLegends = fileLegends
1226 self._pileup = pileup
1227 self._customPileupLabel = customPileupLabel
1228
1229 def digest(self):
1230
1231 return self._label
1232
1233 def label(self):
1234 return self._label
1235
1236 def name(self):
1237 return self._name
1238
1239 def files(self):
1240 return [t[0] for t in self._fileLegends]
1241
1242 def legendLabels(self):
1243 return [t[1] for t in self._fileLegends]
1244
1245 def fastsim(self):
1246
1247 return False
1248
1249 def pileupEnabled(self):
1250 return self._pileup
1251
1252 def customPileupLabel(self):
1253 return self._customPileupLabel
1254
1255 def doElectron(self):
1256 return True
1257
1258 def doConversion(self):
1259 return True
1260
1261 def doBHadron(self):
1262 return True
1263
1264 class SimpleValidation:
1265 def __init__(self, samples, newdir, nProc=0):
1266 self._samples = samples
1267 self._newdir = newdir
1268 self._nProc = nProc
1269 if not os.path.exists(newdir):
1270 os.makedirs(newdir)
1271
1272 self._htmlReport = html.HtmlReportDummy()
1273
1274 def createHtmlReport(self, validationName=""):
1275 if hasattr(self._htmlReport, "write"):
1276 raise Exception("HTML report object already created. There is probably some logic error in the calling code.")
1277 self._htmlReport = html.HtmlReport(validationName, self._newdir)
1278 return self._htmlReport
1279
1280 def doPlots(self, plotters, plotterDrawArgs={}, **kwargs):
1281 self._plotterDrawArgs = plotterDrawArgs
1282
1283 for sample in self._samples:
1284 self._subdirprefix = sample.label()
1285 self._labels = sample.legendLabels()
1286 self._htmlReport.beginSample(sample)
1287
1288 self._openFiles = []
1289 for f in sample.files():
1290 if os.path.exists(f):
1291 self._openFiles.append(ROOT.TFile.Open(f))
1292 else:
1293 print("File %s not found (from sample %s), ignoring it" % (f, sample.name()))
1294 self._openFiles.append(None)
1295
1296 for plotter in plotters:
1297 _doPlotsForPlotter(self, plotter, sample, **kwargs)
1298
1299 for tf in self._openFiles:
1300 if tf is not None:
1301 tf.Close()
1302 self._openFiles = []
1303
1304 def _doPlots(self, plotterFolder, dqmSubFolder, newsubdir, newdir, iProc, return_dict):
1305 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1306
1307 if len(fileList) == 0:
1308 print("No object found in %s" % plotterFolder.getName())
1309
1310 for tableCreator in plotterFolder.getTableCreators():
1311 self._htmlReport.addTable(tableCreator.create(self._openFiles, self._labels, dqmSubFolder))
1312
1313 dups = _findDuplicates(fileList)
1314 if len(dups) > 0:
1315 print("Plotter produced multiple files with names", ", ".join(dups))
1316 print("Typically this is a naming problem in the plotter configuration")
1317 sys.exit(1)
1318
1319 if self._plotterDrawArgs.get("separate", False):
1320 if not os.path.exists("%s/res"%newdir):
1321 os.makedirs("%s/res"%newdir)
1322 downloadables = ["index.php", "res/jquery-ui.js", "res/jquery.js", "res/style.css", "res/style.js", "res/theme.css"]
1323 for d in downloadables:
1324 if not os.path.exists("%s/%s" % (newdir,d)):
1325 urllib.request.urlretrieve("https://raw.githubusercontent.com/musella/php-plots/master/%s"%d, "%s/%s"%(newdir,d))
1326
1327 print("Created plots in %s" % newdir)
1328 return_dict[iProc] = list(map(lambda n: n.replace(newdir, newsubdir), fileList))
1329
1330 class SeparateValidation:
1331
1332
1333 def __init__(self, samples, newdir, nProc=0):
1334 self._samples = samples
1335 self._newdir = newdir
1336 self._nProc = nProc
1337 if not os.path.exists(newdir):
1338 os.makedirs(newdir)
1339
1340 self._htmlReport = html.HtmlReportDummy()
1341
1342 def createHtmlReport(self, validationName=""):
1343 if hasattr(self._htmlReport, "write"):
1344 raise Exception("HTML report object already created. There is probably some logic error in the calling code.")
1345 self._htmlReport = html.HtmlReport(validationName, self._newdir)
1346 return self._htmlReport
1347
1348 def doPlots(self, plotters, plotterDrawArgs={}, **kwargs):
1349 self._plotterDrawArgs = plotterDrawArgs
1350
1351 for sample in self._samples:
1352 self._subdirprefix = sample.label()
1353 self._labels = sample.legendLabels()
1354 self._htmlReport.beginSample(sample)
1355
1356 self._openFiles = []
1357 for f in sample.files():
1358 if os.path.exists(f):
1359 self._openFiles.append(ROOT.TFile.Open(f))
1360 else:
1361 print("File %s not found (from sample %s), ignoring it" % (f, sample.name()))
1362 self._openFiles.append(None)
1363
1364 for plotter in plotters:
1365 _doPlotsForPlotter(self, plotter, sample, **kwargs)
1366
1367 for tf in self._openFiles:
1368 if tf is not None:
1369 tf.Close()
1370 self._openFiles = []
1371
1372 def _doPlots(self, plotterFolder, dqmSubFolder, newsubdir, newdir, iProc, return_dict):
1373 fileList = plotterFolder.draw(directory=newdir, **self._plotterDrawArgs)
1374
1375
1376 if len(fileList) == 0:
1377 print("No object found in %s" % plotterFolder.getName())
1378
1379
1380 dups = _findDuplicates(fileList)
1381 if len(dups) > 0:
1382 print("Plotter produced multiple files with names", ", ".join(dups))
1383 print("Typically this is a naming problem in the plotter configuration")
1384 sys.exit(1)
1385
1386 linkList = []
1387 for f in fileList:
1388 if f[:f.rfind("/")] not in linkList :
1389 if str(f[:f.rfind("/")]) != str(newdir) :
1390 linkList.append(f[:f.rfind("/")])
1391
1392 for tableCreator in plotterFolder.getTableCreators():
1393 self._htmlReport.addTable(tableCreator.create(self._openFiles, self._labels, dqmSubFolder))
1394
1395 for link in linkList :
1396 if not os.path.exists("%s/res"%link):
1397 os.makedirs("%s/res"%link)
1398 downloadables = ["index.php", "res/jquery-ui.js", "res/jquery.js", "res/style.css", "res/style.js", "res/theme.css"]
1399 for d in downloadables:
1400 if not os.path.exists("%s/%s" % (link,d)):
1401 urllib.request.urlretrieve("https://raw.githubusercontent.com/rovere/php-plots/master/%s"%d, "%s/%s"%(link,d))
1402
1403 print("Created separated plots in %s" % newdir)
1404 return_dict[iProc] = list(map(lambda n: n.replace(newdir, newsubdir), linkList))
