test/macros/RelValHarvest.py

0001 #!/usr/bin/env python3
0002
0003 import sys, os, re
0004
0005 from optparse import OptionParser # Command line parsing
0006 usage = "usage: %prog summary files"
0007 version = "%prog."
0008 parser = OptionParser(usage=usage,version=version)
0009 parser.add_option("-p", "--printDataSets", action="store_true", dest="printDS", default=False, help="Print datasets without attempting to download.")
0010 parser.add_option("-M", "--MC", action="store_true", dest="getMC", default=False, help="Get DQM files for MC campaign.")
0011 parser.add_option("-D", "--DATA", action="store_true", dest="getDATA", default=False, help="Get DQM files for DATA campaign.")
0012 parser.add_option("-2", "--2023", action="store_true", dest="get2023", default=False, help="Get DQM files for 2023 campaign.")
0013 (options, args) = parser.parse_args()
0014
0015 ##Begin declaration of Functions
0016
0017 #getDataSets if used to discover and download the datasets. It is seperated as a function so that we can easily switch between
0018 #MC and DATA datasets
0019
0020 def getDataSets( dsFlags = {'RelValMinBias_13__':'MinBias'},
0021                  curl = "/usr/bin/curl -O -L --capath %(CERT_DIR)s --key %(USER_PROXY)s --cert %(USER_PROXY)s https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/%(relValDIR)s",
0022                  ofnBlank = "HcalRecHitValidationRelVal_%(sample)s_%(label)s_%(info)s.root",
0023                  label = "CMSSW_X_Y_Z",
0024                  slabel = "XYZ",
0025                  X509_CERT_DIR = os.getenv('X509_CERT_DIR', "/etc/grid-security/certificates"),
0026                  X509_USER_PROXY = os.getenv('X509_USER_PROXY'),
0027                  relValDIR = "CMSSW_?_?_x",
0028                  printDS = False,
0029                  camType = "MC"):
0030
0031     print ("Taking filenames from directory %s"%relValDIR)
0032
0033     # retrieve the list of datasets
0034     if not os.path.isfile(relValDIR):
0035         curlCommand = curl%{"CERT_DIR":X509_CERT_DIR, "USER_PROXY":X509_USER_PROXY, "relValDIR":relValDIR}
0036         print (curlCommand)
0037         os.system(curlCommand)
0038
0039     # open raw input file
0040     fin = open(relValDIR, "r")
0041
0042     # loop over file and pull out lines of interest
0043     for line in fin:
0044         # limit to one entry per dataset
0045         if label in line:
0046             # select datasets of interest
0047             for str in dsFlags.keys():
0048                 if str in line:
0049                     # extract dataset path
0050                     path = line.split('\'')[1].strip()
0051                     #print ("Getting DQM output from dataset: %s"%path)
0052                     if (path.find("Ideal") > 0 or path.find("design") > 0 or path.find("FastSim") > 0 or path.find("DQM") < 0 or path.find("Pixel") > 0 ):  #skip for unnecessary samples
0053                         continue
0054                     print (path.split("/")[-1]) #path
0055                     if printDS:
0056                         continue
0057
0058                     # construct file name
0059                     fname = path.split("/")[-1]
0060
0061                     # create file name for use with hcal scripts
0062                     info = fname.split("__")[2].replace(label, "").strip("-")
0063
0064                     #The Data sample have an additional piece put in. We strip it out so that the MC and DATA code can be common
0065
0066                     if camType == "DATA":
0067                         iparts = info.split("_")
0068                         info = ""
0069                         skip = False
0070                         for fragment in iparts:
0071                             if skip:
0072                                 info = info.strip("_")
0073                                 skip = False
0074                                 continue
0075                             if fragment == "RelVal":
0076                                 skip = True
0077                                 continue
0078                             info += fragment
0079                             info += "_"
0080                         info = info.strip("_")
0081
0082                     ofn = ofnBlank%{"sample":dsFlags[str],"label":slabel,"info":info}
0083                     print ("ofn = ",ofn)
0084                     #Check if file exists already
0085                     if not os.path.isfile(ofn):
0086                         # copy file with curl
0087                         curlCommand = curl%{"CERT_DIR":X509_CERT_DIR,"USER_PROXY":X509_USER_PROXY, "relValDIR":relValDIR} + "/" + fname
0088                         print (curlCommand)
0089                         os.system(curlCommand)
0090
0091                         # Rename file for use with HCAL scripts
0092                         mvCommand = "mv %(fn)s %(ofn)s"%{"fn":fname,"ofn":ofn}
0093                         print (mvCommand)
0094                         os.system(mvCommand)
0095                         #print ""
0096
0097     fin.close();
0098     rmCommand = "rm %(ofn)s"%{"ofn":relValDIR}
0099     print (rmCommand)
0100     os.system(rmCommand)
0101
0102     if printDS:
0103         return
0104
0105     # Copy the single pion scan part from Salavat's directory
0106     #    spFileName = "pi50scan%s_fullGeom_ECALHCAL_CaloTowers.root"%slabel #->original line
0107     #spFileName = "pi50scan%s_ECALHCAL_CaloTowers.root"%slabel
0108     #cpCommand = "cp /afs/cern.ch/user/a/abdullin/public/pi50_scan/%s ."%spFileName
0109     #if not os.path.isfile(spFileName):
0110     #    print cpCommand
0111     #    os.system(cpCommand)
0112     #    print ""
0113
0114 ##End Functions
0115
0116
0117
0118 # This is a dictionary of flags to pull out the datasets of interest mapped to the desired name from the hcal script
0119 dsMCFlags = {'RelValTTbar_13__':'TTbar', 'RelValQCD_Pt_80_120_13__':'QCD', 'RelValQCD_Pt_3000_3500_13__':'HighPtQCD', 'RelValMinBias_13__':'MinBias'}
0120 ds2023Flags = {'RelValTTbar_14TeV__':'TTbar', 'RelValMinBias_14TeV__':'MinBias'}
0121
0122 #dsDATAFlags = {'301998__JetHT__':'JetHT', '301998__ZeroBias__':'ZeroBias'}  #Original
0123 #dsDATAFlags = {'305064__JetHT__':'JetHT','305064__ZeroBias__':'ZeroBias'} # 2017F
0124 dsDATAFlags = {'297557__JetHT__':'JetHT','297557__ZeroBias__':'ZeroBias'} # 2017B
0125 #dsDATAFlags = {'274199__JetHT__':'JetHT','274199__ZeroBias__':'ZeroBias','297227__JetHT__':'JetHT','297227__ZeroBias__':'ZeroBias'} #2016B & 2017B dataset
0126
0127 #dsDATAFlags = {'274199__JetHT__':'JetHT','274199__ZeroBias__':'ZeroBias','297227__JetHT__':'JetHT','297227__ZeroBias__':'ZeroBias','302663__JetHT__':'JetHT','302663__ZeroBias__':'ZeroBias'} #2016B & 2017B & 2017D dataset
0128
0129
0130 #dsDATAFlags = {'274199__JetHT__':'JetHT','274199__ZeroBias__':'ZeroBias','297557__JetHT__':'JetHT','297557__ZeroBias__':'ZeroBias','305064__JetHT__':'JetHT','305064__ZeroBias__':'ZeroBias'} #2016B & 2017B & 2017D dataset
0131 #dsDATAFlags = {'256677__SingleMuon__':'SingleMuon'} #New_original
0132 #dsDATAFlags = {'254790__JetHT__':'JetHT','254790__ZeroBias__':'ZeroBias','254790__SingleMuon__':'SingleMuon'} #New_new
0133 # filename prefix
0134 #fnPrefix = "DQM_V0001_R000000001"
0135 #MinBiasPrefix = "DQM_V0001_R000149011"
0136 #JetPrefix = "DQM_V0001_R000191226"
0137
0138 # blank curl command
0139 curlMC = "/usr/bin/curl -O -L --capath %(CERT_DIR)s --key %(USER_PROXY)s --cert %(USER_PROXY)s https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/%(relValDIR)s"
0140 curlDATA = "/usr/bin/curl -O -L --capath %(CERT_DIR)s --key %(USER_PROXY)s --cert %(USER_PROXY)s https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelValData/%(relValDIR)s"
0141 # output file name blank
0142 ofnBlank = "HcalRecHitValidationRelVal_%(sample)s_%(label)s_%(info)s.root"
0143
0144 # default release file for MC stub
0145 #dfTextFile = "%s_%s.txt"
0146 dfTextFile = "%s"
0147
0148 # ensure all required parameters are included
0149 if len(args) < 1:
0150     print ("Usage: ./RelValHarvest.py -M (or -D) fullReleaseName")
0151     print ("fullReleaseName : CMSSW_7_4_0_pre8")
0152     exit(0)
0153
0154 #Make sure a Dataset is specified
0155 if not options.getMC and not options.getDATA and not options.get2023:
0156     print ("You must specify a dataset:")
0157     print ("    -M : Monte Carlo")
0158     print ("    -D : Data")
0159     print ("    -2 : 2023")
0160     exit(0)
0161
0162 # gather input parameter
0163 label     = args[0]
0164
0165 #Now we check if the release provided works
0166 pattern = re.compile(r'CMSSW_\d{1,2}_\d{1,2}_\d{1,2}.*') #We are checking if the string begins with CMSSW_?_?_?, posibly with two digits in each position
0167 match = pattern.match(label)
0168 if match:
0169     slabel = match.group().replace('CMSSW','').replace("_","")
0170 else:
0171     print (label, " is an invalid CMMSW release name.")
0172     print ("Please provide a release name in the form: CMSSW_X_Y_Z")
0173     exit(0)
0174
0175 # gather necessary proxy info for curl
0176 X509_CERT_DIR = os.getenv('X509_CERT_DIR', "/etc/grid-security/certificates")
0177 X509_USER_PROXY = os.getenv('X509_USER_PROXY')
0178
0179 # modify label to shortened format (remove CMSSW and '_')
0180 #slabel = label.replace('CMSSW','').replace("_","")
0181
0182 # get relval dir from label
0183 clabel = label.split("_")
0184 relValDIR = "%s_%s_%s_x"%(clabel[0], clabel[1], clabel[2])
0185
0186 if options.getMC:
0187     getDataSets( dsFlags = dsMCFlags,
0188                  curl = curlMC,
0189                  label = label,
0190                  slabel = slabel,
0191                  relValDIR = relValDIR,
0192                  printDS = options.printDS,
0193                  camType = "MC")
0194
0195 if options.get2023:
0196     getDataSets( dsFlags = ds2023Flags,
0197                  curl = curlMC,
0198                  label = label,
0199                  slabel = slabel,
0200                  relValDIR = relValDIR,
0201                  printDS = options.printDS,
0202                  camType = "2023")
0203
0204 if options.getDATA:
0205     getDataSets( dsFlags = dsDATAFlags,
0206                  curl = curlDATA,
0207                  label = label,
0208                  slabel = slabel,
0209                  relValDIR = relValDIR,
0210                  printDS = options.printDS,
0211                  camType = "DATA")
0212