Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-04-06 12:31:55

0001 #!/usr/bin/env python3
0002 
0003 import sys, os, re
0004 
0005 from optparse import OptionParser # Command line parsing
0006 usage = "usage: %prog summary files"
0007 version = "%prog."
0008 parser = OptionParser(usage=usage,version=version)
0009 parser.add_option("-p", "--printDataSets", action="store_true", dest="printDS", default=False, help="Print datasets without attempting to download.")
0010 parser.add_option("-M", "--MC", action="store_true", dest="getMC", default=False, help="Get DQM files for MC campaign.")
0011 parser.add_option("-D", "--DATA", action="store_true", dest="getDATA", default=False, help="Get DQM files for DATA campaign.")
0012 parser.add_option("-2", "--2023", action="store_true", dest="get2023", default=False, help="Get DQM files for 2023 campaign.")
0013 (options, args) = parser.parse_args()
0014 
0015 ##Begin declaration of Functions
0016 
0017 #getDataSets if used to discover and download the datasets. It is seperated as a function so that we can easily switch between
0018 #MC and DATA datasets
0019 
0020 def getDataSets( dsFlags = {'RelValMinBias_14TeV__':'MinBias'},
0021                  curl = "/usr/bin/curl -O -L --capath %(CERT_DIR)s --key %(USER_PROXY)s --cert %(USER_PROXY)s https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/%(relValDIR)s",
0022                  ofnBlank = "HcalRecHitValidationRelVal_%(sample)s_%(label)s_%(info)s.root",
0023                  label = "CMSSW_X_Y_Z",
0024                  slabel = "XYZ",
0025                  X509_CERT_DIR = os.getenv('X509_CERT_DIR', "/etc/grid-security/certificates"),
0026                  X509_USER_PROXY = os.getenv('X509_USER_PROXY'),
0027                  relValDIR = "CMSSW_?_?_x",
0028                  printDS = False,
0029                  camType = "MC"):
0030                
0031     print("Taking filenames from directory ",relValDIR)
0032 
0033     # retrieve the list of datasets
0034     if not os.path.isfile(relValDIR):
0035         curlCommand = curl%{"CERT_DIR":X509_CERT_DIR, "USER_PROXY":X509_USER_PROXY, "relValDIR":relValDIR}
0036         print(curlCommand)
0037         os.system(curlCommand)
0038 
0039     # open raw input file 
0040     fin = open(relValDIR, "r")
0041 
0042     # loop over file and pull out lines of interest
0043     for line in fin:
0044         # limit to one entry per dataset
0045         if label in line:
0046             # select datasets of interest
0047             for str in dsFlags.keys():
0048                 if str in line:
0049                     # extract dataset path
0050                     path = line.split('\'')[1].strip()
0051                     if (path.find("2023") < 0):
0052                         continue    
0053                         print(path.split("/")[-1]) #path
0054                         if printDS:
0055                            continue
0056 
0057                     # construct file name
0058                     fname = path.split("/")[-1]
0059                     # create file name for use with hcal scripts
0060                     info = fname.split("__")[2].replace(label, "").strip("-")
0061 
0062                     ofn = ofnBlank%{"sample":dsFlags[str],"label":slabel,"info":info}
0063                     if camType == "DATA":
0064                         ofn = ofn.replace("zb","")
0065                         ofn = ofn.replace("jetHT","")
0066                     print("ofn = ",ofn)
0067                     #Check if file exists already
0068                     if not os.path.isfile(ofn):
0069                         # copy file with curl
0070                         curlCommand = curl%{"CERT_DIR":X509_CERT_DIR,"USER_PROXY":X509_USER_PROXY, "relValDIR":relValDIR} + "/" + fname
0071                         print(curlCommand)
0072                         os.system(curlCommand)
0073 
0074                         # Rename file for use with HCAL scripts
0075                         mvCommand = "mv %(fn)s %(ofn)s"%{"fn":fname,"ofn":ofn}
0076                         print(mvCommand)
0077                         os.system(mvCommand)
0078                         print("")
0079                         # sys.exit();
0080                         
0081 
0082     fin.close();
0083     rmCommand = "rm %(ofn)s"%{"ofn":relValDIR}
0084     print(rmCommand)
0085     os.system(rmCommand)
0086 
0087     if printDS:
0088         return
0089 
0090 ##End Functions
0091 
0092 
0093 
0094 # This is a dictionary of flags to pull out the datasets of interest mapped to the desired name from the hcal script
0095 dsMCFlags = {'RelValTTbar_14TeV__':'TTbar', 'RelValMinBias_14TeV__':'MinBias'}
0096 #ds2023Flags = {'RelValTTbar_14TeV__':'TTbar', 'RelValMinBias_14TeV__':'MinBias'}
0097 
0098 dsDATAFlags = {'297557__JetHT__':'JetHT','297557__ZeroBias__':'ZeroBias', #2017B
0099                '315489__JetHT__':'JetHT','315489__ZeroBias__':'ZeroBias', #2018A
0100                '317435__JetHT__':'JetHT','317435__ZeroBias__':'ZeroBias', #2018B
0101                '319450__JetHT__':'JetHT','319450__ZeroBias__':'ZeroBias', #2018C
0102                '320822__JetHT__':'JetHT','320822__ZeroBias__':'ZeroBias'} # 2018D
0103 
0104 
0105 # blank curl command 
0106 curlMC = "/usr/bin/curl -O -L --capath %(CERT_DIR)s --key %(USER_PROXY)s --cert %(USER_PROXY)s https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/%(relValDIR)s"
0107 curlDATA = "/usr/bin/curl -O -L --capath %(CERT_DIR)s --key %(USER_PROXY)s --cert %(USER_PROXY)s https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelValData/%(relValDIR)s"
0108 print("SMdebug: Curl MC ") 
0109 # output file name blank
0110 ofnBlank = "HcalRecHitValidationRelVal_%(sample)s_%(label)s_%(info)s.root"
0111 
0112 # default release file for MC stub
0113 #dfTextFile = "%s_%s.txt"
0114 dfTextFile = "%s"
0115 
0116 # ensure all required parameters are included
0117 if len(args) < 1:
0118     print("Usage: ./RelValHarvest.py -M (or -D) fullReleaseName")
0119     print("fullReleaseName : CMSSW_7_4_0_pre8")
0120     exit(0)
0121 
0122 #Make sure a Dataset is specified
0123 if not options.getMC and not options.getDATA and not options.get2023:
0124     print("You must specify a dataset:")
0125     print("    -M : Monte Carlo")
0126     print("    -D : Data")
0127     print("    -2 : 2023")
0128     exit(0)
0129 
0130 # gather input parameter
0131 label     = args[0]
0132 
0133 #Now we check if the release provided works
0134 pattern = re.compile(r'CMSSW_\d{1,2}_\d{1,2}_\d{1,2}.*') #We are checking if the string begins with CMSSW_?_?_?, posibly with two digits in each position
0135 match = pattern.match(label)
0136 if match:
0137     slabel = match.group().replace('CMSSW','').replace("_","")
0138 else:
0139     print(label, " is an invalid CMMSW release name.")
0140     print("Please provide a release name in the form: CMSSW_X_Y_Z")
0141     exit(0)
0142 
0143 # gather necessary proxy info for curl
0144 X509_CERT_DIR = os.getenv('X509_CERT_DIR', "/etc/grid-security/certificates")
0145 X509_USER_PROXY = os.getenv('X509_USER_PROXY')
0146 
0147 # modify label to shortened format (remove CMSSW and '_')
0148 #slabel = label.replace('CMSSW','').replace("_","")
0149 
0150 # get relval dir from label
0151 clabel = label.split("_")
0152 relValDIR = "%s_%s_%s_x"%(clabel[0], clabel[1], clabel[2])
0153 
0154 if options.getMC:
0155     getDataSets( dsFlags = dsMCFlags,
0156                  curl = curlMC,
0157                  label = label,
0158                  slabel = slabel,
0159                  relValDIR = relValDIR,
0160                  printDS = options.printDS,
0161                  camType = "MC")
0162 
0163 if options.get2023:
0164     getDataSets( dsFlags = ds2023Flags,
0165                  curl = curlMC,
0166                  label = label,
0167                  slabel = slabel,
0168                  relValDIR = relValDIR,
0169                  printDS = options.printDS,
0170                  camType = "2023")
0171 
0172 if options.getDATA:
0173     getDataSets( dsFlags = dsDATAFlags,
0174                  curl = curlDATA,
0175                  label = label,
0176                  slabel = slabel,
0177                  relValDIR = relValDIR,
0178                  printDS = options.printDS,
0179                  camType = "DATA")
0180