File indexing completed on 2024-04-06 11:59:29
0001
0002
0003 from __future__ import print_function
0004 import sys, os, os.path, re, string, httplib, mimetypes, urllib, urllib2, httplib, gzip, md5
0005 from cStringIO import StringIO
0006 from stat import *
0007 import optparse
0008
0009 try:
0010 import hashlib
0011 except:
0012 pass
0013
0014 HTTPS = httplib.HTTPS
0015 if sys.version_info[:3] >= (2, 4, 0):
0016 HTTPS = httplib.HTTPSConnection
0017
0018 ssl_key_file = None
0019 ssl_cert_file = None
0020
0021 class HTTPSCertAuth(HTTPS):
0022 def __init__(self, host, *args, **kwargs):
0023 HTTPS.__init__(self, host, key_file = ssl_key_file, cert_file = ssl_cert_file, **kwargs)
0024
0025 class HTTPSCertAuthenticate(urllib2.AbstractHTTPHandler):
0026 def default_open(self, req):
0027 return self.do_open(HTTPSCertAuth, req)
0028
0029 def filetype(filename):
0030 return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
0031
0032 def encode(args, files):
0033 """
0034 Encode form (name, value) and (name, filename, type) elements into
0035 multi-part/form-data. We don't actually need to know what we are
0036 uploading here, so just claim it's all text/plain.
0037 """
0038 boundary = '----------=_DQM_FILE_BOUNDARY_=-----------'
0039 (body, crlf) = ('', '\r\n')
0040 for (key, value) in args.items():
0041 body += '--' + boundary + crlf
0042 body += ('Content-disposition: form-data; name="%s"' % key) + crlf
0043 body += crlf + str(value) + crlf
0044 for (key, filename) in files.items():
0045 body += '--' + boundary + crlf
0046 body += ('Content-Disposition: form-data; name="%s"; filename="%s"'
0047 % (key, os.path.basename(filename))) + crlf
0048 body += ('Content-Type: %s' % filetype(filename)) + crlf
0049 body += crlf + open(filename, "r").read() + crlf
0050 body += '--' + boundary + '--' + crlf + crlf
0051 return ('multipart/form-data; boundary=' + boundary, body)
0052
0053 def marshall(args, files, request):
0054 """
0055 Marshalls the arguments to the CGI script as multi-part/form-data,
0056 not the default application/x-www-form-url-encoded. This improves
0057 the transfer of the large inputs and eases command line invocation
0058 of the CGI script.
0059 """
0060 (type, body) = encode(args, files)
0061 request.add_header('Content-type', type)
0062 request.add_header('Content-length', str(len(body)))
0063 request.add_data(body)
0064
0065 def upload(url, args, files):
0066 ident = "visDQMUpload DQMGUI/%s CMSSW/%s python/%s" % \
0067 (os.getenv('DQMGUI_VERSION', '?'),
0068 os.getenv('DQM_CMSSW_VERSION', os.getenv('CMSSW_VERSION', '?')),
0069 "%d.%d.%d" % sys.version_info[:3])
0070 cookie = None
0071 if url.startswith("https:"):
0072 authreq = urllib2.Request(url + '/authenticate')
0073 authreq.add_header('User-agent', ident)
0074 result = urllib2.build_opener(HTTPSCertAuthenticate()).open(authreq)
0075 cookie = result.headers.get('Set-Cookie')
0076 if not cookie:
0077 raise RuntimeError("Did not receive authentication cookie")
0078 cookie = cookie.split(";")[0]
0079
0080 datareq = urllib2.Request(url + '/data/put')
0081 datareq.add_header('Accept-encoding', 'gzip')
0082 datareq.add_header('User-agent', ident)
0083 if cookie:
0084 datareq.add_header('Cookie', cookie)
0085 marshall(args, files, datareq)
0086 result = urllib2.build_opener().open(datareq)
0087 data = result.read()
0088 if result.headers.get ('Content-encoding', '') == 'gzip':
0089 data = gzip.GzipFile (fileobj=StringIO(data)).read ()
0090 return (result.headers, data)
0091
0092 def print_help(*args):
0093 sys.stdout.write("\n")
0094 sys.stdout.write("This scripts intends to do the upload of files to the DQM server. It\n")
0095 sys.stdout.write("runs some basic checks on the file name as it is crucial to follow\n")
0096 sys.stdout.write("the naming convention.\n")
0097 sys.stdout.write("\n")
0098 sys.stdout.write("Mandatory Option\n")
0099 sys.stdout.write(" -d, --destination parameter to specify the DQM server\n")
0100 sys.stdout.write("\n")
0101 sys.stdout.write(" Proxy Options\n")
0102 sys.stdout.write(" The script will try to find your grid proxy automatically. To\n")
0103 sys.stdout.write(" do so, it checks $X509_* environment variables and your globus\n")
0104 sys.stdout.write(" directory (~/.globus). To override this automatism, you can use\n")
0105 sys.stdout.write(" the following two options:\n")
0106 sys.stdout.write(" --ssl-key-file location of your private key\n")
0107 sys.stdout.write(" --ssl-cert-file location of your public key\n")
0108 sys.stdout.write("\n")
0109 sys.stdout.write("Other Options\n")
0110 sys.stdout.write(" -h, --help show this help message\n")
0111 sys.stdout.write(" -s, --no-submission suppress the submission\n")
0112 sys.stdout.write(" -r, --no-registration suppress the submission\n")
0113 sys.stdout.write(" --no-filename-check omit the file name check\n")
0114 sys.stdout.write("\n")
0115 sys.exit(0)
0116
0117 def checkSSL(opts):
0118 global ssl_key_file
0119 global ssl_cert_file
0120
0121 if opts.ssl_key_file and os.path.exists(opts.ssl_key_file):
0122 ssl_key_file = opts.ssl_key_file
0123 if opts.ssl_cert_file and os.path.exists(opts.ssl_cert_file):
0124 ssl_cert_file = opts.ssl_cert_file
0125
0126 if not ssl_key_file:
0127 x509_path = os.getenv("X509_USER_PROXY", None)
0128 if x509_path and os.path.exists(x509_path):
0129 ssl_key_file = ssl_cert_file = x509_path
0130
0131 if not ssl_key_file:
0132 x509_path = os.getenv("X509_USER_KEY", None)
0133 if x509_path and os.path.exists(x509_path):
0134 ssl_key_file = x509_path
0135
0136 if not ssl_cert_file:
0137 x509_path = os.getenv("X509_USER_CERT", None)
0138 if x509_path and os.path.exists(x509_path):
0139 ssl_cert_file = x509_path
0140
0141 if not ssl_key_file:
0142 x509_path = os.getenv("HOME") + "/.globus/userkey.pem"
0143 if os.path.exists(x509_path):
0144 ssl_key_file = x509_path
0145
0146 if not ssl_cert_file:
0147 x509_path = os.getenv("HOME") + "/.globus/usercert.pem"
0148 if os.path.exists(x509_path):
0149 ssl_cert_file = x509_path
0150
0151 if not ssl_key_file or not os.path.exists(ssl_key_file):
0152 sys.stderr.write("no certificate private key file found, please specify one via $X509_USER_PROXY, $X509_USER_KEY or --ssl-key-file\n")
0153 sys.exit(2)
0154
0155 if not ssl_cert_file or not os.path.exists(ssl_cert_file):
0156 sys.stderr.write("no certificate public key file found, please specify one via $X509_USER_CERT or --ssl-cert-file\n")
0157 sys.exit(3)
0158
0159 print("Using SSL private key", ssl_key_file)
0160 print("Using SSL public key", ssl_cert_file)
0161
0162
0163 def checkFileName(fileName):
0164 regWhitespace = re.compile('.*\s.*')
0165 if regWhitespace.match(fileName):
0166 sys.stderr.write("whitespace detected!\n")
0167 return False
0168
0169 regRelval=re.compile('.*relval.*')
0170 regCMSSW=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_.*')
0171 regCMSSWpre=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_pre[0-9]+_.*')
0172 if regRelval.match(fileName):
0173
0174 if not regCMSSW.match(fileName):
0175 print("no CMSSW")
0176 return True
0177
0178
0179 def startUpload(url, filename):
0180 global ssl_key_file
0181 global ssl_cert_file
0182
0183 print(url, filename)
0184 try:
0185 (headers, data) = \
0186 upload(url,
0187 { 'size': os.stat(filename).st_size,
0188 'checksum': "md5:%s" % md5.new(file(filename).read()).hexdigest() },
0189 { 'file': filename })
0190 print('Status code: ', headers.get("Dqm-Status-Code", "None"))
0191 print('Message: ', headers.get("Dqm-Status-Message", "None"))
0192 print('Detail: ', headers.get("Dqm-Status-Detail", "None"))
0193 print(data)
0194 except urllib2.HTTPError as e:
0195 print("ERROR", e)
0196 print('Status code: ', e.hdrs.get("Dqm-Status-Code", "None"))
0197 print('Message: ', e.hdrs.get("Dqm-Status-Message", "None"))
0198 print('Detail: ', e.hdrs.get("Dqm-Status-Detail", "None"))
0199 sys.exit(1)
0200
0201 def getURL(filename, destination):
0202 filename = filename.split("/")[-1]
0203 regMC=re.compile('.*_R([0-9]*)__*')
0204 if regMC.match(filename):
0205 m = re.search('.*_R([0-9]*)(__.*).root',filename)
0206 runNr = m.group(1)
0207 dataset = m.group(2).replace("__","/")
0208 else:
0209 m = re.search('.*_R([0-9]*)(_?.*).root',filename)
0210 runNr = m.group(1)
0211 dataset = m.group(2).replace("__","/")
0212 if dataset=="":
0213 dataset="/Global/Online/ALL"
0214 if not runNr:
0215 runNr="1"
0216 if (int(runNr)==1):
0217 return destination+"start?workspace=summary;dataset="+dataset+";sampletype=offline_data"
0218 else:
0219 return destination+"start?workspace=summary;runnr="+runNr+";dataset="+dataset+";sampletype=online_data"
0220
0221 def registerFileAtLogServer(filename, destination, tags):
0222 filename = filename.split("/")[-1]
0223 regMC=re.compile('.*_R([0-9]*)__*')
0224 if regMC.match(filename):
0225 m = re.search('.*_R([0-9]*)(__.*).root',filename)
0226 runNr = m.group(1)
0227 dataset = m.group(2).replace("__","/")
0228 else:
0229 m = re.search('.*_R([0-9]*)(_?.*).root',filename)
0230 runNr = m.group(1)
0231 dataset = m.group(2).replace("__","/")
0232 if dataset=="":
0233 dataset="/Global/Online/ALL"
0234 tempurl = "https://www-ekp.physik.uni-karlsruhe.de/~zeise/cgi-bin/register.py?run="+runNr+"&dataset="+dataset+"&filename="+filename+"&tags="+tags+"&instance="+destination
0235 print("Link that is used to register: ", tempurl)
0236 urllib.urlopen(tempurl)
0237
0238 def main(args):
0239 global opts
0240 parser = optparse.OptionParser(add_help_option=False)
0241 parser.add_option("-h", "--help", action="callback", callback=print_help),
0242 parser.add_option("", "--no-filename-check", dest="no_filename_check", default=False, action="store_true")
0243 parser.add_option("-d", "--destination", dest="destination", default="", action="store")
0244 parser.add_option("-t", "--tags", dest="tags", default="", action="store")
0245 parser.add_option("-s", "--no-submission", dest="submission", default=True, action="store_false")
0246 parser.add_option("-r", "--no-registration", dest="registration", default=True, action="store_false")
0247 parser.add_option("","--ssl-key-file", dest="ssl_key_file", default="", action="store")
0248 parser.add_option("","--ssl-cert-file", dest="ssl_cert_file", default="", action="store")
0249 (opts, args) = parser.parse_args()
0250 opts.abort = False
0251
0252 if not opts.destination:
0253 sys.stderr.write("no destination specified\n")
0254 sys.exit(4)
0255 checkSSL(opts)
0256
0257 if len(args)==0:
0258 sys.stderr.write("no input files specified\n")
0259 sys.exit(1)
0260 for fileName in args:
0261 fileName=fileName.strip()
0262 if not os.path.exists(fileName):
0263 sys.stderr.write("file '%s' doesn't exist!\n" % fileName)
0264 continue
0265 if not opts.no_filename_check and not checkFileName(fileName):
0266 continue
0267 sys.stderr.write("file '%s' passed name check, upload will follow!\n" % fileName)
0268 if opts.submission:
0269 startUpload(opts.destination, fileName)
0270 else:
0271 sys.stdout.write("file '%s' would be uploaded to '%s'\n" % (fileName, opts.destination))
0272 if opts.registration:
0273 registerFileAtLogServer(fileName, opts.destination, opts.tags)
0274 print("You should see the plots here: "+getURL(fileName, opts.destination))
0275
0276 if __name__ == '__main__':
0277 sys.exit(main(sys.argv[1:]))
0278
0279
