FileIO/scripts/visDQMUpload.py

0001 #!/usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import sys
0005 import os
0006 import re
0007 import string
0008 import mimetypes
0009 import http.client as httplib
0010 import ssl
0011 import gzip
0012 import hashlib
0013 from stat import *
0014
0015 import urllib.request as urllib2
0016 from subprocess import getstatusoutput
0017
0018 try:
0019   from Monitoring.DQM import visDQMUtils
0020 except:
0021   from DQMServices.FileIO import visDQMUtils
0022
0023 if sys.version_info[:3] >= (2, 4, 0):
0024   HTTPS = httplib.HTTPSConnection
0025 else:
0026   HTTPS = httplib.HTTPS
0027
0028 ssl_key_file = None
0029 ssl_cert_file = None
0030 context = None
0031
0032 class HTTPSCertAuth(HTTPS):
0033   def __init__(self, host, context = None, *args, **kwargs):
0034     if context is None:
0035        context = ssl._create_default_https_context()
0036     if ssl_key_file or ssl_cert_file:
0037             context.load_cert_chain(ssl_cert_file, ssl_key_file)
0038     HTTPS.__init__(self, host, context = context, **kwargs)
0039
0040 class HTTPSCertAuthenticate(urllib2.AbstractHTTPHandler):
0041   def default_open(self, req):
0042     return self.do_open(HTTPSCertAuth, req)
0043
0044 def filetype(filename):
0045   return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
0046
0047 def encode(args, files):
0048   """
0049     Encode form (name, value) and (name, filename, type) elements into
0050     multi-part/form-data. We don't actually need to know what we are
0051     uploading here, so just claim it's all text/plain.
0052   """
0053   boundary = b'----------=_DQM_FILE_BOUNDARY_=-----------'
0054   (body, crlf) = (b'', b'\r\n')
0055   for (key, value) in args.items():
0056     payload = str(value).encode('utf-8')
0057     body += b'--' + boundary + crlf
0058     body += (b'Content-Disposition: form-data; name="%s"' % key.encode('utf-8')) + crlf
0059     body += crlf + payload + crlf
0060   for (key, filename) in files.items():
0061     body += b'--' + boundary + crlf
0062     body += (b'Content-Disposition: form-data; name="%s"; filename="%s"'
0063              % (key.encode('utf-8'), os.path.basename(filename).encode('utf-8'))) + crlf
0064     body += (b'Content-Type: %s' % filetype(filename).encode('utf-8')) + crlf
0065     body += (b'Content-Length: %d' % os.stat(filename)[ST_SIZE]) + crlf
0066     with open(filename, 'rb') as file:
0067       body += crlf + file.read() + crlf
0068   body += b'--' + boundary + b'--' + crlf + crlf
0069   return (b'multipart/form-data; boundary=' + boundary, body)
0070
0071 def marshall(args, files, request):
0072   """
0073     Marshalls the arguments to the CGI script as multi-part/form-data,
0074     not the default application/x-www-form-url-encoded.  This improves
0075     the transfer of the large inputs and eases command line invocation
0076     of the CGI script.
0077   """
0078   (type, body) = encode(args, files)
0079   request.add_header('Content-Type', type)
0080   request.add_header('Content-Length', str(len(body)))
0081   request.data = body
0082
0083 def upload(url, args, files):
0084   ident = "visDQMUpload DQMGUI/%s python/%s" % \
0085     (os.getenv('DQMGUI_VERSION', '?'), "%d.%d.%d" % sys.version_info[:3])
0086   datareq = urllib2.Request(url + '/data/put')
0087   datareq.add_header('Accept-encoding', 'gzip')
0088   datareq.add_header('User-agent', ident)
0089   marshall(args, files, datareq)
0090   if 'https://' in url:
0091     result = urllib2.build_opener(HTTPSCertAuthenticate()).open(datareq)
0092   else:
0093     result = urllib2.build_opener(urllib2.ProxyHandler({})).open(datareq)
0094
0095   data = result.read()
0096   if result.headers.get ('Content-encoding', '') == 'gzip':
0097     data = gzip.GzipFile (fileobj=StringIO(data)).read ()
0098   return (result.headers, data)
0099
0100 x509_path = os.getenv("X509_USER_PROXY", None)
0101 if x509_path and os.path.exists(x509_path):
0102   ssl_key_file = ssl_cert_file = x509_path
0103
0104 if not ssl_key_file:
0105   x509_path = os.getenv("X509_USER_KEY", None)
0106   if x509_path and os.path.exists(x509_path):
0107     ssl_key_file = x509_path
0108
0109 if not ssl_cert_file:
0110   x509_path = os.getenv("X509_USER_CERT", None)
0111   if x509_path and os.path.exists(x509_path):
0112     ssl_cert_file = x509_path
0113
0114 if not ssl_key_file and not ssl_cert_file:
0115   (status, uid) = getstatusoutput("id -u")
0116   if os.path.exists("/tmp/x509up_u%s" % uid):
0117     ssl_key_file = ssl_cert_file = "/tmp/x509up_u%s" % uid
0118
0119 if not ssl_key_file:
0120   x509_path = os.getenv("HOME") + "/.globus/userkey.pem"
0121   if os.path.exists(x509_path):
0122     ssl_key_file = x509_path
0123
0124 if not ssl_cert_file:
0125   x509_path = os.getenv("HOME") + "/.globus/usercert.pem"
0126   if os.path.exists(x509_path):
0127     ssl_cert_file = x509_path
0128
0129 if 'https://' in sys.argv[1] and (not ssl_key_file or not os.path.exists(ssl_key_file)):
0130   print("no certificate private key file found", file=sys.stderr)
0131   sys.exit(1)
0132
0133 if 'https://' in sys.argv[1] and (not ssl_cert_file or not os.path.exists(ssl_cert_file)):
0134   print("no certificate public key file found", file=sys.stderr)
0135   sys.exit(1)
0136
0137 try:
0138   for file_path in sys.argv[2:]:
0139     # Before even trying to make a call to the other side, we first do a check on
0140     # the filename:
0141     classification_ok, classification_result = visDQMUtils.classifyDQMFile(file_path)
0142     if not classification_ok:
0143       print("Check of filename before upload failed with following message:")
0144       print(classification_result)
0145       sys.exit(1)
0146     # If file check was fine, we continue with the upload method:
0147     else:
0148       print("Using SSL private key", ssl_key_file)
0149       print("Using SSL public key", ssl_cert_file)
0150
0151       hasher = hashlib.md5()
0152       with open(sys.argv[2], 'rb') as file:
0153         buf = file.read()
0154         hasher.update(buf)
0155
0156       (headers, data) = \
0157         upload(sys.argv[1],
0158                { 'size': os.stat(sys.argv[2])[ST_SIZE],
0159                  'checksum': 'md5:%s' % hasher.hexdigest() },
0160                { 'file': file_path })
0161       print('Status code: ', headers.get("Dqm-Status-Code", "None"))
0162       print('Message:     ', headers.get("Dqm-Status-Message", "None"))
0163       print('Detail:      ', headers.get("Dqm-Status-Detail", "None"))
0164       print(data.decode('utf-8'))
0165   sys.exit(0)
0166 except urllib2.HTTPError as e:
0167   print('ERROR', e)
0168   print('Status code: ', e.hdrs.get("Dqm-Status-Code", "None"))
0169   print('Message:     ', e.hdrs.get("Dqm-Status-Message", "None"))
0170   print('Detail:      ', e.hdrs.get("Dqm-Status-Detail", "None"))
0171   sys.exit(1)