Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2022-03-03 02:23:16

0001 #!/usr/bin/env python3
0002 
0003 from __future__ import print_function
0004 import sys, os, os.path, re, string, httplib, mimetypes, urllib, urllib2, httplib, gzip, md5
0005 from cStringIO import StringIO
0006 from stat import *
0007 import optparse
0008 
0009 try:
0010     import hashlib
0011 except:
0012     pass
0013 
0014 HTTPS = httplib.HTTPS
0015 if sys.version_info[:3] >= (2, 4, 0):
0016   HTTPS = httplib.HTTPSConnection
0017 
0018 ssl_key_file = None
0019 ssl_cert_file = None
0020 
0021 class HTTPSCertAuth(HTTPS):
0022   def __init__(self, host, *args, **kwargs):
0023     HTTPS.__init__(self, host, key_file = ssl_key_file, cert_file = ssl_cert_file, **kwargs)
0024 
0025 class HTTPSCertAuthenticate(urllib2.AbstractHTTPHandler):
0026   def default_open(self, req):
0027     return self.do_open(HTTPSCertAuth, req)
0028 
0029 def filetype(filename):
0030   return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
0031 
0032 def encode(args, files):
0033   """
0034     Encode form (name, value) and (name, filename, type) elements into
0035     multi-part/form-data. We don't actually need to know what we are
0036     uploading here, so just claim it's all text/plain.
0037   """
0038   boundary = '----------=_DQM_FILE_BOUNDARY_=-----------'
0039   (body, crlf) = ('', '\r\n')
0040   for (key, value) in args.items():
0041     body += '--' + boundary + crlf
0042     body += ('Content-disposition: form-data; name="%s"' % key) + crlf
0043     body += crlf + str(value) + crlf
0044   for (key, filename) in files.items():
0045     body += '--' + boundary + crlf
0046     body += ('Content-Disposition: form-data; name="%s"; filename="%s"'
0047              % (key, os.path.basename(filename))) + crlf
0048     body += ('Content-Type: %s' % filetype(filename)) + crlf
0049     body += crlf + open(filename, "r").read() + crlf
0050   body += '--' + boundary + '--' + crlf + crlf
0051   return ('multipart/form-data; boundary=' + boundary, body)
0052 
0053 def marshall(args, files, request):
0054   """
0055     Marshalls the arguments to the CGI script as multi-part/form-data,
0056     not the default application/x-www-form-url-encoded.  This improves
0057     the transfer of the large inputs and eases command line invocation
0058     of the CGI script.
0059   """
0060   (type, body) = encode(args, files)
0061   request.add_header('Content-type', type)
0062   request.add_header('Content-length', str(len(body)))
0063   request.add_data(body)
0064 
0065 def upload(url, args, files):
0066   ident = "visDQMUpload DQMGUI/%s CMSSW/%s python/%s" % \
0067     (os.getenv('DQMGUI_VERSION', '?'),
0068      os.getenv('DQM_CMSSW_VERSION', os.getenv('CMSSW_VERSION', '?')),
0069      "%d.%d.%d" % sys.version_info[:3])
0070   cookie = None
0071   if url.startswith("https:"):
0072     authreq = urllib2.Request(url + '/authenticate')
0073     authreq.add_header('User-agent', ident)
0074     result = urllib2.build_opener(HTTPSCertAuthenticate()).open(authreq)
0075     cookie = result.headers.get('Set-Cookie')
0076     if not cookie:
0077       raise RuntimeError("Did not receive authentication cookie")
0078     cookie = cookie.split(";")[0]
0079 
0080   datareq = urllib2.Request(url + '/data/put')
0081   datareq.add_header('Accept-encoding', 'gzip')
0082   datareq.add_header('User-agent', ident)
0083   if cookie:
0084     datareq.add_header('Cookie', cookie)
0085   marshall(args, files, datareq)
0086   result = urllib2.build_opener().open(datareq)
0087   data = result.read()
0088   if result.headers.get ('Content-encoding', '') == 'gzip':
0089     data = gzip.GzipFile (fileobj=StringIO(data)).read ()
0090   return (result.headers, data)
0091 
0092 def print_help(*args):
0093     sys.stdout.write("\n")
0094     sys.stdout.write("This scripts intends to do the upload of files to the DQM server. It\n")
0095     sys.stdout.write("runs some basic checks on the file name as it is crucial to follow\n")
0096     sys.stdout.write("the naming convention.\n")
0097     sys.stdout.write("\n")
0098     sys.stdout.write("Mandatory Option\n")
0099     sys.stdout.write("    -d, --destination parameter to specify the DQM server\n")
0100     sys.stdout.write("\n")
0101     sys.stdout.write("  Proxy Options\n")
0102     sys.stdout.write("  The script will try to find your grid proxy automatically. To\n")
0103     sys.stdout.write("  do so, it checks $X509_* environment variables and your globus\n")
0104     sys.stdout.write("  directory (~/.globus). To override this automatism, you can use\n")
0105     sys.stdout.write("  the following two options:\n")
0106     sys.stdout.write("    --ssl-key-file          location of your private key\n")
0107     sys.stdout.write("    --ssl-cert-file         location of your public key\n")
0108     sys.stdout.write("\n")
0109     sys.stdout.write("Other Options\n")
0110     sys.stdout.write("    -h, --help                show this help message\n")
0111     sys.stdout.write("    -s, --no-submission       suppress the submission\n")
0112     sys.stdout.write("    -r, --no-registration     suppress the submission\n")
0113     sys.stdout.write("    --no-filename-check       omit the file name check\n")
0114     sys.stdout.write("\n")
0115     sys.exit(0)
0116 
0117 def checkSSL(opts):
0118     global ssl_key_file
0119     global ssl_cert_file
0120     
0121     if opts.ssl_key_file and os.path.exists(opts.ssl_key_file):
0122         ssl_key_file = opts.ssl_key_file
0123     if opts.ssl_cert_file and os.path.exists(opts.ssl_cert_file):
0124         ssl_cert_file = opts.ssl_cert_file
0125 
0126     if not ssl_key_file:
0127         x509_path = os.getenv("X509_USER_PROXY", None)
0128         if x509_path and os.path.exists(x509_path):
0129             ssl_key_file = ssl_cert_file = x509_path
0130 
0131     if not ssl_key_file:
0132         x509_path = os.getenv("X509_USER_KEY", None)
0133         if x509_path and os.path.exists(x509_path):
0134             ssl_key_file = x509_path
0135 
0136     if not ssl_cert_file:
0137         x509_path = os.getenv("X509_USER_CERT", None)
0138         if x509_path and os.path.exists(x509_path):
0139             ssl_cert_file = x509_path
0140     
0141     if not ssl_key_file:
0142         x509_path = os.getenv("HOME") + "/.globus/userkey.pem"
0143         if os.path.exists(x509_path):
0144             ssl_key_file = x509_path
0145 
0146     if not ssl_cert_file:
0147         x509_path = os.getenv("HOME") + "/.globus/usercert.pem"
0148         if os.path.exists(x509_path):
0149             ssl_cert_file = x509_path
0150     
0151     if not ssl_key_file or not os.path.exists(ssl_key_file):
0152         sys.stderr.write("no certificate private key file found, please specify one via $X509_USER_PROXY, $X509_USER_KEY or --ssl-key-file\n")
0153         sys.exit(2)
0154                   
0155     if not ssl_cert_file or not os.path.exists(ssl_cert_file):
0156         sys.stderr.write("no certificate public key file found, please specify one via $X509_USER_CERT or --ssl-cert-file\n")
0157         sys.exit(3)
0158 
0159     print("Using SSL private key", ssl_key_file)
0160     print("Using SSL public key", ssl_cert_file)
0161 
0162                                                                                   
0163 def checkFileName(fileName):
0164     regWhitespace = re.compile('.*\s.*')
0165     if regWhitespace.match(fileName):
0166         sys.stderr.write("whitespace detected!\n")
0167         return False
0168             
0169     regRelval=re.compile('.*relval.*')
0170     regCMSSW=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_.*')
0171     regCMSSWpre=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_pre[0-9]+_.*')
0172     if regRelval.match(fileName):
0173         # TODO check for pre-versions
0174         if not regCMSSW.match(fileName):
0175             print("no CMSSW")
0176     return True
0177     # DQM stuff
0178 
0179 def startUpload(url, filename):
0180     global ssl_key_file
0181     global ssl_cert_file
0182 
0183     print(url, filename)
0184     try:
0185         (headers, data) = \
0186                   upload(url,
0187                      { 'size': os.stat(filename).st_size,
0188                        'checksum': "md5:%s" % md5.new(file(filename).read()).hexdigest() },
0189                      { 'file': filename })
0190         print('Status code: ', headers.get("Dqm-Status-Code", "None"))
0191         print('Message:     ', headers.get("Dqm-Status-Message", "None"))
0192         print('Detail:      ', headers.get("Dqm-Status-Detail", "None"))
0193         print(data)
0194     except urllib2.HTTPError as e:
0195         print("ERROR", e)
0196         print('Status code: ', e.hdrs.get("Dqm-Status-Code", "None"))
0197         print('Message:     ', e.hdrs.get("Dqm-Status-Message", "None"))
0198         print('Detail:      ', e.hdrs.get("Dqm-Status-Detail", "None"))
0199         sys.exit(1)
0200 
0201 def getURL(filename, destination):
0202     filename = filename.split("/")[-1]
0203     regMC=re.compile('.*_R([0-9]*)__*')
0204     if regMC.match(filename):
0205         m = re.search('.*_R([0-9]*)(__.*).root',filename)
0206         runNr = m.group(1)
0207         dataset = m.group(2).replace("__","/")
0208     else:
0209         m = re.search('.*_R([0-9]*)(_?.*).root',filename)
0210         runNr = m.group(1)
0211         dataset = m.group(2).replace("__","/")
0212         if dataset=="":
0213             dataset="/Global/Online/ALL"
0214     if not runNr:
0215         runNr="1"
0216     if (int(runNr)==1):
0217         return destination+"start?workspace=summary;dataset="+dataset+";sampletype=offline_data"
0218     else:
0219         return destination+"start?workspace=summary;runnr="+runNr+";dataset="+dataset+";sampletype=online_data"
0220     
0221 def registerFileAtLogServer(filename, destination, tags):
0222     filename = filename.split("/")[-1]
0223     regMC=re.compile('.*_R([0-9]*)__*')
0224     if regMC.match(filename):
0225         m = re.search('.*_R([0-9]*)(__.*).root',filename)
0226         runNr = m.group(1)
0227         dataset = m.group(2).replace("__","/")
0228     else:
0229         m = re.search('.*_R([0-9]*)(_?.*).root',filename)
0230         runNr = m.group(1)
0231         dataset = m.group(2).replace("__","/")
0232         if dataset=="":
0233             dataset="/Global/Online/ALL"
0234     tempurl = "https://www-ekp.physik.uni-karlsruhe.de/~zeise/cgi-bin/register.py?run="+runNr+"&dataset="+dataset+"&filename="+filename+"&tags="+tags+"&instance="+destination
0235     print("Link that is used to register: ", tempurl)
0236     urllib.urlopen(tempurl)
0237 
0238 def main(args):
0239     global opts
0240     parser = optparse.OptionParser(add_help_option=False)
0241     parser.add_option("-h", "--help",          action="callback", callback=print_help),
0242     parser.add_option("",   "--no-filename-check", dest="no_filename_check",  default=False, action="store_true")
0243     parser.add_option("-d", "--destination", dest="destination", default="",  action="store")
0244     parser.add_option("-t", "--tags", dest="tags", default="",  action="store")
0245     parser.add_option("-s", "--no-submission", dest="submission", default=True,  action="store_false")
0246     parser.add_option("-r", "--no-registration", dest="registration", default=True,  action="store_false")
0247     parser.add_option("","--ssl-key-file", dest="ssl_key_file", default="", action="store")
0248     parser.add_option("","--ssl-cert-file", dest="ssl_cert_file", default="", action="store")
0249     (opts, args) = parser.parse_args()
0250     opts.abort = False
0251 
0252     if not opts.destination:
0253         sys.stderr.write("no destination specified\n")
0254         sys.exit(4)
0255     checkSSL(opts)
0256 
0257     if len(args)==0:
0258         sys.stderr.write("no input files specified\n")
0259         sys.exit(1)
0260     for fileName in args:
0261         fileName=fileName.strip()
0262         if not os.path.exists(fileName):
0263             sys.stderr.write("file '%s' doesn't exist!\n" % fileName)
0264             continue
0265         if not opts.no_filename_check and not checkFileName(fileName):
0266             continue
0267         sys.stderr.write("file '%s' passed name check, upload will follow!\n" % fileName)
0268         if opts.submission:
0269           startUpload(opts.destination, fileName)
0270         else:
0271           sys.stdout.write("file '%s' would be uploaded to '%s'\n" % (fileName, opts.destination))
0272         if opts.registration:
0273           registerFileAtLogServer(fileName, opts.destination, opts.tags)       
0274         print("You should see the plots here: "+getURL(fileName, opts.destination))
0275 
0276 if __name__ == '__main__':
0277     sys.exit(main(sys.argv[1:]))
0278     
0279