File indexing completed on 2024-11-25 02:29:10
0001
0002
0003 import sys, os, os.path, re, string, httplib, mimetypes, urllib, urllib2, httplib, gzip, md5
0004 from cStringIO import StringIO
0005 from stat import *
0006 import optparse
0007
0008 try:
0009 import hashlib
0010 except:
0011 pass
0012
0013 HTTPS = httplib.HTTPS
0014 if sys.version_info[:3] >= (2, 4, 0):
0015 HTTPS = httplib.HTTPSConnection
0016
0017 ssl_key_file = None
0018 ssl_cert_file = None
0019
0020 class HTTPSCertAuth(HTTPS):
0021 def __init__(self, host, *args, **kwargs):
0022 HTTPS.__init__(self, host, key_file = ssl_key_file, cert_file = ssl_cert_file, **kwargs)
0023
0024 class HTTPSCertAuthenticate(urllib2.AbstractHTTPHandler):
0025 def default_open(self, req):
0026 return self.do_open(HTTPSCertAuth, req)
0027
0028 def filetype(filename):
0029 return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
0030
0031 def encode(args, files):
0032 """
0033 Encode form (name, value) and (name, filename, type) elements into
0034 multi-part/form-data. We don't actually need to know what we are
0035 uploading here, so just claim it's all text/plain.
0036 """
0037 boundary = '----------=_DQM_FILE_BOUNDARY_=-----------'
0038 (body, crlf) = ('', '\r\n')
0039 for (key, value) in args.items():
0040 body += '--' + boundary + crlf
0041 body += ('Content-disposition: form-data; name="%s"' % key) + crlf
0042 body += crlf + str(value) + crlf
0043 for (key, filename) in files.items():
0044 body += '--' + boundary + crlf
0045 body += ('Content-Disposition: form-data; name="%s"; filename="%s"'
0046 % (key, os.path.basename(filename))) + crlf
0047 body += ('Content-Type: %s' % filetype(filename)) + crlf
0048 body += crlf + open(filename, "r").read() + crlf
0049 body += '--' + boundary + '--' + crlf + crlf
0050 return ('multipart/form-data; boundary=' + boundary, body)
0051
0052 def marshall(args, files, request):
0053 """
0054 Marshalls the arguments to the CGI script as multi-part/form-data,
0055 not the default application/x-www-form-url-encoded. This improves
0056 the transfer of the large inputs and eases command line invocation
0057 of the CGI script.
0058 """
0059 (type, body) = encode(args, files)
0060 request.add_header('Content-type', type)
0061 request.add_header('Content-length', str(len(body)))
0062 request.add_data(body)
0063
0064 def upload(url, args, files):
0065 ident = "visDQMUpload DQMGUI/%s CMSSW/%s python/%s" % \
0066 (os.getenv('DQMGUI_VERSION', '?'),
0067 os.getenv('DQM_CMSSW_VERSION', os.getenv('CMSSW_VERSION', '?')),
0068 "%d.%d.%d" % sys.version_info[:3])
0069 cookie = None
0070 if url.startswith("https:"):
0071 authreq = urllib2.Request(url + '/authenticate')
0072 authreq.add_header('User-agent', ident)
0073 result = urllib2.build_opener(HTTPSCertAuthenticate()).open(authreq)
0074 cookie = result.headers.get('Set-Cookie')
0075 if not cookie:
0076 raise RuntimeError("Did not receive authentication cookie")
0077 cookie = cookie.split(";")[0]
0078
0079 datareq = urllib2.Request(url + '/data/put')
0080 datareq.add_header('Accept-encoding', 'gzip')
0081 datareq.add_header('User-agent', ident)
0082 if cookie:
0083 datareq.add_header('Cookie', cookie)
0084 marshall(args, files, datareq)
0085 result = urllib2.build_opener().open(datareq)
0086 data = result.read()
0087 if result.headers.get ('Content-encoding', '') == 'gzip':
0088 data = gzip.GzipFile (fileobj=StringIO(data)).read ()
0089 return (result.headers, data)
0090
0091 def print_help(*args):
0092 sys.stdout.write("\n")
0093 sys.stdout.write("This scripts intends to do the upload of files to the DQM server. It\n")
0094 sys.stdout.write("runs some basic checks on the file name as it is crucial to follow\n")
0095 sys.stdout.write("the naming convention.\n")
0096 sys.stdout.write("\n")
0097 sys.stdout.write("Mandatory Option\n")
0098 sys.stdout.write(" -d, --destination parameter to specify the DQM server\n")
0099 sys.stdout.write("\n")
0100 sys.stdout.write(" Proxy Options\n")
0101 sys.stdout.write(" The script will try to find your grid proxy automatically. To\n")
0102 sys.stdout.write(" do so, it checks $X509_* environment variables and your globus\n")
0103 sys.stdout.write(" directory (~/.globus). To override this automatism, you can use\n")
0104 sys.stdout.write(" the following two options:\n")
0105 sys.stdout.write(" --ssl-key-file location of your private key\n")
0106 sys.stdout.write(" --ssl-cert-file location of your public key\n")
0107 sys.stdout.write("\n")
0108 sys.stdout.write("Other Options\n")
0109 sys.stdout.write(" -h, --help show this help message\n")
0110 sys.stdout.write(" -s, --no-submission suppress the submission\n")
0111 sys.stdout.write(" -r, --no-registration suppress the submission\n")
0112 sys.stdout.write(" --no-filename-check omit the file name check\n")
0113 sys.stdout.write("\n")
0114 sys.exit(0)
0115
0116 def checkSSL(opts):
0117 global ssl_key_file
0118 global ssl_cert_file
0119
0120 if opts.ssl_key_file and os.path.exists(opts.ssl_key_file):
0121 ssl_key_file = opts.ssl_key_file
0122 if opts.ssl_cert_file and os.path.exists(opts.ssl_cert_file):
0123 ssl_cert_file = opts.ssl_cert_file
0124
0125 if not ssl_key_file:
0126 x509_path = os.getenv("X509_USER_PROXY", None)
0127 if x509_path and os.path.exists(x509_path):
0128 ssl_key_file = ssl_cert_file = x509_path
0129
0130 if not ssl_key_file:
0131 x509_path = os.getenv("X509_USER_KEY", None)
0132 if x509_path and os.path.exists(x509_path):
0133 ssl_key_file = x509_path
0134
0135 if not ssl_cert_file:
0136 x509_path = os.getenv("X509_USER_CERT", None)
0137 if x509_path and os.path.exists(x509_path):
0138 ssl_cert_file = x509_path
0139
0140 if not ssl_key_file:
0141 x509_path = os.getenv("HOME") + "/.globus/userkey.pem"
0142 if os.path.exists(x509_path):
0143 ssl_key_file = x509_path
0144
0145 if not ssl_cert_file:
0146 x509_path = os.getenv("HOME") + "/.globus/usercert.pem"
0147 if os.path.exists(x509_path):
0148 ssl_cert_file = x509_path
0149
0150 if not ssl_key_file or not os.path.exists(ssl_key_file):
0151 sys.stderr.write("no certificate private key file found, please specify one via $X509_USER_PROXY, $X509_USER_KEY or --ssl-key-file\n")
0152 sys.exit(2)
0153
0154 if not ssl_cert_file or not os.path.exists(ssl_cert_file):
0155 sys.stderr.write("no certificate public key file found, please specify one via $X509_USER_CERT or --ssl-cert-file\n")
0156 sys.exit(3)
0157
0158 print("Using SSL private key", ssl_key_file)
0159 print("Using SSL public key", ssl_cert_file)
0160
0161
0162 def checkFileName(fileName):
0163 regWhitespace = re.compile('.*\s.*')
0164 if regWhitespace.match(fileName):
0165 sys.stderr.write("whitespace detected!\n")
0166 return False
0167
0168 regRelval=re.compile('.*relval.*')
0169 regCMSSW=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_.*')
0170 regCMSSWpre=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_pre[0-9]+_.*')
0171 if regRelval.match(fileName):
0172
0173 if not regCMSSW.match(fileName):
0174 print("no CMSSW")
0175 return True
0176
0177
0178 def startUpload(url, filename):
0179 global ssl_key_file
0180 global ssl_cert_file
0181
0182 print(url, filename)
0183 try:
0184 (headers, data) = \
0185 upload(url,
0186 { 'size': os.stat(filename).st_size,
0187 'checksum': "md5:%s" % md5.new(file(filename).read()).hexdigest() },
0188 { 'file': filename })
0189 print('Status code: ', headers.get("Dqm-Status-Code", "None"))
0190 print('Message: ', headers.get("Dqm-Status-Message", "None"))
0191 print('Detail: ', headers.get("Dqm-Status-Detail", "None"))
0192 print(data)
0193 except urllib2.HTTPError as e:
0194 print("ERROR", e)
0195 print('Status code: ', e.hdrs.get("Dqm-Status-Code", "None"))
0196 print('Message: ', e.hdrs.get("Dqm-Status-Message", "None"))
0197 print('Detail: ', e.hdrs.get("Dqm-Status-Detail", "None"))
0198 sys.exit(1)
0199
0200 def getURL(filename, destination):
0201 filename = filename.split("/")[-1]
0202 regMC=re.compile('.*_R([0-9]*)__*')
0203 if regMC.match(filename):
0204 m = re.search('.*_R([0-9]*)(__.*).root',filename)
0205 runNr = m.group(1)
0206 dataset = m.group(2).replace("__","/")
0207 else:
0208 m = re.search('.*_R([0-9]*)(_?.*).root',filename)
0209 runNr = m.group(1)
0210 dataset = m.group(2).replace("__","/")
0211 if dataset=="":
0212 dataset="/Global/Online/ALL"
0213 if not runNr:
0214 runNr="1"
0215 if (int(runNr)==1):
0216 return destination+"start?workspace=summary;dataset="+dataset+";sampletype=offline_data"
0217 else:
0218 return destination+"start?workspace=summary;runnr="+runNr+";dataset="+dataset+";sampletype=online_data"
0219
0220 def registerFileAtLogServer(filename, destination, tags):
0221 filename = filename.split("/")[-1]
0222 regMC=re.compile('.*_R([0-9]*)__*')
0223 if regMC.match(filename):
0224 m = re.search('.*_R([0-9]*)(__.*).root',filename)
0225 runNr = m.group(1)
0226 dataset = m.group(2).replace("__","/")
0227 else:
0228 m = re.search('.*_R([0-9]*)(_?.*).root',filename)
0229 runNr = m.group(1)
0230 dataset = m.group(2).replace("__","/")
0231 if dataset=="":
0232 dataset="/Global/Online/ALL"
0233 tempurl = "https://www-ekp.physik.uni-karlsruhe.de/~zeise/cgi-bin/register.py?run="+runNr+"&dataset="+dataset+"&filename="+filename+"&tags="+tags+"&instance="+destination
0234 print("Link that is used to register: ", tempurl)
0235 urllib.urlopen(tempurl)
0236
0237 def main(args):
0238 global opts
0239 parser = optparse.OptionParser(add_help_option=False)
0240 parser.add_option("-h", "--help", action="callback", callback=print_help),
0241 parser.add_option("", "--no-filename-check", dest="no_filename_check", default=False, action="store_true")
0242 parser.add_option("-d", "--destination", dest="destination", default="", action="store")
0243 parser.add_option("-t", "--tags", dest="tags", default="", action="store")
0244 parser.add_option("-s", "--no-submission", dest="submission", default=True, action="store_false")
0245 parser.add_option("-r", "--no-registration", dest="registration", default=True, action="store_false")
0246 parser.add_option("","--ssl-key-file", dest="ssl_key_file", default="", action="store")
0247 parser.add_option("","--ssl-cert-file", dest="ssl_cert_file", default="", action="store")
0248 (opts, args) = parser.parse_args()
0249 opts.abort = False
0250
0251 if not opts.destination:
0252 sys.stderr.write("no destination specified\n")
0253 sys.exit(4)
0254 checkSSL(opts)
0255
0256 if len(args)==0:
0257 sys.stderr.write("no input files specified\n")
0258 sys.exit(1)
0259 for fileName in args:
0260 fileName=fileName.strip()
0261 if not os.path.exists(fileName):
0262 sys.stderr.write("file '%s' doesn't exist!\n" % fileName)
0263 continue
0264 if not opts.no_filename_check and not checkFileName(fileName):
0265 continue
0266 sys.stderr.write("file '%s' passed name check, upload will follow!\n" % fileName)
0267 if opts.submission:
0268 startUpload(opts.destination, fileName)
0269 else:
0270 sys.stdout.write("file '%s' would be uploaded to '%s'\n" % (fileName, opts.destination))
0271 if opts.registration:
0272 registerFileAtLogServer(fileName, opts.destination, opts.tags)
0273 print("You should see the plots here: "+getURL(fileName, opts.destination))
0274
0275 if __name__ == '__main__':
0276 sys.exit(main(sys.argv[1:]))
0277
0278
