File indexing completed on 2024-11-25 02:29:50
0001
0002
0003
0004 """
0005 DAS command line tool
0006 """
0007 __author__ = "Valentin Kuznetsov"
0008
0009 import sys
0010 if sys.version_info < (2, 6):
0011 raise Exception("DAS requires python 2.6 or greater")
0012
0013 import re
0014 import time
0015 import json
0016 import urllib
0017 import urllib2
0018 from optparse import OptionParser, OptionGroup
0019
0020 class DASOptionParser:
0021 """
0022 DAS cache client option parser
0023 """
0024 def __init__(self, usage = None):
0025 if usage is None:
0026 usage = 'usage: %prog [options] --query "dataset=/HT/Run2011A-*/AOD"'
0027 self.parser = OptionParser(usage=usage)
0028 group = OptionGroup(self.parser,"Das options","These options relate to the Das client interface.")
0029 group.add_option("-v", "--verbose", action="store",
0030 type="int", default=0, dest="verbose",
0031 help="verbose output")
0032 group.add_option("--query", action="store", type="string",
0033 default=False, dest="query",
0034 help="specify query for your request")
0035 group.add_option("--host", action="store", type="string",
0036 default='https://cmsweb.cern.ch', dest="host",
0037 help="specify host name of DAS cache server, default https://cmsweb.cern.ch")
0038 group.add_option("--idx", action="store", type="int",
0039 default=0, dest="idx",
0040 help="start index for returned result set, aka pagination, use w/ limit")
0041 group.add_option("--limit", action="store", type="int",
0042 default=10, dest="limit",
0043 help="number of returned results (results per page)")
0044 group.add_option("--format", action="store", type="string",
0045 default="json", dest="format",
0046 help="specify return data format (json or plain), default json")
0047 self.parser.add_option_group(group)
0048 def get_opt(self):
0049 """
0050 Returns parse list of options
0051 """
0052 return self.parser.parse_args()
0053
0054 def get_value(data, filters):
0055 """Filter data from a row for given list of filters"""
0056 for ftr in filters:
0057 if ftr.find('>') != -1 or ftr.find('<') != -1 or ftr.find('=') != -1:
0058 continue
0059 row = dict(data)
0060 for key in ftr.split('.'):
0061 if isinstance(row, dict) and key in row:
0062 row = row[key]
0063 if isinstance(row, list):
0064 for item in row:
0065 if isinstance(item, dict) and key in item:
0066 row = item[key]
0067 break
0068 yield str(row)
0069
0070 def get_data(host, query, idx, limit, debug):
0071 """Contact DAS server and retrieve data for given DAS query"""
0072 params = {'input':query, 'idx':idx, 'limit':limit}
0073 path = '/das/cache'
0074 pat = re.compile('http[s]{0,1}://')
0075 if not pat.match(host):
0076 msg = 'Invalid hostname: %s' % host
0077 raise Exception(msg)
0078 url = host + path
0079 headers = {"Accept": "application/json"}
0080 encoded_data = urllib.urlencode(params, doseq=True)
0081 url += '?%s' % encoded_data
0082 req = urllib2.Request(url=url, headers=headers)
0083 if debug:
0084 hdlr = urllib2.HTTPHandler(debuglevel=1)
0085 opener = urllib2.build_opener(hdlr)
0086 else:
0087 opener = urllib2.build_opener()
0088 fdesc = opener.open(req)
0089 data = fdesc.read()
0090 fdesc.close()
0091
0092 pat = re.compile(r'^[a-z0-9]{32}')
0093 if data and isinstance(data, str) and pat.match(data) and len(data) == 32:
0094 pid = data
0095 else:
0096 pid = None
0097 count = 5
0098 timeout = 30
0099 while pid:
0100 params.update({'pid':data})
0101 encoded_data = urllib.urlencode(params, doseq=True)
0102 url = host + path + '?%s' % encoded_data
0103 req = urllib2.Request(url=url, headers=headers)
0104 try:
0105 fdesc = opener.open(req)
0106 data = fdesc.read()
0107 fdesc.close()
0108 except urllib2.HTTPError as err:
0109 print(err)
0110 return ""
0111 if data and isinstance(data, str) and pat.match(data) and len(data) == 32:
0112 pid = data
0113 else:
0114 pid = None
0115 time.sleep(count)
0116 if count < timeout:
0117 count *= 2
0118 else:
0119 count = timeout
0120 return data
0121
0122 def main():
0123 """Main function"""
0124 optmgr = DASOptionParser()
0125 opts, _ = optmgr.get_opt()
0126 host = opts.host
0127 debug = opts.verbose
0128 query = opts.query
0129 idx = opts.idx
0130 limit = opts.limit
0131 if not query:
0132 raise Exception('You must provide input query')
0133 data = get_data(host, query, idx, limit, debug)
0134 if opts.format == 'plain':
0135 jsondict = json.loads(data)
0136 mongo_query = jsondict['mongo_query']
0137 if 'filters' in mongo_query:
0138 filters = mongo_query['filters']
0139 data = jsondict['data']
0140 if isinstance(data, dict):
0141 rows = [r for r in get_value(data, filters)]
0142 print(' '.join(rows))
0143 elif isinstance(data, list):
0144 for row in data:
0145 rows = [r for r in get_value(row, filters)]
0146 print(' '.join(rows))
0147 else:
0148 print(jsondict)
0149 else:
0150 print(data)
0151