Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-25 02:29:50

0001 #!/usr/bin/env python3
0002 #pylint: disable-msg=C0301,C0103
0003 
0004 """
0005 DAS command line tool
0006 """
0007 __author__ = "Valentin Kuznetsov"
0008 
0009 import sys
0010 if  sys.version_info < (2, 6):
0011     raise Exception("DAS requires python 2.6 or greater")
0012 
0013 import re
0014 import time
0015 import json
0016 import urllib
0017 import urllib2
0018 from   optparse import OptionParser, OptionGroup
0019 
0020 class DASOptionParser: 
0021     """
0022     DAS cache client option parser
0023     """
0024     def __init__(self, usage = None):
0025         if usage is None:
0026             usage = 'usage: %prog [options] --query "dataset=/HT/Run2011A-*/AOD"'
0027         self.parser = OptionParser(usage=usage)
0028         group = OptionGroup(self.parser,"Das options","These options relate to the Das client interface.")
0029         group.add_option("-v", "--verbose", action="store", 
0030                                type="int", default=0, dest="verbose",
0031              help="verbose output")
0032         group.add_option("--query", action="store", type="string", 
0033                                default=False, dest="query",
0034              help="specify query for your request")
0035         group.add_option("--host", action="store", type="string", 
0036                                default='https://cmsweb.cern.ch', dest="host",
0037              help="specify host name of DAS cache server, default https://cmsweb.cern.ch")
0038         group.add_option("--idx", action="store", type="int", 
0039                                default=0, dest="idx",
0040              help="start index for returned result set, aka pagination, use w/ limit")
0041         group.add_option("--limit", action="store", type="int", 
0042                                default=10, dest="limit",
0043              help="number of returned results (results per page)")
0044         group.add_option("--format", action="store", type="string", 
0045                                default="json", dest="format",
0046              help="specify return data format (json or plain), default json")
0047         self.parser.add_option_group(group)
0048     def get_opt(self):
0049         """
0050         Returns parse list of options
0051         """
0052         return self.parser.parse_args()
0053 
0054 def get_value(data, filters):
0055     """Filter data from a row for given list of filters"""
0056     for ftr in filters:
0057         if  ftr.find('>') != -1 or ftr.find('<') != -1 or ftr.find('=') != -1:
0058             continue
0059         row = dict(data)
0060         for key in ftr.split('.'):
0061             if  isinstance(row, dict) and key in row:
0062                 row = row[key]
0063             if  isinstance(row, list):
0064                 for item in row:
0065                     if  isinstance(item, dict) and key in item:
0066                         row = item[key]
0067                         break
0068         yield str(row)
0069 
0070 def get_data(host, query, idx, limit, debug):
0071     """Contact DAS server and retrieve data for given DAS query"""
0072     params  = {'input':query, 'idx':idx, 'limit':limit}
0073     path    = '/das/cache'
0074     pat     = re.compile('http[s]{0,1}://')
0075     if  not pat.match(host):
0076         msg = 'Invalid hostname: %s' % host
0077         raise Exception(msg)
0078     url = host + path
0079     headers = {"Accept": "application/json"}
0080     encoded_data = urllib.urlencode(params, doseq=True)
0081     url += '?%s' % encoded_data
0082     req  = urllib2.Request(url=url, headers=headers)
0083     if  debug:
0084         hdlr = urllib2.HTTPHandler(debuglevel=1)
0085         opener = urllib2.build_opener(hdlr)
0086     else:
0087         opener = urllib2.build_opener()
0088     fdesc = opener.open(req)
0089     data = fdesc.read()
0090     fdesc.close()
0091 
0092     pat = re.compile(r'^[a-z0-9]{32}')
0093     if  data and isinstance(data, str) and pat.match(data) and len(data) == 32:
0094         pid = data
0095     else:
0096         pid = None
0097     count = 5  # initial waiting time in seconds
0098     timeout = 30 # final waiting time in seconds
0099     while pid:
0100         params.update({'pid':data})
0101         encoded_data = urllib.urlencode(params, doseq=True)
0102         url  = host + path + '?%s' % encoded_data
0103         req  = urllib2.Request(url=url, headers=headers)
0104         try:
0105             fdesc = opener.open(req)
0106             data = fdesc.read()
0107             fdesc.close()
0108         except urllib2.HTTPError as err:
0109             print(err)
0110             return ""
0111         if  data and isinstance(data, str) and pat.match(data) and len(data) == 32:
0112             pid = data
0113         else:
0114             pid = None
0115         time.sleep(count)
0116         if  count < timeout:
0117             count *= 2
0118         else:
0119             count = timeout
0120     return data
0121 
0122 def main():
0123     """Main function"""
0124     optmgr  = DASOptionParser()
0125     opts, _ = optmgr.get_opt()
0126     host    = opts.host
0127     debug   = opts.verbose
0128     query   = opts.query
0129     idx     = opts.idx
0130     limit   = opts.limit
0131     if  not query:
0132         raise Exception('You must provide input query')
0133     data    = get_data(host, query, idx, limit, debug)
0134     if  opts.format == 'plain':
0135         jsondict = json.loads(data)
0136         mongo_query = jsondict['mongo_query']
0137         if  'filters' in mongo_query:
0138             filters = mongo_query['filters']
0139             data = jsondict['data']
0140             if  isinstance(data, dict):
0141                 rows = [r for r in get_value(data, filters)]
0142                 print(' '.join(rows))
0143             elif isinstance(data, list):
0144                 for row in data:
0145                     rows = [r for r in get_value(row, filters)]
0146                     print(' '.join(rows))
0147             else:
0148                 print(jsondict)
0149     else:
0150         print(data)
0151