File indexing completed on 2023-03-17 11:26:46
0001 import time
0002 import os
0003 from json import loads, dumps
0004 from types import GeneratorType
0005 import subprocess
0006
0007
0008
0009 def convert_time(val):
0010 "Convert given timestamp into human readable format"
0011 if isinstance(val, int) or isinstance(val, float):
0012 return time.strftime('%d/%b/%Y_%H:%M:%S_GMT', time.gmtime(val))
0013 return val
0014
0015 def size_format(uinput, ibase=0):
0016 """
0017 Format file size utility, it converts file size into KB, MB, GB, TB, PB units
0018 """
0019 if not ibase:
0020 return uinput
0021 try:
0022 num = float(uinput)
0023 except Exception as _exc:
0024 return uinput
0025 if ibase == 2.:
0026 base = 1024.
0027 xlist = ['', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']
0028 else:
0029 base = 1000.
0030 xlist = ['', 'KB', 'MB', 'GB', 'TB', 'PB']
0031 for xxx in xlist:
0032 if num < base:
0033 return "%3.1f%s" % (num, xxx)
0034 num /= base
0035
0036 def extract_value(row, key, base=10):
0037 """Generator which extracts row[key] value"""
0038 if isinstance(row, dict) and key in row:
0039 if key == 'creation_time':
0040 row = convert_time(row[key])
0041 elif key == 'size':
0042 row = size_format(row[key], base)
0043 else:
0044 row = row[key]
0045 yield row
0046 if isinstance(row, list) or isinstance(row, GeneratorType):
0047 for item in row:
0048 for vvv in extract_value(item, key, base):
0049 yield vvv
0050
0051 def get_value(data, filters, base=10):
0052 """Filter data from a row for given list of filters"""
0053 for ftr in filters:
0054 if ftr.find('>') != -1 or ftr.find('<') != -1 or ftr.find('=') != -1:
0055 continue
0056 row = dict(data)
0057 values = []
0058 keys = ftr.split('.')
0059 for key in keys:
0060 val = [v for v in extract_value(row, key, base)]
0061 if key == keys[-1]:
0062 values += [dumps(i) for i in val]
0063 else:
0064 row = val
0065 if len(values) == 1:
0066 yield values[0]
0067 else:
0068 yield values
0069
0070 def get_data(query, limit=None, threshold=None, idx=None, host=None, cmd=None):
0071 cmd_opts = "--format=json"
0072 if threshold is not None: cmd_opts += " --threshold=%s" % threshold
0073 if limit is not None: cmd_opts += " --limit=%s" % limit
0074 if idx is not None: cmd_opts += " --idx=%s" % idx
0075 if host is not None: cmd_opts += " --host=%s" % host
0076 if not cmd:
0077 cmd = "das_client"
0078 for path in os.getenv('PATH').split(':'):
0079 if os.path.isfile(os.path.join(path, 'dasgoclient')):
0080 cmd = "dasgoclient"
0081 break
0082
0083 p = subprocess.Popen("%s %s --query '%s'" % (cmd, cmd_opts, query),shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
0084 stdout, stderr = p.communicate()
0085 if not p.returncode: return loads(stdout)
0086 return {'status' : 'error', 'reason' : stdout}