PyReleaseValidation/python/relval_machine.py

0001 from  Configuration.PyReleaseValidation.relval_steps import Matrix, InputInfo, Steps
0002 import os
0003 import json
0004 import collections
0005
0006
0007 workflows = Matrix()
0008 steps = Steps()
0009
0010
0011 def get_json_files():
0012     cwd = os.path.join(os.getcwd(), "json_data")
0013     if not os.path.exists(cwd):
0014         return []
0015
0016     json_files = []
0017     for f in os.listdir(cwd):
0018         full_path = os.path.join(cwd, f)
0019         if os.path.isfile(full_path) and f.endswith(".json"):
0020             json_files.append(full_path)
0021     return json_files
0022
0023
0024 def fix_run(run):
0025     runs = run.replace(" ", "").replace("[", "").replace("]", "").split(",")
0026     int_runs = []
0027     for item in runs:
0028         if item.isdigit():
0029             int_runs.append(int(item))
0030         else:
0031             print("WARNING: run is in bad format: {0}".format(run))
0032     return int_runs
0033
0034 def convert_keys_to_string(dictionary):
0035     """ Recursively converts dictionary keys to strings.
0036         Utility to help deal with unicode keys in dictionaries created from json requests.
0037         In order to pass dict to function as **kwarg we should transform key/value to str.
0038     """
0039     if isinstance(dictionary, str):
0040         return str(dictionary)
0041     elif isinstance(dictionary, collections.Mapping):
0042         return dict(map(convert_keys_to_string, dictionary.items()))
0043     elif isinstance(dictionary, collections.Iterable):
0044         return type(dictionary)(map(convert_keys_to_string, dictionary))
0045     else:
0046         return dictionary
0047
0048 def load_steps_and_workflows():
0049     data_files = get_json_files()
0050     for index, data_file in enumerate(data_files):
0051         with open(data_file, "r") as f:
0052             data = json.load(f)
0053             data = convert_keys_to_string(data)
0054             label = data["label"]
0055             steps_names = []
0056             for step_name, step in data["steps"].items():
0057                 steps_names.append((step_name, step["sequence_number"]))
0058                 if step_name in steps:
0059                     continue  # this step was inserted already
0060
0061                 # inputInfo case
0062                 if "inputInfo" in step:
0063                     input_info = step["inputInfo"]
0064                     if "run" in input_info:
0065                         input_info["run"] = fix_run(input_info["run"])
0066
0067                     steps[step_name] = {
0068                         'INPUT': InputInfo(**input_info)
0069                     }
0070                 # step with parameters
0071                 elif "parameters" in step:
0072                     steps[step_name] = step["parameters"]
0073                 else:
0074                     raise Exception("Wrong step format in {0} file".format(data_file))
0075
0076             sorted_steps = sorted(steps_names, key=lambda step: step[1]) # sort steps by sequence number
0077             sorted_steps_names = [step_name[0] for step_name in sorted_steps]  # filter only step names
0078
0079             workflows[1000000.0 + 0.1*index] = [label, sorted_steps_names]
0080
0081
0082 load_steps_and_workflows()
0083