PyReleaseValidation/python/relval_machine.py

0001 from __future__ import print_function
0002 from  Configuration.PyReleaseValidation.relval_steps import Matrix, InputInfo, Steps
0003 import os
0004 import json
0005 import collections
0006
0007
0008 workflows = Matrix()
0009 steps = Steps()
0010
0011
0012 def get_json_files():
0013     cwd = os.path.join(os.getcwd(), "json_data")
0014     if not os.path.exists(cwd):
0015         return []
0016
0017     json_files = []
0018     for f in os.listdir(cwd):
0019         full_path = os.path.join(cwd, f)
0020         if os.path.isfile(full_path) and f.endswith(".json"):
0021             json_files.append(full_path)
0022     return json_files
0023
0024
0025 def fix_run(run):
0026     runs = run.replace(" ", "").replace("[", "").replace("]", "").split(",")
0027     int_runs = []
0028     for item in runs:
0029         if item.isdigit():
0030             int_runs.append(int(item))
0031         else:
0032             print("WARNING: run is in bad format: {0}".format(run))
0033     return int_runs
0034
0035 def convert_keys_to_string(dictionary):
0036     """ Recursively converts dictionary keys to strings.
0037         Utility to help deal with unicode keys in dictionaries created from json requests.
0038         In order to pass dict to function as **kwarg we should transform key/value to str.
0039     """
0040     if isinstance(dictionary, str):
0041         return str(dictionary)
0042     elif isinstance(dictionary, collections.Mapping):
0043         return dict(map(convert_keys_to_string, dictionary.items()))
0044     elif isinstance(dictionary, collections.Iterable):
0045         return type(dictionary)(map(convert_keys_to_string, dictionary))
0046     else:
0047         return dictionary
0048
0049 def load_steps_and_workflows():
0050     data_files = get_json_files()
0051     for index, data_file in enumerate(data_files):
0052         with open(data_file, "r") as f:
0053             data = json.load(f)
0054             data = convert_keys_to_string(data)
0055             label = data["label"]
0056             steps_names = []
0057             for step_name, step in data["steps"].items():
0058                 steps_names.append((step_name, step["sequence_number"]))
0059                 if step_name in steps:
0060                     continue  # this step was inserted already
0061
0062                 # inputInfo case
0063                 if "inputInfo" in step:
0064                     input_info = step["inputInfo"]
0065                     if "run" in input_info:
0066                         input_info["run"] = fix_run(input_info["run"])
0067
0068                     steps[step_name] = {
0069                         'INPUT': InputInfo(**input_info)
0070                     }
0071                 # step with parameters
0072                 elif "parameters" in step:
0073                     steps[step_name] = step["parameters"]
0074                 else:
0075                     raise Exception("Wrong step format in {0} file".format(data_file))
0076
0077             sorted_steps = sorted(steps_names, key=lambda step: step[1]) # sort steps by sequence number
0078             sorted_steps_names = [step_name[0] for step_name in sorted_steps]  # filter only step names
0079
0080             workflows[1000000.0 + 0.1*index] = [label, sorted_steps_names]
0081
0082
0083 load_steps_and_workflows()
0084