From 144912deb8338799183d04a369a5a2e798afb525 Mon Sep 17 00:00:00 2001 From: michiboo Date: Fri, 28 Jul 2023 20:12:04 +0300 Subject: [PATCH] test: [RLOS2023][WIP] add option for storing output and grid language redefinition (#4627) * test: redesign grid lang * test: add option for store output * test: change list to dict for config vars --- python/tests/test_framework/conftest.py | 18 ++ .../tests/test_framework/test_configs/cb.json | 285 +++++++----------- .../test_configs/classification.json | 14 +- .../test_configs/regression.json | 151 ++++------ python/tests/test_framework/test_core.py | 33 +- python/tests/test_framework/test_helper.py | 84 ++---- 6 files changed, 239 insertions(+), 346 deletions(-) create mode 100644 python/tests/test_framework/conftest.py diff --git a/python/tests/test_framework/conftest.py b/python/tests/test_framework/conftest.py new file mode 100644 index 00000000000..7c53cc6e3bb --- /dev/null +++ b/python/tests/test_framework/conftest.py @@ -0,0 +1,18 @@ +import pytest + +# conftest.py +def pytest_addoption(parser): + parser.addoption( + "--store_output", + action="store", + default=False, + help="Store output file for tests.", + ) + + +def pytest_configure(config): + _store_output = config.getoption("--store_output") + # Store the custom_arg_value in a global variable or a custom configuration object. + # For example, you can store it in a global variable like this: + global STORE_OUTPUT + STORE_OUTPUT = _store_output diff --git a/python/tests/test_framework/test_configs/cb.json b/python/tests/test_framework/test_configs/cb.json index 56a1d7b59e7..880141d444c 100644 --- a/python/tests/test_framework/test_configs/cb.json +++ b/python/tests/test_framework/test_configs/cb.json @@ -43,96 +43,62 @@ } } ], - "grids": [ - { + "grids": { + "g0": { "#base": [ "--cb_explore 2" ] }, - [ - {"*": [ - { - "+": [ - { - "--epsilon": [ - 0.1, - 0.2, - 0.3 - ] - } - ] - }, - { - "+": [ - { - "--first": [ - 1, - 2 - ] - } - ] - }, - { - "+": [ - { - "--bag": [ - 5, - 6, - 7 - ] - } - ] - }, - { - "+": [ - { - "--cover": [ - 1, - 2, - 3 - ] - } - ] - }, - { - "+": [ - { - "--squarecb": [ - "--gamma_scale 1000", - "--gamma_scale 10000" - ] - } - ] - }, - { - "+": [ - { - "--synthcover": [ - "" - ] - } - ] - }, - { - "+": [ - { - "--regcb": [ - "" - ] - } - ] - }, - { - "+": [ - { - "--softmax": [ - "" - ] - } - ] - } - ]} - ]], + "g1": { + "--epsilon": [ + 0.1, + 0.2, + 0.3 + ] + }, + "g2": { + "--first": [ + 1, + 2 + ] + }, + "g3": { + "--bag": [ + 5, + 6, + 7 + ] + }, + "g4": { + "--cover": [ + 1, + 2, + 3 + ] + }, + "g5": { + "--squarecb": [ + "--gamma_scale 1000", + "--gamma_scale 10000" + ] + }, + "g6": { + "--synthcover": [ + "" + ] + }, + "g7": { + "--regcb": [ + "" + ] + }, + "g8": { + "--softmax": [ + "" + ] + } + }, + "grids_expression": "g0 * (g1 + g2 + g3 + g4 + g5 +g6 + g7 + g8)", "output": [ "--readable_model", "-p" @@ -173,13 +139,13 @@ } } ], - "grids": [ - { + "grids": { + "g0": { "#base": [ "--cb 1 --preserve_performance_counters --save_resume" ] }, - { + "g1": { "--cb_type": [ "ips", "mtr", @@ -187,7 +153,8 @@ "dm" ] } - ], + }, + "grids_expression": "g0 * g1", "output": [ "--readable_model", "-p" @@ -235,96 +202,62 @@ } } ], - "grids": [ - { + "grids": { + "g0": { "#base": [ "--cb_explore_adf" ] }, - [ - {"*": [ - { - "+": [ - { - "--epsilon": [ - 0.1, - 0.2, - 0.3 - ] - } - ] - }, - { - "+": [ - { - "--first": [ - 1, - 2 - ] - } - ] - }, - { - "+": [ - { - "--bag": [ - 5, - 6, - 7 - ] - } - ] - }, - { - "+": [ - { - "--cover": [ - 1, - 2, - 3 - ] - } - ] - }, - { - "+": [ - { - "--squarecb": [ - "--gamma_scale 1000", - "--gamma_scale 10000" - ] - } - ] - }, - { - "+": [ - { - "--synthcover": [ - "" - ] - } - ] - }, - { - "+": [ - { - "--regcb": [ - "" - ] - } - ] - }, - { - "+": [ - { - "--softmax": [ - "" - ] - } - ] - } - ]} - ]], + "g1": { + "--epsilon": [ + 0.1, + 0.2, + 0.3 + ] + }, + "g2": { + "--first": [ + 1, + 2 + ] + }, + "g3": { + "--bag": [ + 5, + 6, + 7 + ] + }, + "g4": { + "--cover": [ + 1, + 2, + 3 + ] + }, + "g5": { + "--squarecb": [ + "--gamma_scale 1000", + "--gamma_scale 10000" + ] + }, + "g6": { + "--synthcover": [ + "" + ] + }, + "g7": { + "--regcb": [ + "" + ] + }, + "g8": { + "--softmax": [ + "" + ] + } + }, + "grids_expression": "g0 * (g1 + g2 + g3 + g4 + g5 +g6 + g7 + g8)", "output": [ "--readable_model", "-p" diff --git a/python/tests/test_framework/test_configs/classification.json b/python/tests/test_framework/test_configs/classification.json index 2ad2afbfbba..bcb269f8910 100644 --- a/python/tests/test_framework/test_configs/classification.json +++ b/python/tests/test_framework/test_configs/classification.json @@ -39,13 +39,14 @@ } } ], - "grids": [ - { + "grids": { + "g0": { "#base": [ "--oaa 3" ] } - ], + }, + "grids_expression": "g0", "output": [ "--readable_model", "-p" @@ -85,13 +86,14 @@ } } ], - "grids": [ - { + "grids": { + "g0": { "#base": [ "--oaa 25" ] } - ], + }, + "grids_expression": "g0", "output": [ "--readable_model", "-p" diff --git a/python/tests/test_framework/test_configs/regression.json b/python/tests/test_framework/test_configs/regression.json index 074c43a4add..4a6aac73944 100644 --- a/python/tests/test_framework/test_configs/regression.json +++ b/python/tests/test_framework/test_configs/regression.json @@ -31,99 +31,72 @@ } } ], - "grids": [ - { + "grids": { + "g0": { "#base": [ "-P 50000 --preserve_performance_counters --save_resume" ] }, - [ - { - "+": [ - { - "--learning_rate": [ - null, - 0.1, - 0.01, - 0.001 - ], - "--decay_learning_rate": [ - null, - 1.1, - 1, - 0.9 - ], - "--power_t": [ - null, - 0.5, - 0.6, - 0.4 - ] - } - ] - }, - { - "+": [ - { - "#reg": [ - "--freegrad", - "--conjugate_gradient", - "--bfgs --passes 1 --cache" - ] - } - ] - }, - { - "+": [ - { - "#reg": [ - "--ftrl", - "--coin", - "--pistol" - ], - "--ftrl_alpha": [ - null, - 0.1 - ], - "--ftrl_beta": [ - null, - 0.1 - ] - } - ] - } - ], - [ - { - "*": [ - { - "+": [ - { - "--loss_function": [ - null, - "poisson", - "quantile" - ] - } - ] - }, - { - "+": [ - { - "--loss_function": [ - "expectile" - ], - "--expectile_q": [ - 0.25, - 0.5 - ] - } - ] - } - ] - } - ] - ], + "g1": { + "--learning_rate": [ + null, + 0.1, + 0.01, + 0.001 + ], + "--decay_learning_rate": [ + null, + 1.1, + 1, + 0.9 + ], + "--power_t": [ + null, + 0.5, + 0.6, + 0.4 + ] + }, + "g2": { + "#reg": [ + "--freegrad", + "--conjugate_gradient", + "--bfgs --passes 1 --cache" + ] + }, + "g3": { + "#reg": [ + "--ftrl", + "--coin", + "--pistol" + ], + "--ftrl_alpha": [ + null, + 0.1 + ], + "--ftrl_beta": [ + null, + 0.1 + ] + }, + "g4": { + "--loss_function": [ + null, + "poisson", + "quantile" + ] + }, + "g5": { + "--loss_function": [ + "expectile" + ], + "--expectile_q": [ + 0.25, + 0.5 + ] + } + }, + "grids_expression": "g0 * (g1 + g2 + g3) * (g5 + g4)", "output": [ "--readable_model", "-p" diff --git a/python/tests/test_framework/test_core.py b/python/tests/test_framework/test_core.py index eabaeeddddb..56f58ddfa6b 100644 --- a/python/tests/test_framework/test_core.py +++ b/python/tests/test_framework/test_core.py @@ -1,6 +1,5 @@ from vw_executor.vw import Vw from vw_executor.vw_opts import Grid -from numpy.testing import assert_allclose import pandas as pd import numpy as np import pytest @@ -11,8 +10,10 @@ dynamic_function_call, get_function_object, evaluate_expression, - generate_mathematical_expression_json, + variable_mapping, + copy_file, ) +from conftest import STORE_OUTPUT CURR_DICT = os.path.dirname(os.path.abspath(__file__)) TEST_CONFIG_FILES_NAME = os.listdir(os.path.join(CURR_DICT, "test_configs")) @@ -58,14 +59,28 @@ def core_test(files, grid, outputs, job_assert, job_assert_args): GENERATED_TEST_CASES.append( [lambda: job_assert(j, **job_assert_args), test_name] ) + if STORE_OUTPUT: + if not os.path.exists(CURR_DICT + "/output"): + os.mkdir(CURR_DICT + "/output") + if not os.path.exists(CURR_DICT + "/output/" + test_name): + os.mkdir(CURR_DICT + "/output/" + test_name) + fileName = str(list(j.outputs.values())[0][0]).split("/")[-1] + for key, value in list(j.outputs.items()): + copy_file( + value[0], + CURR_DICT + "/output/" + test_name + "/" + f"{key}_" + fileName, + ) + copy_file( + os.path.join(j.cache.path, "cacheNone/" + fileName), + CURR_DICT + "/output/" + test_name + "/" + fileName, + ) -def get_options(grids): - grid_expression, variables = generate_mathematical_expression_json(grids) +def get_options(grids, expression): final_variables = {} - for key in variables: - final_variables[key] = Grid(variables[key]) - return evaluate_expression(grid_expression, final_variables) + for key in grids: + final_variables[key] = Grid(grids[key]) + return evaluate_expression(expression, final_variables) @pytest.mark.usefixtures("test_descriptions", TEST_CONFIG_FILES) @@ -74,7 +89,9 @@ def init_all(test_descriptions): if type(tests) is not list: tests = [tests] for test_description in tests: - options = get_options(test_description["grids"]) + options = get_options( + test_description["grids"], test_description["grids_expression"] + ) task_folder = TEST_CONFIG_FILES_NAME[tIndex].split(".")[0] package_name = [task_folder + ".", ""] for dir in package_name: diff --git a/python/tests/test_framework/test_helper.py b/python/tests/test_framework/test_helper.py index 7a56ed00560..01092a12adf 100644 --- a/python/tests/test_framework/test_helper.py +++ b/python/tests/test_framework/test_helper.py @@ -3,6 +3,7 @@ import os import itertools import inspect +import shutil # Get the current directory current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -17,82 +18,19 @@ def json_to_dict_list(file): def evaluate_expression(expression, variables): # Create a dictionary to hold the variable values variables_dict = {} - # Populate the variables_dict with the provided variables for variable_name, variable_value in variables.items(): variables_dict[variable_name] = variable_value - # Evaluate the expression using eval() result = eval(expression, variables_dict) return result -def generate_mathematical_expression_json(config): - expression = [] +def variable_mapping(grids): variables_map = {} - - def add_exp(item): - if isinstance(item, dict): - plus = item.get("+", None) - multiple = item.get("*", None) - tmp = item.copy() - if plus: - del tmp["+"] - if multiple: - del tmp["*"] - if tmp: - expression.append("a" + str(len(variables_map))) - variables_map["a" + str(len(variables_map))] = tmp - - if multiple: - if expression and (expression[-1].isalnum() or expression[-1] == ")"): - expression.append("*") - start_exp(multiple) - if plus: - if expression and (expression[-1].isalnum() or expression[-1] == ")"): - expression.append("+") - expression.append("(") - start_exp(plus) - expression.append(")") - elif isinstance(item, list): - expression.append("(") - start_exp(item) - expression.append(")") - - def start_exp(item): - for i in item: - add_exp(i) - - start_exp(config) - res = [] - ptr = 0 - while ptr < len(expression) - 1: - if ( - expression[ptr].isalnum() - and expression[ptr - 1] == "(" - and expression[ptr + 1] == ")" - ): - res.pop(-1) - res.append(expression[ptr]) - ptr += 2 - else: - res.append(expression[ptr]) - ptr += 1 - res.append(expression[-1]) - - ptr = 0 - while ptr < len(res) - 1: - if ( - res[ptr].isalnum() - and (res[ptr + 1].isalnum() or res[ptr + 1] == "(") - or (res[ptr] == ")" and res[ptr + 1].isalnum()) - or (res[ptr] == ")" and res[ptr + 1] == "(") - ): - res.insert(ptr + 1, "*") - ptr += 2 - else: - ptr += 1 - return "".join(res), variables_map + for i in range(len(grids)): + variables_map["g" + str(len(variables_map))] = grids[i] + return variables_map def dynamic_function_call(module_name, function_name, *args, **kwargs): @@ -128,3 +66,15 @@ def generate_string_combinations(*lists): combinations = list(itertools.product(*lists)) combinations = ["".join(combination) for combination in combinations] return combinations + + +def copy_file(source_file, destination_file): + try: + shutil.copy(source_file, destination_file) + print(f"File copied successfully from '{source_file}' to '{destination_file}'.") + except FileNotFoundError: + print(f"Source file '{source_file}' not found.") + except PermissionError: + print( + f"Permission denied. Unable to copy '{source_file}' to '{destination_file}'." + )