From f8d0f6189c20d9f3f4f292d4347bea4396dba5af Mon Sep 17 00:00:00 2001 From: micky Date: Mon, 31 Jul 2023 17:35:10 +0300 Subject: [PATCH 1/3] test: add test for slate --- .../test_framework/slate/action_space.py | 15 ++ .../test_framework/slate/data_generation.py | 78 +++++++++++ .../test_framework/slate/logging_policies.py | 3 + .../test_framework/slate/reward_functions.py | 11 ++ .../test_framework/test_configs/slate.json | 130 ++++++++++++++++++ python/tests/test_framework/test_core.py | 3 +- 6 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 python/tests/test_framework/slate/action_space.py create mode 100644 python/tests/test_framework/slate/data_generation.py create mode 100644 python/tests/test_framework/slate/logging_policies.py create mode 100644 python/tests/test_framework/slate/reward_functions.py create mode 100644 python/tests/test_framework/test_configs/slate.json diff --git a/python/tests/test_framework/slate/action_space.py b/python/tests/test_framework/slate/action_space.py new file mode 100644 index 00000000000..b44bb5aa7f3 --- /dev/null +++ b/python/tests/test_framework/slate/action_space.py @@ -0,0 +1,15 @@ +def threeSlot_clothes_sunny_raining(**kwargs): + iteration = kwargs.get("iteration", 0) + # before iteration 500, it is sunny and after it is raining + if iteration > 500: + return [ + ["buttonupshirt", "highvis", "rainshirt"], + ["formalpants", "rainpants", "shorts"], + ["rainshoe", "formalshoes", "flipflops"], + ] + + return [ + ["tshirt", "longshirt", "turtleneck"], + ["workpants", "shorts", "formalpants"], + ["formalshoes", "runners", "flipflops"], + ] diff --git a/python/tests/test_framework/slate/data_generation.py b/python/tests/test_framework/slate/data_generation.py new file mode 100644 index 00000000000..fb2d2b55729 --- /dev/null +++ b/python/tests/test_framework/slate/data_generation.py @@ -0,0 +1,78 @@ +import random +import os +from test_helper import get_function_object + +script_directory = os.path.dirname(os.path.realpath(__file__)) +random.seed(10) + + +def random_number_items(items): + num_items_to_select = random.randint(1, len(items)) + return random.sample(items, num_items_to_select) + + +def generate_slate_data( + num_examples, + num_actions, + reward_function, + logging_policy, + action_space, + num_slots=1, + num_context=1, + context_name=None, + slot_name=None, +): + + dataFile = f"slate_test_{num_examples}_{len(num_actions)}_{num_slots}.txt" + + reward_function_obj = get_function_object( + "slate.reward_functions", reward_function["name"] + ) + logging_policy_obj = get_function_object( + "slate.logging_policies", logging_policy["name"] + ) + + action_space_obj = get_function_object("slate.action_space", action_space["name"]) + + def return_cost_probability(chosen_action, chosen_slot, context=1): + cost = reward_function_obj( + chosen_action, context, chosen_slot, **reward_function["params"] + ) + logging_policy["params"]["num_action"] = num_actions[chosen_slot - 1] + logging_policy["params"]["chosen_action"] = chosen_action + probability = logging_policy_obj(**logging_policy["params"]) + return cost, probability + + if not slot_name: + slot_name = [f"slot_{index}" for index in range(1, num_slots + 1)] + with open(os.path.join(script_directory, dataFile), "w") as f: + for i in range(num_examples): + chosen_actions = [] + if num_context > 1: + context = random.randint(1, num_context) + if not context_name: + context_name = [f"{index}" for index in range(1, num_context + 1)] + for s in range(num_slots): + chosen_actions.append(random.randint(1, num_actions[s])) + chosen_actions_cost_prob = [ + return_cost_probability(action, slot + 1, context) + for slot, action in enumerate(chosen_actions) + ] + total_cost = sum([cost for cost, _ in chosen_actions_cost_prob]) + + f.write(f"slates shared {total_cost} |User {context_name[context-1]}\n") + # write actions + action_space["params"]["iteration"] = i + action_spaces = action_space_obj(**action_space["params"]) + for ind, slot in enumerate(action_spaces): + for a in slot: + f.write( + f"slates action {ind} |Action {a}\n", + ) + + for s in range(num_slots): + f.write( + f"slates slot {chosen_actions[s]}:{chosen_actions_cost_prob[s][1]} |Slot {slot_name[s]}\n" + ) + f.write("\n") + return os.path.join(script_directory, dataFile) diff --git a/python/tests/test_framework/slate/logging_policies.py b/python/tests/test_framework/slate/logging_policies.py new file mode 100644 index 00000000000..1b80912ffe7 --- /dev/null +++ b/python/tests/test_framework/slate/logging_policies.py @@ -0,0 +1,3 @@ +def even_probability(chosen_action, **kwargs): + num_actions = kwargs["num_action"] + return round(1 / num_actions, 2) diff --git a/python/tests/test_framework/slate/reward_functions.py b/python/tests/test_framework/slate/reward_functions.py new file mode 100644 index 00000000000..1d3ad1c47dc --- /dev/null +++ b/python/tests/test_framework/slate/reward_functions.py @@ -0,0 +1,11 @@ +def fixed_reward(chosen_action, context, slot, **kwargs): + reward = kwargs["reward"] + return reward[slot - 1][chosen_action - 1] + + +def changing_reward(chosen_action, context, slot, **kwargs): + reward = kwargs["reward"] + iteration = kwargs.get("iteration", 0) + if iteration > 500: + reward = [i[::-1] for i in reward] + return reward[slot - 1][chosen_action - 1] diff --git a/python/tests/test_framework/test_configs/slate.json b/python/tests/test_framework/test_configs/slate.json new file mode 100644 index 00000000000..05861d9f1fe --- /dev/null +++ b/python/tests/test_framework/test_configs/slate.json @@ -0,0 +1,130 @@ +[ + { + "test_name": "slates", + "data_func": { + "name": "generate_slate_data", + "params": { + "num_examples": 1000, + "num_action": [ + 3, + 3, + 3 + ], + "reward_function": { + "name": "changing_reward", + "params": { + "reward": [ + [ + 0, + 1, + 1 + ], + [ + 1, + 0, + 1 + ], + [ + 1, + 1, + 0 + ] + ] + } + }, + "logging_policy": { + "name": "even_probability", + "params": {} + }, + "action_space": { + "name": "threeSlot_clothes_sunny_raining", + "params": {} + }, + "num_slot": 3, + "num_context": 2 + } + }, + "assert_functions": [ + { + "name": "assert_loss", + "params": { + "expected_loss": 3.1, + "decimal": 0.1 + } + }, + { + "name": "assert_prediction", + "params": { + "expected_value": [ + 0.866667, + 0.066667, + 0.066667 + ], + "threshold": 0.8, + "atol": 0.01, + "rtol": 0.01 + } + } + ], + "grids": { + "g0": { + "#base": [ + "--slates" + ] + }, + "g1": { + "--epsilon": [ + 0.1, + 0.2, + 0.3 + ] + }, + "g2": { + "--first": [ + 1, + 2 + ] + }, + "g3": { + "--bag": [ + 5, + 6, + 7 + ] + }, + "g4": { + "--cover": [ + 1, + 2, + 3 + ] + }, + "g5": { + "--squarecb": [ + "--gamma_scale 1000", + "--gamma_scale 10000" + ] + }, + "g6": { + "--synthcover": [ + "" + ] + }, + "g7": { + "--regcb": [ + "" + ] + }, + "g8": { + "--softmax": [ + "" + ] + } + }, + "grids_expression": "g0 * (g1 + g2 + g3 + g4 + g5 +g6 + g7 + g8)", + "output": [ + "--readable_model", + "-p" + ] + } +] \ No newline at end of file diff --git a/python/tests/test_framework/test_core.py b/python/tests/test_framework/test_core.py index 56f58ddfa6b..300f89f8eff 100644 --- a/python/tests/test_framework/test_core.py +++ b/python/tests/test_framework/test_core.py @@ -10,7 +10,6 @@ dynamic_function_call, get_function_object, evaluate_expression, - variable_mapping, copy_file, ) from conftest import STORE_OUTPUT @@ -43,7 +42,7 @@ def cleanup_data_file(): @pytest.fixture def test_descriptions(request): resource = request.param - yield resource # + yield resource cleanup_data_file() From 51c7045ccf187b628ea81788d807a7209bcb55d3 Mon Sep 17 00:00:00 2001 From: micky Date: Tue, 1 Aug 2023 15:53:50 +0300 Subject: [PATCH 2/3] test: test cleanup and slate test update --- .../test_framework/cb/data_generation.py | 3 + .../classification/data_generation.py | 6 +- .../test_framework/slate/action_space.py | 18 ++-- .../tests/test_framework/slate/assert_job.py | 43 ++++++++++ .../test_framework/slate/data_generation.py | 30 +++---- .../test_framework/slate/reward_functions.py | 5 +- .../tests/test_framework/test_configs/cb.json | 53 ++++++------ .../test_configs/classification.json | 22 ++--- .../test_framework/test_configs/slate.json | 82 +++++++++++-------- python/tests/test_framework/test_core.py | 35 ++++---- python/tests/test_framework/test_helper.py | 67 ++++++++++++--- 11 files changed, 225 insertions(+), 139 deletions(-) create mode 100644 python/tests/test_framework/slate/assert_job.py diff --git a/python/tests/test_framework/cb/data_generation.py b/python/tests/test_framework/cb/data_generation.py index 2b218da999a..197c5368369 100644 --- a/python/tests/test_framework/cb/data_generation.py +++ b/python/tests/test_framework/cb/data_generation.py @@ -41,7 +41,10 @@ def return_cost_probability(chosen_action, context=1): cost = reward_function_obj( chosen_action, context, **reward_function["params"] ) + if "params" not in logging_policy: + logging_policy["params"] = {} logging_policy["params"]["chosen_action"] = chosen_action + logging_policy["params"]["num_actions"] = num_actions probability = logging_policy_obj(**logging_policy["params"]) return cost, probability diff --git a/python/tests/test_framework/classification/data_generation.py b/python/tests/test_framework/classification/data_generation.py index fb03ccf5065..48640735117 100644 --- a/python/tests/test_framework/classification/data_generation.py +++ b/python/tests/test_framework/classification/data_generation.py @@ -7,20 +7,20 @@ def generate_classification_data( - num_sample, + num_example, num_classes, num_features, classify_func, bounds=None, ): - dataFile = f"classification_{num_classes}_{num_features}_{num_sample}.txt" + dataFile = f"classification_{num_classes}_{num_features}_{num_example}.txt" classify_func_obj = get_function_object( "classification.classification_functions", classify_func["name"] ) if not bounds: bounds = [[0, 1] for _ in range(num_features)] with open(os.path.join(script_directory, dataFile), "w") as f: - for _ in range(num_sample): + for _ in range(num_example): x = [ random.uniform(bounds[index][0], bounds[index][1]) for index in range(num_features) diff --git a/python/tests/test_framework/slate/action_space.py b/python/tests/test_framework/slate/action_space.py index b44bb5aa7f3..b7015023dc7 100644 --- a/python/tests/test_framework/slate/action_space.py +++ b/python/tests/test_framework/slate/action_space.py @@ -1,15 +1,7 @@ -def threeSlot_clothes_sunny_raining(**kwargs): +def new_action_after_threshold(**kwargs): iteration = kwargs.get("iteration", 0) + threshold = kwargs.get("threshold", 0) # before iteration 500, it is sunny and after it is raining - if iteration > 500: - return [ - ["buttonupshirt", "highvis", "rainshirt"], - ["formalpants", "rainpants", "shorts"], - ["rainshoe", "formalshoes", "flipflops"], - ] - - return [ - ["tshirt", "longshirt", "turtleneck"], - ["workpants", "shorts", "formalpants"], - ["formalshoes", "runners", "flipflops"], - ] + if iteration > threshold: + return kwargs["after"] + return kwargs["before"] diff --git a/python/tests/test_framework/slate/assert_job.py b/python/tests/test_framework/slate/assert_job.py new file mode 100644 index 00000000000..887b05a050d --- /dev/null +++ b/python/tests/test_framework/slate/assert_job.py @@ -0,0 +1,43 @@ +from numpy.testing import assert_allclose, assert_almost_equal +from vw_executor.vw import ExecutionStatus +import numpy as np + + +def majority_close(arr1, arr2, rtol, atol, threshold): + # Check if the majority of elements are close + close_count = np.count_nonzero(np.isclose(arr1, arr2, rtol=rtol, atol=atol)) + return close_count >= len(arr1) * threshold + + +def assert_prediction(job, **kwargs): + assert job.status == ExecutionStatus.Success, "job should be successful" + atol = kwargs.get("atol", 10e-8) + rtol = kwargs.get("rtol", 10e-5) + threshold = kwargs.get("threshold", 0.9) + expected_value = kwargs["expected_value"] + predictions = job.outputs["-p"] + res = [] + with open(predictions[0], "r") as f: + exampleRes = [] + while True: + line = f.readline() + if not line: + break + if line.count(":") == 0: + res.append(exampleRes) + exampleRes = [] + continue + slotRes = [0] * line.count(":") + slot = line.split(",") + for i in range(len(slot)): + actionInd = int(slot[i].split(":")[0]) + slotRes[i] = float(slot[actionInd].split(":")[1]) + exampleRes.append(slotRes) + + assert majority_close( + res, + [expected_value] * len(res), + rtol=rtol, + atol=atol, + threshold=threshold, + ), f"predicted value should be {expected_value}, \n actual values are {res}" diff --git a/python/tests/test_framework/slate/data_generation.py b/python/tests/test_framework/slate/data_generation.py index fb2d2b55729..8404f0d85b2 100644 --- a/python/tests/test_framework/slate/data_generation.py +++ b/python/tests/test_framework/slate/data_generation.py @@ -6,24 +6,16 @@ random.seed(10) -def random_number_items(items): - num_items_to_select = random.randint(1, len(items)) - return random.sample(items, num_items_to_select) - - def generate_slate_data( num_examples, - num_actions, reward_function, logging_policy, action_space, - num_slots=1, num_context=1, context_name=None, - slot_name=None, ): - dataFile = f"slate_test_{num_examples}_{len(num_actions)}_{num_slots}.txt" + action_space_obj = get_function_object("slate.action_space", action_space["name"]) reward_function_obj = get_function_object( "slate.reward_functions", reward_function["name"] @@ -32,9 +24,7 @@ def generate_slate_data( "slate.logging_policies", logging_policy["name"] ) - action_space_obj = get_function_object("slate.action_space", action_space["name"]) - - def return_cost_probability(chosen_action, chosen_slot, context=1): + def return_cost_probability(chosen_action, chosen_slot, context): cost = reward_function_obj( chosen_action, context, chosen_slot, **reward_function["params"] ) @@ -43,15 +33,21 @@ def return_cost_probability(chosen_action, chosen_slot, context=1): probability = logging_policy_obj(**logging_policy["params"]) return cost, probability - if not slot_name: - slot_name = [f"slot_{index}" for index in range(1, num_slots + 1)] + dataFile = f"slate_test_{num_examples}_{generate_slate_data.__name__}.txt" with open(os.path.join(script_directory, dataFile), "w") as f: for i in range(num_examples): + action_space["params"]["iteration"] = i + action_spaces = action_space_obj(**action_space["params"]) + num_slots = len(action_spaces) + num_actions = [len(slot) for slot in action_spaces] + slot_name = [f"slot_{index}" for index in range(1, num_slots + 1)] chosen_actions = [] if num_context > 1: context = random.randint(1, num_context) - if not context_name: - context_name = [f"{index}" for index in range(1, num_context + 1)] + else: + context = 1 + if not context_name: + context_name = [f"{index}" for index in range(1, num_context + 1)] for s in range(num_slots): chosen_actions.append(random.randint(1, num_actions[s])) chosen_actions_cost_prob = [ @@ -62,8 +58,6 @@ def return_cost_probability(chosen_action, chosen_slot, context=1): f.write(f"slates shared {total_cost} |User {context_name[context-1]}\n") # write actions - action_space["params"]["iteration"] = i - action_spaces = action_space_obj(**action_space["params"]) for ind, slot in enumerate(action_spaces): for a in slot: f.write( diff --git a/python/tests/test_framework/slate/reward_functions.py b/python/tests/test_framework/slate/reward_functions.py index 1d3ad1c47dc..8c5b80db755 100644 --- a/python/tests/test_framework/slate/reward_functions.py +++ b/python/tests/test_framework/slate/reward_functions.py @@ -3,9 +3,10 @@ def fixed_reward(chosen_action, context, slot, **kwargs): return reward[slot - 1][chosen_action - 1] -def changing_reward(chosen_action, context, slot, **kwargs): +def reverse_reward_after_iteration(chosen_action, context, slot, **kwargs): reward = kwargs["reward"] iteration = kwargs.get("iteration", 0) - if iteration > 500: + threshold = kwargs.get("threshold", 0) + if iteration > threshold: reward = [i[::-1] for i in reward] return reward[slot - 1][chosen_action - 1] diff --git a/python/tests/test_framework/test_configs/cb.json b/python/tests/test_framework/test_configs/cb.json index 880141d444c..c37bc03bd89 100644 --- a/python/tests/test_framework/test_configs/cb.json +++ b/python/tests/test_framework/test_configs/cb.json @@ -6,7 +6,7 @@ "params": { "num_examples": 100, "num_features": 1, - "num_action": 2, + "num_actions": 2, "reward_function": { "name": "constant_reward", "params": { @@ -19,7 +19,6 @@ "logging_policy": { "name": "even_probability", "params": { - "num_actions": 2 } }, "no_context": 2 @@ -44,61 +43,61 @@ } ], "grids": { - "g0": { + "cb": { "#base": [ "--cb_explore 2" ] }, - "g1": { + "epsilon": { "--epsilon": [ 0.1, 0.2, 0.3 ] }, - "g2": { + "first": { "--first": [ 1, 2 ] }, - "g3": { + "bag": { "--bag": [ 5, 6, 7 ] }, - "g4": { + "cover": { "--cover": [ 1, 2, 3 ] }, - "g5": { + "squarecb": { "--squarecb": [ "--gamma_scale 1000", "--gamma_scale 10000" ] }, - "g6": { + "synthcover": { "--synthcover": [ "" ] }, - "g7": { + "regcb": { "--regcb": [ "" ] }, - "g8": { + "softmax": { "--softmax": [ "" ] } }, - "grids_expression": "g0 * (g1 + g2 + g3 + g4 + g5 +g6 + g7 + g8)", + "grids_expression": "cb * (epsilon + first + bag + cover + squarecb + synthcover + regcb + softmax)", "output": [ "--readable_model", "-p" @@ -111,16 +110,13 @@ "params": { "num_examples": 100, "num_features": 1, - "num_action": 1, + "num_actions": 1, "reward_function": { "name": "fixed_reward", "params": {} }, "logging_policy": { - "name": "even_probability", - "params": { - "num_actions": 1 - } + "name": "even_probability" } } }, @@ -167,7 +163,7 @@ "params": { "num_examples": 100, "num_features": 2, - "num_action": 2, + "num_actions": 2, "reward_function": { "name": "fixed_reward_two_action", "params": {} @@ -175,7 +171,6 @@ "logging_policy": { "name": "even_probability", "params": { - "num_actions": 2 } }, "no_context": 2 @@ -203,61 +198,61 @@ } ], "grids": { - "g0": { + "cb": { "#base": [ "--cb_explore_adf" ] }, - "g1": { + "epsilon": { "--epsilon": [ 0.1, 0.2, 0.3 ] }, - "g2": { + "first": { "--first": [ 1, 2 ] }, - "g3": { + "bag": { "--bag": [ 5, 6, 7 ] }, - "g4": { + "cover": { "--cover": [ 1, 2, 3 ] }, - "g5": { + "squarecb": { "--squarecb": [ "--gamma_scale 1000", "--gamma_scale 10000" ] }, - "g6": { + "synthcover": { "--synthcover": [ "" ] }, - "g7": { + "regcb": { "--regcb": [ "" ] }, - "g8": { + "softmax": { "--softmax": [ "" ] } }, - "grids_expression": "g0 * (g1 + g2 + g3 + g4 + g5 +g6 + g7 + g8)", + "grids_expression": "cb * (epsilon + first + bag + cover + squarecb + synthcover + regcb + softmax)", "output": [ "--readable_model", "-p" diff --git a/python/tests/test_framework/test_configs/classification.json b/python/tests/test_framework/test_configs/classification.json index bcb269f8910..bcd29a04012 100644 --- a/python/tests/test_framework/test_configs/classification.json +++ b/python/tests/test_framework/test_configs/classification.json @@ -4,9 +4,9 @@ "data_func": { "name": "generate_classification_data", "params": { - "no_sample": 2000, - "no_class": 2, - "no_features": 1, + "num_example": 2000, + "num_classes": 2, + "num_features": 1, "classify_func": { "name": "binary_classification_one_feature", "params": {} @@ -26,7 +26,7 @@ "data_func": { "name": "generate_classification_data", "params": { - "no_sample": 100, + "num_example": 100, "no_class": 2, "no_features": 1, "classify_func": { @@ -57,9 +57,9 @@ "data_func": { "name": "generate_classification_data", "params": { - "no_sample": 100000, - "no_class": 25, - "no_features": 2, + "num_example": 100000, + "num_classes": 25, + "num_features": 2, "classify_func": { "name": "multi_classification_two_features", "params": {} @@ -73,16 +73,16 @@ "data_func": { "name": "generate_classification_data", "params": { - "no_sample": 500, - "no_class": 25, - "no_features": 2, + "num_example": 500, + "num_classes": 25, + "num_features": 2, "classify_func": { "name": "multi_classification_two_features", "params": {} } } }, - "accuracy_threshold": 0.5 + "accuracy_threshold": 0.4 } } ], diff --git a/python/tests/test_framework/test_configs/slate.json b/python/tests/test_framework/test_configs/slate.json index 05861d9f1fe..0197fe38780 100644 --- a/python/tests/test_framework/test_configs/slate.json +++ b/python/tests/test_framework/test_configs/slate.json @@ -5,31 +5,20 @@ "name": "generate_slate_data", "params": { "num_examples": 1000, - "num_action": [ - 3, - 3, - 3 - ], "reward_function": { - "name": "changing_reward", + "name": "reverse_reward_after_iteration", "params": { "reward": [ [ - 0, 1, - 1 + 0 ], [ - 1, 0, 1 - ], - [ - 1, - 1, - 0 ] - ] + ], + "threshold": 500 } }, "logging_policy": { @@ -37,18 +26,38 @@ "params": {} }, "action_space": { - "name": "threeSlot_clothes_sunny_raining", - "params": {} - }, - "num_slot": 3, - "num_context": 2 + "name": "new_action_after_threshold", + "params": { + "threshold": 500, + "before": [ + [ + "longshirt", + "tshirt" + ], + [ + "shorts", + "jeans" + ] + ], + "after": [ + [ + "rainshirt", + "buttonupshirt" + ], + [ + "formalpants", + "rainpants" + ] + ] + } + } } }, "assert_functions": [ { "name": "assert_loss", "params": { - "expected_loss": 3.1, + "expected_loss": 0.8, "decimal": 0.1 } }, @@ -56,9 +65,14 @@ "name": "assert_prediction", "params": { "expected_value": [ - 0.866667, - 0.066667, - 0.066667 + [ + 0.1, + 0.9 + ], + [ + 0.9, + 0.1 + ] ], "threshold": 0.8, "atol": 0.01, @@ -67,61 +81,61 @@ } ], "grids": { - "g0": { + "slate": { "#base": [ "--slates" ] }, - "g1": { + "epsilon": { "--epsilon": [ 0.1, 0.2, 0.3 ] }, - "g2": { + "first": { "--first": [ 1, 2 ] }, - "g3": { + "bag": { "--bag": [ 5, 6, 7 ] }, - "g4": { + "cover": { "--cover": [ 1, 2, 3 ] }, - "g5": { + "squarecb": { "--squarecb": [ "--gamma_scale 1000", "--gamma_scale 10000" ] }, - "g6": { + "synthcover": { "--synthcover": [ "" ] }, - "g7": { + "regcb": { "--regcb": [ "" ] }, - "g8": { + "softmax": { "--softmax": [ "" ] } }, - "grids_expression": "g0 * (g1 + g2 + g3 + g4 + g5 +g6 + g7 + g8)", + "grids_expression": "slate * (epsilon + first + bag + cover + squarecb + synthcover + regcb + softmax)", "output": [ "--readable_model", "-p" diff --git a/python/tests/test_framework/test_core.py b/python/tests/test_framework/test_core.py index 300f89f8eff..9e0f6ae056b 100644 --- a/python/tests/test_framework/test_core.py +++ b/python/tests/test_framework/test_core.py @@ -7,10 +7,11 @@ import logging from test_helper import ( json_to_dict_list, - dynamic_function_call, - get_function_object, evaluate_expression, copy_file, + call_function_with_dirs, + custom_sort, + get_function_obj_with_dirs, ) from conftest import STORE_OUTPUT @@ -85,30 +86,28 @@ def get_options(grids, expression): @pytest.mark.usefixtures("test_descriptions", TEST_CONFIG_FILES) def init_all(test_descriptions): for tIndex, tests in enumerate(test_descriptions): + task_folder = TEST_CONFIG_FILES_NAME[tIndex].split(".")[0] + package_name = [task_folder + ".", ""] + package_name = custom_sort(task_folder, package_name) + package_name.append(".") if type(tests) is not list: tests = [tests] for test_description in tests: options = get_options( test_description["grids"], test_description["grids_expression"] ) - task_folder = TEST_CONFIG_FILES_NAME[tIndex].split(".")[0] - package_name = [task_folder + ".", ""] - for dir in package_name: - try: - data = dynamic_function_call( - dir + "data_generation", - test_description["data_func"]["name"], - *test_description["data_func"]["params"].values(), - ) - if data: - break - except: - pass + data = call_function_with_dirs( + package_name, + "data_generation", + test_description["data_func"]["name"], + **test_description["data_func"]["params"], + ) for assert_func in test_description["assert_functions"]: - assert_job = get_function_object("assert_job", assert_func["name"]) - if not assert_job: - continue + + assert_job = get_function_obj_with_dirs( + package_name, "assert_job", assert_func["name"] + ) script_directory = os.path.dirname(os.path.realpath(__file__)) core_test( os.path.join(script_directory, data), diff --git a/python/tests/test_framework/test_helper.py b/python/tests/test_framework/test_helper.py index 01092a12adf..237ff9ac2b1 100644 --- a/python/tests/test_framework/test_helper.py +++ b/python/tests/test_framework/test_helper.py @@ -26,13 +26,6 @@ def evaluate_expression(expression, variables): return result -def variable_mapping(grids): - variables_map = {} - for i in range(len(grids)): - variables_map["g" + str(len(variables_map))] = grids[i] - return variables_map - - def dynamic_function_call(module_name, function_name, *args, **kwargs): try: calling_frame = inspect.stack()[1] @@ -43,12 +36,13 @@ def dynamic_function_call(module_name, function_name, *args, **kwargs): result = function(*args, **kwargs) return result except ImportError: - print(f"Module '{module_name}' not found.") + pass except AttributeError: - print(f"Function '{function_name}' not found in module '{module_name}'.") + pass def get_function_object(module_name, function_name): + function = None try: calling_frame = inspect.stack()[1] calling_module = inspect.getmodule(calling_frame[0]) @@ -57,9 +51,9 @@ def get_function_object(module_name, function_name): function = getattr(module, function_name) return function except ImportError: - print(f"Module '{module_name}' not found.") + pass except AttributeError: - print(f"Function '{function_name}' not found in module '{module_name}'.") + pass def generate_string_combinations(*lists): @@ -78,3 +72,54 @@ def copy_file(source_file, destination_file): print( f"Permission denied. Unable to copy '{source_file}' to '{destination_file}'." ) + + +def call_function_with_dirs(dirs, module_name, function_name, **kargs): + + for dir in dirs: + try: + data = dynamic_function_call( + dir + module_name, + function_name, + **kargs, + ) + if data: + return data + except Exception as error: + if type(error) not in [ModuleNotFoundError]: + raise error + + +def get_function_obj_with_dirs(dirs, module_name, function_name): + obj = None + for dir in dirs: + try: + obj = get_function_object( + dir + module_name, + function_name, + ) + if obj: + return obj + except Exception as error: + if type(error) not in [ModuleNotFoundError]: + raise error + if not obj: + raise ModuleNotFoundError( + f"Module '{module_name}' not found in any of the directories {dirs}." + ) + + +def calculate_similarity(word, string): + # Calculate the similarity score between the string and the word + score = 0 + for char in word: + if char in string: + score += 1 + return score + + +def custom_sort(word, strings): + # Sort the list of strings based on their similarity to the word + return sorted( + strings, key=lambda string: calculate_similarity(word, string), reverse=True + ) From 4e8002e7b5e986a31ba341a8c54b0ad6f55f4341 Mon Sep 17 00:00:00 2001 From: micky Date: Fri, 4 Aug 2023 19:15:27 +0300 Subject: [PATCH 3/3] test: minor cleanup and change assert_loss function to equal instead of lower --- python/tests/test_framework/assert_job.py | 10 ++++++++-- python/tests/test_framework/cb/data_generation.py | 10 +++++----- python/tests/test_framework/slate/data_generation.py | 8 +++----- python/tests/test_framework/slate/reward_functions.py | 2 +- python/tests/test_framework/test_configs/cb.json | 11 ++++++----- python/tests/test_framework/test_configs/slate.json | 4 ++-- 6 files changed, 25 insertions(+), 20 deletions(-) diff --git a/python/tests/test_framework/assert_job.py b/python/tests/test_framework/assert_job.py index b8c51606565..415500c9e81 100644 --- a/python/tests/test_framework/assert_job.py +++ b/python/tests/test_framework/assert_job.py @@ -68,11 +68,17 @@ def assert_loss(job, **kwargs): assert job.status == ExecutionStatus.Success, "job should be successful" assert type(job[0].loss) == float, "loss should be an float" decimal = kwargs.get("decimal", 2) - if job[0].loss < kwargs["expected_loss"]: - return assert_almost_equal(job[0].loss, kwargs["expected_loss"], decimal=decimal) +def assert_loss_below(job, **kwargs): + assert job.status == ExecutionStatus.Success, "job should be successful" + assert type(job[0].loss) == float, "loss should be an float" + assert ( + job[0].loss <= kwargs["expected_loss"] + ), f"loss should be below {kwargs['expected_loss']}" + + def assert_prediction_with_generated_data(job, **kwargs): assert job.status == ExecutionStatus.Success, "job should be successful" expected_class = [] diff --git a/python/tests/test_framework/cb/data_generation.py b/python/tests/test_framework/cb/data_generation.py index 197c5368369..82359233c69 100644 --- a/python/tests/test_framework/cb/data_generation.py +++ b/python/tests/test_framework/cb/data_generation.py @@ -17,8 +17,7 @@ def generate_cb_data( num_actions, reward_function, logging_policy, - no_context=1, - context_name=None, + context_name=["1"], ): dataFile = f"cb_test_{num_examples}_{num_actions}_{num_features}.txt" @@ -32,13 +31,14 @@ def generate_cb_data( features = [f"feature{index}" for index in range(1, num_features + 1)] with open(os.path.join(script_directory, dataFile), "w") as f: for _ in range(num_examples): + no_context = len(context_name) if no_context > 1: context = random.randint(1, no_context) - if not context_name: - context_name = [f"{index}" for index in range(1, no_context + 1)] + else: + context = 1 def return_cost_probability(chosen_action, context=1): - cost = reward_function_obj( + cost = -reward_function_obj( chosen_action, context, **reward_function["params"] ) if "params" not in logging_policy: diff --git a/python/tests/test_framework/slate/data_generation.py b/python/tests/test_framework/slate/data_generation.py index 8404f0d85b2..9f2955f15d0 100644 --- a/python/tests/test_framework/slate/data_generation.py +++ b/python/tests/test_framework/slate/data_generation.py @@ -11,8 +11,7 @@ def generate_slate_data( reward_function, logging_policy, action_space, - num_context=1, - context_name=None, + context_name=["1"], ): action_space_obj = get_function_object("slate.action_space", action_space["name"]) @@ -25,7 +24,7 @@ def generate_slate_data( ) def return_cost_probability(chosen_action, chosen_slot, context): - cost = reward_function_obj( + cost = -reward_function_obj( chosen_action, context, chosen_slot, **reward_function["params"] ) logging_policy["params"]["num_action"] = num_actions[chosen_slot - 1] @@ -42,12 +41,11 @@ def return_cost_probability(chosen_action, chosen_slot, context): num_actions = [len(slot) for slot in action_spaces] slot_name = [f"slot_{index}" for index in range(1, num_slots + 1)] chosen_actions = [] + num_context = len(context_name) if num_context > 1: context = random.randint(1, num_context) else: context = 1 - if not context_name: - context_name = [f"{index}" for index in range(1, num_context + 1)] for s in range(num_slots): chosen_actions.append(random.randint(1, num_actions[s])) chosen_actions_cost_prob = [ diff --git a/python/tests/test_framework/slate/reward_functions.py b/python/tests/test_framework/slate/reward_functions.py index 8c5b80db755..2a95169a508 100644 --- a/python/tests/test_framework/slate/reward_functions.py +++ b/python/tests/test_framework/slate/reward_functions.py @@ -3,7 +3,7 @@ def fixed_reward(chosen_action, context, slot, **kwargs): return reward[slot - 1][chosen_action - 1] -def reverse_reward_after_iteration(chosen_action, context, slot, **kwargs): +def reverse_reward_after_threshold(chosen_action, context, slot, **kwargs): reward = kwargs["reward"] iteration = kwargs.get("iteration", 0) threshold = kwargs.get("threshold", 0) diff --git a/python/tests/test_framework/test_configs/cb.json b/python/tests/test_framework/test_configs/cb.json index c37bc03bd89..26d8ae09119 100644 --- a/python/tests/test_framework/test_configs/cb.json +++ b/python/tests/test_framework/test_configs/cb.json @@ -21,14 +21,15 @@ "params": { } }, - "no_context": 2 + "context_name": ["1", "2"] } }, "assert_functions": [ { "name": "assert_loss", "params": { - "expected_loss": 0.1 + "expected_loss": -1, + "decimal": 1 } }, { @@ -124,7 +125,7 @@ { "name": "assert_loss", "params": { - "expected_loss": 1 + "expected_loss": -1 } }, { @@ -173,14 +174,14 @@ "params": { } }, - "no_context": 2 + "context_name": ["1", "2"] } }, "assert_functions": [ { "name": "assert_loss", "params": { - "expected_loss": 0.6, + "expected_loss": -0.4, "decimal": 1 } }, diff --git a/python/tests/test_framework/test_configs/slate.json b/python/tests/test_framework/test_configs/slate.json index 0197fe38780..bc7f59b49cf 100644 --- a/python/tests/test_framework/test_configs/slate.json +++ b/python/tests/test_framework/test_configs/slate.json @@ -6,7 +6,7 @@ "params": { "num_examples": 1000, "reward_function": { - "name": "reverse_reward_after_iteration", + "name": "reverse_reward_after_threshold", "params": { "reward": [ [ @@ -57,7 +57,7 @@ { "name": "assert_loss", "params": { - "expected_loss": 0.8, + "expected_loss": -1.9, "decimal": 0.1 } },