diff --git a/tools/README.md b/tools/README.md index 0f5dcd3c187..dcc82a39007 100644 --- a/tools/README.md +++ b/tools/README.md @@ -10,7 +10,7 @@ The primary goal is to reduce the manual effort required in running experiments ### Automated Experiment Execution - Given multiple variables, it automatically generates all combinations and runs the experiments. -- Proper model files are selected automatically when the "otx eval" command is executed, based on the preceding command. +- Proper model files are selected automatically when the "otx eval" or "otx optimize" command is executed, based on the preceding command. ### Fault Tolerance @@ -59,7 +59,7 @@ Sample Experiment Recipe YAML File: Arguments for recipe -- output*path (optional) : Output path where all experiment outputs are saved. Default is "./experiment*{executed_time}" +- output_path (optional) : Output path where all experiment outputs are saved. Default is "./experiment\_{executed_time}" - constant (optional) : It's similar as constant or variable in programming languages. You can use it to replace duplicated string by using ${constant_name} in variables or commands. @@ -80,9 +80,9 @@ If there are failed cases, variables and error logs are both printed and saved a Note that all commands within each case are executed within the same workspace, obviating the need to set a template path from the second command. -When the "otx eval" command is executed, the model file (model weight or exported model, etc.) +When the "otx eval" or "otx optimize" command is executed, the model file (model weight or exported model, etc.) is automatically selected based on the preceding command. -The output file of "otx eval" is then stored at "workspace*path/outputs/XXXX*{train, export, optimize, etc.}/" +The output file of "otx eval" is then stored at "workspace_path/outputs/XXXX\_{train, export, optimize, etc.}/" under the name "performance.json". ### Feature 2 : organize experiment result from single workspace diff --git a/tools/experiment.py b/tools/experiment.py index 4b7f1aa7870..abae60a9f40 100644 --- a/tools/experiment.py +++ b/tools/experiment.py @@ -667,7 +667,10 @@ class OtxCommandRunner: repeat_idx (int): repeat index. """ - OUTPUT_FILE_NAME = {"export": "openvino.bin", "optimize": "weights.pth"} + OUTPUT_FILE_NAME: Dict[str, List[str]] = { + "export": ["openvino.bin"], + "optimize": ["weights.pth", "openvino.bin"] + } def __init__(self, command_ins: Command, repeat_idx: int): self._command_ins = command_ins @@ -676,7 +679,7 @@ def __init__(self, command_ins: Command, repeat_idx: int): self._workspace = Path("_".join(self._command_var.values()).replace("/", "_") + f"_repeat_{repeat_idx}") self._command_var["repeat"] = str(repeat_idx) self._fail_logs: List[CommandFailInfo] = [] - self._previous_cmd_entry: Optional[str] = None + self._previous_cmd_entry: Optional[List[str]] = [] @property def fail_logs(self) -> List[CommandFailInfo]: @@ -696,7 +699,7 @@ def run_command_list(self, dryrun: bool = False): else: print(" ".join(command)) - self._previous_cmd_entry = command[1] + self._previous_cmd_entry.append(command[1]) if not dryrun: organize_exp_result(self._workspace, self._command_var) @@ -704,11 +707,16 @@ def run_command_list(self, dryrun: bool = False): def _prepare_run_command(self, command: List[str]) -> bool: self.set_arguments_to_cmd(command, "--workspace", str(self._workspace)) cmd_entry = command[1] + previous_cmd = None + for previous_cmd in reversed(self._previous_cmd_entry): + if previous_cmd != "eval": + break + if cmd_entry == "train": self.set_arguments_to_cmd(command, "--seed", str(self._repeat_idx)) elif cmd_entry == "eval": - if self._previous_cmd_entry in self.OUTPUT_FILE_NAME: - file_path = self._find_model_path(self._previous_cmd_entry) + if previous_cmd in ["export", "optimize"]: + file_path = self._find_model_path(previous_cmd) if file_path is None: return False self.set_arguments_to_cmd(command, "--load-weights", str(file_path)) @@ -716,6 +724,12 @@ def _prepare_run_command(self, command: List[str]) -> bool: else: output_path = str(self._workspace / "outputs" / "latest_trained_model") self.set_arguments_to_cmd(command, "--output", output_path) + elif cmd_entry == "optimize": + if previous_cmd == "export": # execute PTQ. If not, execute QAT + file_path = self._find_model_path(previous_cmd) + if file_path is None: + return False + self.set_arguments_to_cmd(command, "--load-weights", str(file_path)) return True @@ -731,11 +745,13 @@ def _find_model_path(self, cmd_entry: str): if output_dir is None: print(f"There is no {cmd_entry} output directory.") return None - file_path = list(output_dir.rglob(self.OUTPUT_FILE_NAME[cmd_entry])) - if not file_path: - print(f"{self.OUTPUT_FILE_NAME[cmd_entry]} can't be found.") - return None - return file_path[0] + for file_name in self.OUTPUT_FILE_NAME[cmd_entry]: + file_path = list(output_dir.rglob(file_name)) + if file_path: + return file_path[0] + + print(f"{', '.join(self.OUTPUT_FILE_NAME[cmd_entry])} can't be found.") + return None @staticmethod def set_arguments_to_cmd(command: List[str], key: str, value: Optional[str] = None, before_params: bool = True):