From 969297177ce22a3e2345ba0ce3b67c66ef182beb Mon Sep 17 00:00:00 2001 From: DreamGenX Date: Sat, 27 Jan 2024 16:17:41 +0100 Subject: [PATCH 1/3] Make sure test_dataset are used and treat val_set_size. --- src/axolotl/core/trainer_builder.py | 3 ++- src/axolotl/utils/data.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/axolotl/core/trainer_builder.py b/src/axolotl/core/trainer_builder.py index 63f9272acc..ac8fdf9ed5 100644 --- a/src/axolotl/core/trainer_builder.py +++ b/src/axolotl/core/trainer_builder.py @@ -735,7 +735,7 @@ def build(self, total_num_steps): elif self.cfg.sample_packing and self.cfg.eval_sample_packing is False: training_arguments_kwargs["dataloader_drop_last"] = True - if self.cfg.val_set_size == 0: + if not hasattr(self.cfg, "test_datasets") and self.cfg.val_set_size == 0: # no eval set, so don't eval training_arguments_kwargs["evaluation_strategy"] = "no" elif self.cfg.eval_steps: @@ -822,6 +822,7 @@ def build(self, total_num_steps): self.cfg.load_best_model_at_end is not False or self.cfg.early_stopping_patience ) + and not hasattr(self.cfg, "test_datasets") and self.cfg.val_set_size > 0 and self.cfg.save_steps and self.cfg.eval_steps diff --git a/src/axolotl/utils/data.py b/src/axolotl/utils/data.py index d9a590bc3c..39766868ea 100644 --- a/src/axolotl/utils/data.py +++ b/src/axolotl/utils/data.py @@ -440,7 +440,7 @@ def load_prepare_datasets( split="train", ) -> Tuple[Dataset, Dataset, List[Prompter]]: dataset, prompters = load_tokenized_prepared_datasets( - tokenizer, cfg, default_dataset_prepared_path + tokenizer, cfg, default_dataset_prepared_path, split=split ) if cfg.dataset_shard_num and cfg.dataset_shard_idx is not None: From 1c2a73df3143b1f8f3940a540de759b18fa4092d Mon Sep 17 00:00:00 2001 From: DreamGenX Date: Mon, 29 Jan 2024 06:43:00 +0100 Subject: [PATCH 2/3] Add test_datasets docs. --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 95e5f530cd..7d95d3451f 100644 --- a/README.md +++ b/README.md @@ -607,6 +607,17 @@ datasets: # For `completion` datsets only, uses the provided field instead of `text` column field: +# A list of one or more datasets to eval the model with. +# You can use either test_datasets, or val_set_size, but not both. +test_datasets: + - path: /workspace/data/eval.jsonl + ds_type: json + # You need to specify a split. For "json" datasets the default split is called "train". + split: train + type: completion + data_files: + - /workspace/data/eval.jsonl + # use RL training: dpo, ipo, kto_pair rl: From 059bf7d12ec6848cf896621c380e9e60b92e2cc6 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Tue, 30 Jan 2024 20:04:36 -0500 Subject: [PATCH 3/3] Apply suggestions from code review --- src/axolotl/core/trainer_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/axolotl/core/trainer_builder.py b/src/axolotl/core/trainer_builder.py index ac8fdf9ed5..96054dc50f 100644 --- a/src/axolotl/core/trainer_builder.py +++ b/src/axolotl/core/trainer_builder.py @@ -735,7 +735,7 @@ def build(self, total_num_steps): elif self.cfg.sample_packing and self.cfg.eval_sample_packing is False: training_arguments_kwargs["dataloader_drop_last"] = True - if not hasattr(self.cfg, "test_datasets") and self.cfg.val_set_size == 0: + if not self.cfg.test_datasets and self.cfg.val_set_size == 0: # no eval set, so don't eval training_arguments_kwargs["evaluation_strategy"] = "no" elif self.cfg.eval_steps: @@ -822,7 +822,7 @@ def build(self, total_num_steps): self.cfg.load_best_model_at_end is not False or self.cfg.early_stopping_patience ) - and not hasattr(self.cfg, "test_datasets") + and not self.cfg.test_datasets and self.cfg.val_set_size > 0 and self.cfg.save_steps and self.cfg.eval_steps