From 31a8d1edbec017da3d198fc70c22f41bac5f41fb Mon Sep 17 00:00:00 2001 From: fpaissan Date: Wed, 22 Nov 2023 11:07:08 +0100 Subject: [PATCH 1/3] fix metrics and some polishing --- micromind/core.py | 47 ++++++++++++------- micromind/utils/yolo_helpers.py | 2 +- .../extra_requirements.txt | 2 + recipes/objection_detection/train.py | 6 +-- 4 files changed, 37 insertions(+), 20 deletions(-) create mode 100644 recipes/objection_detection/extra_requirements.txt diff --git a/micromind/core.py b/micromind/core.py index 047acbe..cb8981b 100644 --- a/micromind/core.py +++ b/micromind/core.py @@ -5,7 +5,7 @@ Authors: - Francesco Paissan, 2023 """ -from typing import Dict, Union, Tuple, Callable, List +from typing import Dict, Union, Tuple, Callable, List, Optional from abc import ABC, abstractmethod from dataclasses import dataclass from argparse import Namespace @@ -82,15 +82,23 @@ class Metric: 0.5 """ - def __init__(self, name: str, fn: Callable, reduction="mean"): + def __init__( + self, + name: str, + fn: Callable, + reduction: Optional[str] = "mean", + eval_only: Optional[bool] = False, + eval_period: Optional[int] = 1, + ): self.name = name self.fn = fn self.reduction = reduction + self.eval_only = eval_only + self.eval_period = eval_period + self.history = {s: [] for s in [Stage.train, Stage.val, Stage.test]} def __call__(self, pred, batch, stage, device="cpu"): - # if pred.device != device: - # pred = pred.to(device) dat = self.fn(pred, batch) if dat.ndim == 0: dat = dat.unsqueeze(0) @@ -385,6 +393,7 @@ def train( ) with self.accelerator.autocast(): for e in range(self.start_epoch, epochs): + self.current_epoch = e pbar = tqdm( self.datasets["train"], unit="batches", @@ -409,10 +418,13 @@ def train( self.opt.step() for m in self.metrics: - m(model_out, batch, Stage.train, self.device) + if not m.eval_only and (e + 1) % m.eval_period == 0: + m(model_out, batch, Stage.train, self.device) running_train = { - "train_" + m.name: m.reduce(Stage.train) for m in self.metrics + "train_" + m.name: m.reduce(Stage.train) + for m in self.metrics + if not m.eval_only } running_train.update({"train_loss": loss_epoch / (idx + 1)}) @@ -425,7 +437,9 @@ def train( pbar.close() train_metrics = { - "train_" + m.name: m.reduce(Stage.train, True) for m in self.metrics + "train_" + m.name: m.reduce(Stage.train, True) + for m in self.metrics + if not m.eval_only } train_metrics.update({"train_loss": loss_epoch / (idx + 1)}) @@ -477,7 +491,8 @@ def validate(self) -> Dict: model_out = self(batch) loss = self.compute_loss(model_out, batch) for m in self.metrics: - m(model_out, batch, Stage.val, self.device) + if (self.current_epoch + 1) % m.eval_period == 0: + m(model_out, batch, Stage.val, self.device) loss_epoch += loss.item() pbar.set_postfix(loss=loss_epoch / (idx + 1)) @@ -485,7 +500,12 @@ def validate(self) -> Dict: if self.debug and idx > 10: break - val_metrics = {"val_" + m.name: m.reduce(Stage.val, True) for m in self.metrics} + if (self.current_epoch + 1) % m.eval_period == 0: + val_metrics = { + "val_" + m.name: m.reduce(Stage.val, True) for m in self.metrics + } + else: + val_metrics = {} val_metrics.update({"val_loss": loss_epoch / (idx + 1)}) pbar.close() @@ -493,10 +513,7 @@ def validate(self) -> Dict: return val_metrics @torch.no_grad() - def test( - self, - datasets: Dict = {}, - metrics: List[Metric] = []) -> None: + def test(self, datasets: Dict = {}, metrics: List[Metric] = []) -> None: """Runs the test steps.""" assert "test" in datasets, "Test dataloader was not specified." self.modules.eval() @@ -525,9 +542,7 @@ def test( pbar.close() - test_metrics = { - "test_" + m.name: m.reduce(Stage.test, True) for m in metrics - } + test_metrics = {"test_" + m.name: m.reduce(Stage.test, True) for m in metrics} test_metrics.update({"test_loss": loss_epoch / (idx + 1)}) s_out = ( "Testing " diff --git a/micromind/utils/yolo_helpers.py b/micromind/utils/yolo_helpers.py index b58dfc3..2258aee 100644 --- a/micromind/utils/yolo_helpers.py +++ b/micromind/utils/yolo_helpers.py @@ -57,7 +57,7 @@ def load_config(file_path): "path": path, "train": train.as_posix(), "val": val.as_posix(), - "test": test.as_posix(), + "test": test, "names": config["names"], "download": config.get("download"), "yaml_file": file_path, diff --git a/recipes/objection_detection/extra_requirements.txt b/recipes/objection_detection/extra_requirements.txt new file mode 100644 index 0000000..8a371b3 --- /dev/null +++ b/recipes/objection_detection/extra_requirements.txt @@ -0,0 +1,2 @@ +opencv-python +ultralytics==8.0.215 diff --git a/recipes/objection_detection/train.py b/recipes/objection_detection/train.py index be1dc22..9ffa0d1 100644 --- a/recipes/objection_detection/train.py +++ b/recipes/objection_detection/train.py @@ -255,12 +255,12 @@ def mAP(self, pred, batch): if __name__ == "__main__": batch_size = 8 - m_cfg, data_cfg = load_config("cfg/coco.yaml") + m_cfg, data_cfg = load_config("cfg/coco8.yaml") mode = "train" coco8_dataset = build_yolo_dataset( m_cfg, - "datasets/coco/images/train2017", + "datasets/coco8/images/train", batch_size, data_cfg, mode=mode, @@ -278,7 +278,7 @@ def mAP(self, pred, batch): mode = "val" coco8_dataset = build_yolo_dataset( m_cfg, - "datasets/coco/images/val2017", + "datasets/coco8/images/val", batch_size, data_cfg, mode=mode, From 8f441ffbb6e940548a32f73fabd167f304b80277 Mon Sep 17 00:00:00 2001 From: fpaissan Date: Wed, 22 Nov 2023 11:26:47 +0100 Subject: [PATCH 2/3] minor bug fixes --- micromind/core.py | 38 +++++++++++++++------------- recipes/objection_detection/train.py | 4 +-- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/micromind/core.py b/micromind/core.py index cb8981b..16ff3a3 100644 --- a/micromind/core.py +++ b/micromind/core.py @@ -349,6 +349,10 @@ def on_train_end(self): logger.info(f"Removed temporary folder {self.experiment_folder}.") shutil.rmtree(self.experiment_folder) + def eval(self): + for m self.modules: + self.modules[m].eval() + def train( self, epochs: int = 1, @@ -418,14 +422,13 @@ def train( self.opt.step() for m in self.metrics: - if not m.eval_only and (e + 1) % m.eval_period == 0: + if (self.current_epoch + 1) % m.eval_period == 0 and not m.eval_only: m(model_out, batch, Stage.train, self.device) - running_train = { - "train_" + m.name: m.reduce(Stage.train) - for m in self.metrics - if not m.eval_only - } + running_train = {} + for m in self.metrics: + if (self.current_epoch + 1) % m.eval_period == 0 and not m.eval_only: + running_train["train_" + m.name] = m.reduce(Stage.train) running_train.update({"train_loss": loss_epoch / (idx + 1)}) @@ -436,11 +439,11 @@ def train( pbar.close() - train_metrics = { - "train_" + m.name: m.reduce(Stage.train, True) - for m in self.metrics - if not m.eval_only - } + train_metrics = {} + for m in self.metrics: + if (self.current_epoch + 1) % m.eval_period == 0 and not m.eval_only: + train_metrics["train_" + m.name] = m.reduce(Stage.train, True) + train_metrics.update({"train_loss": loss_epoch / (idx + 1)}) if "val" in datasets: @@ -500,12 +503,13 @@ def validate(self) -> Dict: if self.debug and idx > 10: break - if (self.current_epoch + 1) % m.eval_period == 0: - val_metrics = { - "val_" + m.name: m.reduce(Stage.val, True) for m in self.metrics - } - else: - val_metrics = {} + val_metrics = {} + for m in self.metrics: + if (self.current_epoch + 1) % m.eval_period == 0: + val_metrics = { + "val_" + m.name: m.reduce(Stage.val, True) + } + val_metrics.update({"val_loss": loss_epoch / (idx + 1)}) pbar.close() diff --git a/recipes/objection_detection/train.py b/recipes/objection_detection/train.py index 9ffa0d1..33f7523 100644 --- a/recipes/objection_detection/train.py +++ b/recipes/objection_detection/train.py @@ -295,10 +295,10 @@ def mAP(self, pred, batch): hparams = parse_arguments() m = YOLO(m_cfg, hparams=hparams) - mAP = Metric("mAP", m.mAP) + mAP = Metric("mAP", m.mAP, eval_only=False, eval_period=2) m.train( - epochs=50, + epochs=2, datasets={"train": train_loader, "val": val_loader}, metrics=[mAP], debug=hparams.debug, From 083d164b082a5c1264a065cc9b98f48235948ccf Mon Sep 17 00:00:00 2001 From: fpaissan Date: Wed, 22 Nov 2023 11:36:29 +0100 Subject: [PATCH 3/3] cosmetic --- micromind/core.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/micromind/core.py b/micromind/core.py index 16ff3a3..4d149d0 100644 --- a/micromind/core.py +++ b/micromind/core.py @@ -350,8 +350,7 @@ def on_train_end(self): shutil.rmtree(self.experiment_folder) def eval(self): - for m self.modules: - self.modules[m].eval() + self.modules.eval() def train( self, @@ -422,12 +421,16 @@ def train( self.opt.step() for m in self.metrics: - if (self.current_epoch + 1) % m.eval_period == 0 and not m.eval_only: + if ( + self.current_epoch + 1 + ) % m.eval_period == 0 and not m.eval_only: m(model_out, batch, Stage.train, self.device) running_train = {} for m in self.metrics: - if (self.current_epoch + 1) % m.eval_period == 0 and not m.eval_only: + if ( + self.current_epoch + 1 + ) % m.eval_period == 0 and not m.eval_only: running_train["train_" + m.name] = m.reduce(Stage.train) running_train.update({"train_loss": loss_epoch / (idx + 1)}) @@ -441,7 +444,9 @@ def train( train_metrics = {} for m in self.metrics: - if (self.current_epoch + 1) % m.eval_period == 0 and not m.eval_only: + if ( + self.current_epoch + 1 + ) % m.eval_period == 0 and not m.eval_only: train_metrics["train_" + m.name] = m.reduce(Stage.train, True) train_metrics.update({"train_loss": loss_epoch / (idx + 1)}) @@ -506,9 +511,7 @@ def validate(self) -> Dict: val_metrics = {} for m in self.metrics: if (self.current_epoch + 1) % m.eval_period == 0: - val_metrics = { - "val_" + m.name: m.reduce(Stage.val, True) - } + val_metrics = {"val_" + m.name: m.reduce(Stage.val, True)} val_metrics.update({"val_loss": loss_epoch / (idx + 1)})