diff --git a/QUICK_START_GUIDE.md b/QUICK_START_GUIDE.md index 5d7387a6a2b..40d04ef845e 100644 --- a/QUICK_START_GUIDE.md +++ b/QUICK_START_GUIDE.md @@ -173,6 +173,9 @@ usage: ote train template params [-h] [--learning_parameters.learning_rate LEARNING_RATE] [--learning_parameters.learning_rate_warmup_iters LEARNING_RATE_WARMUP_ITERS] [--learning_parameters.num_iters NUM_ITERS] + [--learning_parameters.enable_early_stopping ENABLE_EARLY_STOPPING] + [--learning_parameters.early_stop_patience EARLY_STOP_PATIENCE] + [--learning_parameters.early_stop_iteration_patience EARLY_STOP_ITERATION_PATIENCE] [--postprocessing.confidence_threshold CONFIDENCE_THRESHOLD] [--postprocessing.result_based_confidence_threshold RESULT_BASED_CONFIDENCE_THRESHOLD] [--nncf_optimization.enable_quantization ENABLE_QUANTIZATION] @@ -205,6 +208,22 @@ optional arguments: default_value: 300 max_value: 100000 min_value: 1 + --learning_parameters.enable_early_stopping ENABLE_EARLY_STOPPING + header: Enable early stopping of the training + type: BOOLEAN + default_value: True + --learning_parameters.early_stop_patience EARLY_STOP_PATIENCE + header: Patience for early stopping + type: INTEGER + default_value: 10 + max_value: 50 + min_value: 0 + --learning_parameters.early_stop_iteration_patience EARLY_STOP_ITERATION_PATIENCE + header: Iteration patience for early stopping + type: INTEGER + default_value: 0 + max_value: 1000 + min_value: 0 --postprocessing.confidence_threshold CONFIDENCE_THRESHOLD header: Confidence threshold type: FLOAT diff --git a/external/model-preparation-algorithm/configs/classification/configuration.yaml b/external/model-preparation-algorithm/configs/classification/configuration.yaml index db876bbbf5c..f3aecbbe8d3 100644 --- a/external/model-preparation-algorithm/configs/classification/configuration.yaml +++ b/external/model-preparation-algorithm/configs/classification/configuration.yaml @@ -121,17 +121,68 @@ learning_parameters: enable_early_stopping: affects_outcome_of: TRAINING default_value: true - description: Adaptive early exit from training when accuracy isn't changed or decreased for several epochs. + description: Early exit from training when validation accuracy isn't changed or decreased for several epochs. editable: true - header: Enable adaptive early stopping of the training + header: Enable early stopping of the training type: BOOLEAN ui_rules: action: DISABLE_EDITING operator: AND rules: [] type: UI_RULES - visible_in_ui: false + visible_in_ui: true warning: null + early_stop_start: + affects_outcome_of: TRAINING + default_value: 3 + editable: true + header: Start epoch for early stopping + max_value: 1000 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 3 + visible_in_ui: false + early_stop_patience: + affects_outcome_of: TRAINING + default_value: 8 + description: Training will stop if the model does not improve within the number of epochs of patience. + editable: true + header: Patience for early stopping + max_value: 50 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 8 + visible_in_ui: true + warning: This is applied exclusively when early stopping is enabled. + early_stop_iteration_patience: + affects_outcome_of: TRAINING + default_value: 0 + description: + Training will stop if the model does not improve within the number of iterations of patience. + This ensures the model is trained enough with the number of iterations of patience before early stopping. + editable: true + header: Iteration patience for early stopping + max_value: 1000 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 0 + visible_in_ui: true + warning: This is applied exclusively when early stopping is enabled. type: PARAMETER_GROUP visible_in_ui: true pot_parameters: diff --git a/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/hpo_config.yaml b/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/hpo_config.yaml index 5ea8a9b33a0..613f82e7c36 100644 --- a/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/hpo_config.yaml +++ b/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/hpo_config.yaml @@ -4,9 +4,9 @@ hp_space: learning_parameters.learning_rate: param_type: qloguniform range: - - 0.0014 - - 0.035 - - 0.0001 + - 0.00098 + - 0.0245 + - 0.00001 learning_parameters.batch_size: param_type: qloguniform range: diff --git a/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/model.py b/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/model.py index e69d67c39c3..c8ecf77a965 100644 --- a/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/model.py +++ b/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/model.py @@ -2,5 +2,4 @@ '../../../submodule/models/classification/ote_efficientnet_b0.yaml', ] -runner = dict(max_epochs=20) fp16 = dict(loss_scale=512.) diff --git a/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/template.yaml b/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/template.yaml index ba2ddf6f950..f4facd27467 100644 --- a/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/template.yaml +++ b/external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/template.yaml @@ -32,7 +32,7 @@ hyper_parameters: num_workers: default_value: 0 learning_rate: - default_value: 0.007 + default_value: 0.0049 auto_hpo_state: POSSIBLE learning_rate_warmup_iters: default_value: 0 diff --git a/external/model-preparation-algorithm/configs/classification/efficientnet_v2_s_cls_incr/model.py b/external/model-preparation-algorithm/configs/classification/efficientnet_v2_s_cls_incr/model.py index 83e00e0e3a5..5c7357febe9 100644 --- a/external/model-preparation-algorithm/configs/classification/efficientnet_v2_s_cls_incr/model.py +++ b/external/model-preparation-algorithm/configs/classification/efficientnet_v2_s_cls_incr/model.py @@ -2,5 +2,4 @@ '../../../submodule/models/classification/ote_efficientnet_v2_s.yaml', ] -runner = dict(max_epochs=20) fp16 = dict(loss_scale=512.) diff --git a/external/model-preparation-algorithm/configs/classification/efficientnet_v2_s_cls_incr/template.yaml b/external/model-preparation-algorithm/configs/classification/efficientnet_v2_s_cls_incr/template.yaml index c890e33787e..ea8254c8567 100644 --- a/external/model-preparation-algorithm/configs/classification/efficientnet_v2_s_cls_incr/template.yaml +++ b/external/model-preparation-algorithm/configs/classification/efficientnet_v2_s_cls_incr/template.yaml @@ -32,7 +32,7 @@ hyper_parameters: num_workers: default_value: 0 learning_rate: - default_value: 0.007 + default_value: 0.0071 auto_hpo_state: POSSIBLE learning_rate_warmup_iters: default_value: 0 diff --git a/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/hpo_config.yaml b/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/hpo_config.yaml index 35a77712ea5..2202588ff32 100644 --- a/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/hpo_config.yaml +++ b/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/hpo_config.yaml @@ -4,9 +4,9 @@ hp_space: learning_parameters.learning_rate: param_type: qloguniform range: - - 0.0008 + - 0.00029 - 0.1 - - 0.0001 + - 0.00001 learning_parameters.batch_size: param_type: qloguniform range: diff --git a/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/model.py b/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/model.py index ac4e2807d09..3afad41353c 100644 --- a/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/model.py +++ b/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/model.py @@ -2,5 +2,4 @@ '../../../submodule/models/classification/ote_mobilenet_v3_large.yaml', ] -runner = dict(max_epochs=20) fp16 = dict(loss_scale=512.) diff --git a/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/template.yaml b/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/template.yaml index cf5ad5caf39..4e3ef04a706 100644 --- a/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/template.yaml +++ b/external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/template.yaml @@ -32,10 +32,10 @@ hyper_parameters: num_workers: default_value: 0 learning_rate: - default_value: 0.016 + default_value: 0.0058 auto_hpo_state: POSSIBLE learning_rate_warmup_iters: - default_value: 100 + default_value: 10 num_iters: default_value: 90 nncf_optimization: diff --git a/external/model-preparation-algorithm/configs/detection/configuration.yaml b/external/model-preparation-algorithm/configs/detection/configuration.yaml index 870a7dce126..7b03fbd1f97 100644 --- a/external/model-preparation-algorithm/configs/detection/configuration.yaml +++ b/external/model-preparation-algorithm/configs/detection/configuration.yaml @@ -119,6 +119,85 @@ learning_parameters: value: 0 visible_in_ui: true warning: null + enable_early_stopping: + affects_outcome_of: TRAINING + default_value: true + description: Early exit from training when validation accuracy isn't changed or decreased for several epochs. + editable: true + header: Enable early stopping of the training + type: BOOLEAN + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + visible_in_ui: true + warning: null + early_stop_start: + affects_outcome_of: TRAINING + default_value: 3 + editable: true + header: Start epoch for early stopping + max_value: 1000 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 3 + visible_in_ui: false + early_stop_patience: + affects_outcome_of: TRAINING + default_value: 10 + description: Training will stop if the model does not improve within the number of epochs of patience. + editable: true + header: Patience for early stopping + max_value: 50 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 10 + visible_in_ui: true + warning: This is applied exclusively when early stopping is enabled. + early_stop_iteration_patience: + affects_outcome_of: TRAINING + default_value: 0 + description: + Training will stop if the model does not improve within the number of iterations of patience. + This ensures the model is trained enough with the number of iterations of patience before early stopping. + editable: true + header: Iteration patience for early stopping + max_value: 1000 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 0 + visible_in_ui: true + warning: This is applied exclusively when early stopping is enabled. + use_adaptive_interval: + affects_outcome_of: TRAINING + default_value: true + description: Depending on the size of iteration per epoch, adaptively update the validation interval and related values. + editable: true + header: Use adaptive validation interval + type: BOOLEAN + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + visible_in_ui: true + warning: This will automatically control the patience and interval when early stopping is enabled. type: PARAMETER_GROUP visible_in_ui: true postprocessing: diff --git a/external/model-preparation-algorithm/configs/instance-segmentation/configuration.yaml b/external/model-preparation-algorithm/configs/instance-segmentation/configuration.yaml index d16831b9a92..8664d6f9ce5 100644 --- a/external/model-preparation-algorithm/configs/instance-segmentation/configuration.yaml +++ b/external/model-preparation-algorithm/configs/instance-segmentation/configuration.yaml @@ -119,6 +119,85 @@ learning_parameters: value: 0 visible_in_ui: true warning: null + enable_early_stopping: + affects_outcome_of: TRAINING + default_value: true + description: Early exit from training when validation accuracy isn't changed or decreased for several epochs. + editable: true + header: Enable early stopping of the training + type: BOOLEAN + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + visible_in_ui: true + warning: null + early_stop_start: + affects_outcome_of: TRAINING + default_value: 3 + editable: true + header: Start epoch for early stopping + max_value: 1000 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 3 + visible_in_ui: false + early_stop_patience: + affects_outcome_of: TRAINING + default_value: 10 + description: Training will stop if the model does not improve within the number of epochs of patience. + editable: true + header: Patience for early stopping + max_value: 50 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 10 + visible_in_ui: true + warning: This is applied exclusively when early stopping is enabled. + early_stop_iteration_patience: + affects_outcome_of: TRAINING + default_value: 0 + description: + Training will stop if the model does not improve within the number of iterations of patience. + This ensures the model is trained enough with the number of iterations of patience before early stopping. + editable: true + header: Iteration patience for early stopping + max_value: 1000 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 0 + visible_in_ui: true + warning: This is applied exclusively when early stopping is enabled. + use_adaptive_interval: + affects_outcome_of: TRAINING + default_value: true + description: Depending on the size of iteration per epoch, adaptively update the validation interval and related values. + editable: true + header: Use adaptive validation interval + type: BOOLEAN + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + visible_in_ui: true + warning: This will automatically control the patience and interval when early stopping is enabled. type: PARAMETER_GROUP visible_in_ui: true postprocessing: diff --git a/external/model-preparation-algorithm/configs/rotated-detection/configuration.yaml b/external/model-preparation-algorithm/configs/rotated-detection/configuration.yaml index 13fab1a1da9..6ae96ca835d 100644 --- a/external/model-preparation-algorithm/configs/rotated-detection/configuration.yaml +++ b/external/model-preparation-algorithm/configs/rotated-detection/configuration.yaml @@ -119,6 +119,71 @@ learning_parameters: value: 0 visible_in_ui: true warning: null + enable_early_stopping: + affects_outcome_of: TRAINING + default_value: true + description: Early exit from training when validation accuracy isn't changed or decreased for several epochs. + editable: true + header: Enable early stopping of the training + type: BOOLEAN + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + visible_in_ui: true + warning: null + early_stop_start: + affects_outcome_of: TRAINING + default_value: 3 + editable: true + header: Start epoch for early stopping + max_value: 1000 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 3 + visible_in_ui: false + early_stop_patience: + affects_outcome_of: TRAINING + default_value: 10 + description: Training will stop if the model does not improve within the number of epochs of patience. + editable: true + header: Patience for early stopping + max_value: 50 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 10 + visible_in_ui: true + warning: This is applied exclusively when early stopping is enabled. + early_stop_iteration_patience: + affects_outcome_of: TRAINING + default_value: 0 + description: + Training will stop if the model does not improve within the number of iterations of patience. + This ensures the model is trained enough with the number of iterations of patience before early stopping. + editable: true + header: Iteration patience for early stopping + max_value: 1000 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 0 + visible_in_ui: true + warning: This is applied exclusively when early stopping is enabled. type: PARAMETER_GROUP visible_in_ui: true postprocessing: diff --git a/external/model-preparation-algorithm/configs/segmentation/configuration.yaml b/external/model-preparation-algorithm/configs/segmentation/configuration.yaml index 189815d80c9..8d0a6198270 100644 --- a/external/model-preparation-algorithm/configs/segmentation/configuration.yaml +++ b/external/model-preparation-algorithm/configs/segmentation/configuration.yaml @@ -147,6 +147,71 @@ learning_parameters: value: 0 visible_in_ui: true warning: null + enable_early_stopping: + affects_outcome_of: TRAINING + default_value: true + description: Early exit from training when validation accuracy isn't changed or decreased for several epochs. + editable: true + header: Enable early stopping of the training + type: BOOLEAN + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + visible_in_ui: true + warning: null + early_stop_start: + affects_outcome_of: TRAINING + default_value: 3 + editable: true + header: Start epoch for early stopping + max_value: 1000 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 3 + visible_in_ui: false + early_stop_patience: + affects_outcome_of: TRAINING + default_value: 8 + description: Training will stop if the model does not improve within the number of epochs of patience. + editable: true + header: Patience for early stopping + max_value: 50 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 8 + visible_in_ui: true + warning: This is applied exclusively when early stopping is enabled. + early_stop_iteration_patience: + affects_outcome_of: TRAINING + default_value: 0 + description: + Training will stop if the model does not improve within the number of iterations of patience. + This ensures the model is trained enough with the number of iterations of patience before early stopping. + editable: true + header: Iteration patience for early stopping + max_value: 1000 + min_value: 0 + type: INTEGER + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: 0 + visible_in_ui: true + warning: This is applied exclusively when early stopping is enabled. type: PARAMETER_GROUP visible_in_ui: true postprocessing: diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py index 0ee5f73b840..48268bc5152 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py @@ -258,9 +258,18 @@ def _init_recipe_hparam(self) -> dict: warmup_iters = int(self._hyperparams.learning_parameters.learning_rate_warmup_iters) lr_config = ConfigDict(warmup_iters=warmup_iters) if warmup_iters > 0 \ else ConfigDict(warmup_iters=warmup_iters, warmup=None) + + if self._hyperparams.learning_parameters.enable_early_stopping: + early_stop = ConfigDict(start=int(self._hyperparams.learning_parameters.early_stop_start), + patience=int(self._hyperparams.learning_parameters.early_stop_patience), + iteration_patience=int(self._hyperparams.learning_parameters.early_stop_iteration_patience)) + else: + early_stop = False + return ConfigDict( optimizer=ConfigDict(lr=self._hyperparams.learning_parameters.learning_rate), lr_config=lr_config, + early_stop=early_stop, data=ConfigDict( samples_per_gpu=int(self._hyperparams.learning_parameters.batch_size), workers_per_gpu=int(self._hyperparams.learning_parameters.num_workers), @@ -302,14 +311,6 @@ def _init_model_cfg(self): cfg_path = os.path.join(base_dir, 'model.py') cfg = MPAConfig.fromfile(cfg_path) - # To initialize different HP according to task / Support HP change via CLI & UI - if not self._multilabel: - template = MPAConfig.fromfile(self.template_file_path) - template_params = template.hyper_parameters.parameter_overrides.learning_parameters - incoming_params = self._hyperparams.learning_parameters - if cfg.get('runner', False) and (template_params.num_iters.default_value != incoming_params.num_iters): - cfg.runner.max_epochs = incoming_params.num_iters - cfg.model.multilabel = self._multilabel cfg.model.hierarchical = self._hierarchical if self._hierarchical: @@ -381,10 +382,13 @@ def _patch_evaluation(self, config: MPAConfig): cfg = config.evaluation if self._multilabel: cfg.metric = ['accuracy-mlc', 'mAP', 'CP', 'OP', 'CR', 'OR', 'CF1', 'OF1'] + config.early_stop_metric = 'mAP' elif self._hierarchical: cfg.metric = ['MHAcc', 'avgClsAcc', 'mAP'] + config.early_stop_metric = 'MHAcc' else: cfg.metric = ['accuracy', 'class_accuracy'] + config.early_stop_metric = 'accuracy' class ClassificationTrainTask(ClassificationInferenceTask): diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/config.py b/external/model-preparation-algorithm/mpa_tasks/apis/config.py index 042d6e0f9f0..0b7db2f015d 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/config.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/config.py @@ -77,6 +77,52 @@ class BaseLearningParameters(ParameterGroup): affects_outcome_of=ModelLifecycle.TRAINING ) + use_adaptive_interval = configurable_boolean( + default_value=False, + header="Use adaptive validation interval", + description="Depending on the size of iteration per epoch, adaptively update the validation interval and related values.", + warning="This will automatically control the patience and interval when early stopping is enabled.", + affects_outcome_of=ModelLifecycle.TRAINING + ) + + enable_early_stopping = configurable_boolean( + default_value=True, + header="Enable early stopping of the training", + description="Early exit from training when validation accuracy isn't \ + changed or decreased for several epochs.", + affects_outcome_of=ModelLifecycle.TRAINING + ) + + early_stop_start = configurable_integer( + default_value=3, + min_value=0, + max_value=1000, + header="Start epoch for early stopping", + affects_outcome_of=ModelLifecycle.TRAINING + ) + + early_stop_patience = configurable_integer( + default_value=5, + min_value=0, + max_value=50, + header="Patience for early stopping", + description="Training will stop if the model does not improve within the number of epochs of patience.", + warning="This is applied exclusively when early stopping is enabled.", + affects_outcome_of=ModelLifecycle.TRAINING + ) + + early_stop_iteration_patience = configurable_integer( + default_value=0, + min_value=0, + max_value=1000, + header="Iteration patience for early stopping", + description="Training will stop if the model does not improve within the number of iterations of patience. \ + This ensures the model is trained enough with the number of iterations of patience before early stopping.", + warning="This is applied exclusively when early stopping is enabled.", + affects_outcome_of=ModelLifecycle.TRAINING + ) + + num_workers = configurable_integer( default_value=0, min_value=0, diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py index 78265ec08f4..65e990c917d 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py @@ -184,9 +184,19 @@ def _init_recipe_hparam(self) -> dict: warmup_iters = int(self._hyperparams.learning_parameters.learning_rate_warmup_iters) lr_config = ConfigDict(warmup_iters=warmup_iters) if warmup_iters > 0 \ else ConfigDict(warmup_iters=warmup_iters, warmup=None) + + if self._hyperparams.learning_parameters.enable_early_stopping: + early_stop = ConfigDict(start=int(self._hyperparams.learning_parameters.early_stop_start), + patience=int(self._hyperparams.learning_parameters.early_stop_patience), + iteration_patience=int(self._hyperparams.learning_parameters.early_stop_iteration_patience)) + else: + early_stop = False + return ConfigDict( optimizer=ConfigDict(lr=self._hyperparams.learning_parameters.learning_rate), lr_config=lr_config, + early_stop=early_stop, + use_adaptive_interval=self._hyperparams.learning_parameters.use_adaptive_interval, data=ConfigDict( samples_per_gpu=int(self._hyperparams.learning_parameters.batch_size), workers_per_gpu=int(self._hyperparams.learning_parameters.num_workers), @@ -330,9 +340,7 @@ def _patch_evaluation(config: MPAConfig): cfg.metric = 'mAP' cfg.save_best = 'mAP' # EarlyStoppingHook - for cfg in config.get('custom_hooks', []): - if 'EarlyStoppingHook' in cfg.type: - cfg.metric = 'mAP' + config.early_stop_metric = 'mAP' def _det_add_predictions_to_dataset(self, all_results, width, height, confidence_threshold): shapes = [] diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py index 49362a711cb..e9ee76cdd21 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py @@ -149,9 +149,18 @@ def _init_recipe_hparam(self) -> dict: warmup_iters = int(self._hyperparams.learning_parameters.learning_rate_warmup_iters) lr_config = ConfigDict(warmup_iters=warmup_iters) if warmup_iters > 0 \ else ConfigDict(warmup_iters=warmup_iters, warmup=None) + + if self._hyperparams.learning_parameters.enable_early_stopping: + early_stop = ConfigDict(start=int(self._hyperparams.learning_parameters.early_stop_start), + patience=int(self._hyperparams.learning_parameters.early_stop_patience), + iteration_patience=int(self._hyperparams.learning_parameters.early_stop_iteration_patience)) + else: + early_stop = False + return ConfigDict( optimizer=ConfigDict(lr=self._hyperparams.learning_parameters.learning_rate), lr_config=lr_config, + early_stop=early_stop, data=ConfigDict( samples_per_gpu=int(self._hyperparams.learning_parameters.batch_size), workers_per_gpu=int(self._hyperparams.learning_parameters.num_workers), @@ -291,10 +300,7 @@ def _patch_evaluation(config: MPAConfig): cfg.save_best = 'mDice' cfg.rule = 'greater' # EarlyStoppingHook - for cfg in config.get('custom_hooks', []): - if 'EarlyStoppingHook' in cfg.type: - cfg.metric = 'mDice' - + config.early_stop_metric = 'mDice' class SegmentationTrainTask(SegmentationInferenceTask, ITrainingTask): def save_model(self, output_model: ModelEntity): diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/task.py index bc46e2417f7..33b903888f9 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/task.py @@ -3,6 +3,7 @@ # import abc +from collections import defaultdict import io import os import shutil @@ -14,7 +15,7 @@ from mpa.builder import build from mpa.modules.hooks.cancel_interface_hook import CancelInterfaceHook from mpa.stage import Stage -from mpa.utils.config_utils import update_or_add_custom_hook +from mpa.utils.config_utils import update_or_add_custom_hook, remove_custom_hook from mpa.utils.logger import get_logger from ote_sdk.entities.datasets import DatasetEntity from ote_sdk.entities.model import ModelEntity, ModelPrecision @@ -155,10 +156,12 @@ def _initialize(self, dataset=None, output_model=None, export=False): """ logger.info('initializing....') self._init_recipe() + if not export: recipe_hparams = self._init_recipe_hparam() if len(recipe_hparams) > 0: self._recipe_cfg.merge_from_dict(recipe_hparams) + if "custom_hooks" in self.override_configs: override_custom_hooks = self.override_configs.pop("custom_hooks") for override_custom_hook in override_custom_hooks: @@ -181,6 +184,30 @@ def _initialize(self, dataset=None, output_model=None, export=False): del self._model_cfg['fp16'] self._precision = [ModelPrecision.FP32] + # Add/remove adaptive interval hook + if self._recipe_cfg.get('use_adaptive_interval', False): + self._recipe_cfg.adaptive_validation_interval = self._recipe_cfg.get('adaptive_validation_interval', dict(max_interval=5)) + else: + self._recipe_cfg.pop('adaptive_validation_interval', None) + + # Add/remove early stop hook + if 'early_stop' in self._recipe_cfg: + remove_custom_hook(self._recipe_cfg, 'EarlyStoppingHook') + early_stop = self._recipe_cfg.get('early_stop', False) + if early_stop: + early_stop_hook = ConfigDict( + type='LazyEarlyStoppingHook', + start=early_stop.start, + patience=early_stop.patience, + iteration_patience=early_stop.iteration_patience, + interval=1, + metric=self._recipe_cfg.early_stop_metric, + priority=75, + ) + update_or_add_custom_hook(self._recipe_cfg, early_stop_hook) + else: + remove_custom_hook(self._recipe_cfg, 'LazyEarlyStoppingHook') + # add Cancel tranining hook update_or_add_custom_hook(self._recipe_cfg, ConfigDict( type='CancelInterfaceHook', init_callback=self.on_hook_initialized)) diff --git a/external/model-preparation-algorithm/submodule b/external/model-preparation-algorithm/submodule index 484f18f531e..19cf6ea4239 160000 --- a/external/model-preparation-algorithm/submodule +++ b/external/model-preparation-algorithm/submodule @@ -1 +1 @@ -Subproject commit 484f18f531ecf0987d62bc2c6890f4aa8007b206 +Subproject commit 19cf6ea4239505954bfc1809b5cac630db61dc3b