Expose early stopping hyper-parameters for all tasks (#1241)

QUICK_START_GUIDE.md

@@ -173,6 +173,9 @@ usage: ote train template params [-h]
                                  [--learning_parameters.learning_rate LEARNING_RATE]
                                  [--learning_parameters.learning_rate_warmup_iters LEARNING_RATE_WARMUP_ITERS]
                                  [--learning_parameters.num_iters NUM_ITERS]
+                                 [--learning_parameters.enable_early_stopping ENABLE_EARLY_STOPPING]
+                                 [--learning_parameters.early_stop_patience EARLY_STOP_PATIENCE]
+                                 [--learning_parameters.early_stop_iteration_patience EARLY_STOP_ITERATION_PATIENCE]
                                  [--postprocessing.confidence_threshold CONFIDENCE_THRESHOLD]
                                  [--postprocessing.result_based_confidence_threshold RESULT_BASED_CONFIDENCE_THRESHOLD]
                                  [--nncf_optimization.enable_quantization ENABLE_QUANTIZATION]
@@ -205,6 +208,22 @@ optional arguments:
                         default_value: 300
                         max_value: 100000
                         min_value: 1
+  --learning_parameters.enable_early_stopping ENABLE_EARLY_STOPPING
+                        header: Enable early stopping of the training
+                        type: BOOLEAN
+                        default_value: True
+  --learning_parameters.early_stop_patience EARLY_STOP_PATIENCE
+                        header: Patience for early stopping
+                        type: INTEGER
+                        default_value: 10
+                        max_value: 50
+                        min_value: 0
+  --learning_parameters.early_stop_iteration_patience EARLY_STOP_ITERATION_PATIENCE
+                        header: Iteration patience for early stopping
+                        type: INTEGER
+                        default_value: 0
+                        max_value: 1000
+                        min_value: 0
   --postprocessing.confidence_threshold CONFIDENCE_THRESHOLD
                         header: Confidence threshold
                         type: FLOAT

external/model-preparation-algorithm/configs/classification/configuration.yaml

@@ -121,17 +121,68 @@ learning_parameters:
   enable_early_stopping:
     affects_outcome_of: TRAINING
     default_value: true
-    description: Adaptive early exit from training when accuracy isn't changed or decreased for several epochs.
+    description: Early exit from training when validation accuracy isn't changed or decreased for several epochs.
     editable: true
-    header: Enable adaptive early stopping of the training
+    header: Enable early stopping of the training
     type: BOOLEAN
     ui_rules:
       action: DISABLE_EDITING
       operator: AND
       rules: []
       type: UI_RULES
-    visible_in_ui: false
+    visible_in_ui: true
     warning: null
+  early_stop_start:
+    affects_outcome_of: TRAINING
+    default_value: 3
+    editable: true
+    header: Start epoch for early stopping
+    max_value: 1000
+    min_value: 0
+    type: INTEGER
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    value: 3
+    visible_in_ui: false
+  early_stop_patience:
+    affects_outcome_of: TRAINING
+    default_value: 8
+    description: Training will stop if the model does not improve within the number of epochs of patience.
+    editable: true
+    header: Patience for early stopping
+    max_value: 50
+    min_value: 0
+    type: INTEGER
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    value: 8
+    visible_in_ui: true
+    warning: This is applied exclusively when early stopping is enabled.
+  early_stop_iteration_patience:
+    affects_outcome_of: TRAINING
+    default_value: 0
+    description:
+      Training will stop if the model does not improve within the number of iterations of patience. 
+      This ensures the model is trained enough with the number of iterations of patience before early stopping.
+    editable: true
+    header: Iteration patience for early stopping
+    max_value: 1000
+    min_value: 0
+    type: INTEGER
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    value: 0
+    visible_in_ui: true
+    warning: This is applied exclusively when early stopping is enabled.
   type: PARAMETER_GROUP
   visible_in_ui: true
 pot_parameters:

external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/hpo_config.yaml

@@ -4,9 +4,9 @@ hp_space:
   learning_parameters.learning_rate:
     param_type: qloguniform
     range:
-      - 0.0014
-      - 0.035
-      - 0.0001
+      - 0.00098
+      - 0.0245
+      - 0.00001
   learning_parameters.batch_size:
     param_type: qloguniform
     range:

external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/model.py

@@ -2,5 +2,4 @@
   '../../../submodule/models/classification/ote_efficientnet_b0.yaml',
 ]
 
-runner = dict(max_epochs=20)
 fp16 = dict(loss_scale=512.)

external/model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/template.yaml

@@ -32,7 +32,7 @@ hyper_parameters:
       num_workers:
         default_value: 0
       learning_rate:
-        default_value: 0.007
+        default_value: 0.0049
         auto_hpo_state: POSSIBLE
       learning_rate_warmup_iters:
         default_value: 0

external/model-preparation-algorithm/configs/classification/efficientnet_v2_s_cls_incr/model.py

@@ -2,5 +2,4 @@
   '../../../submodule/models/classification/ote_efficientnet_v2_s.yaml',
 ]
 
-runner = dict(max_epochs=20)
 fp16 = dict(loss_scale=512.)

external/model-preparation-algorithm/configs/classification/efficientnet_v2_s_cls_incr/template.yaml

@@ -32,7 +32,7 @@ hyper_parameters:
       num_workers:
         default_value: 0
       learning_rate:
-        default_value: 0.007
+        default_value: 0.0071
         auto_hpo_state: POSSIBLE
       learning_rate_warmup_iters:
         default_value: 0

external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/hpo_config.yaml

@@ -4,9 +4,9 @@ hp_space:
   learning_parameters.learning_rate:
     param_type: qloguniform
     range:
-      - 0.0008
+      - 0.00029
       - 0.1
-      - 0.0001
+      - 0.00001
   learning_parameters.batch_size:
     param_type: qloguniform
     range:

external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/model.py

@@ -2,5 +2,4 @@
   '../../../submodule/models/classification/ote_mobilenet_v3_large.yaml',
 ]
 
-runner = dict(max_epochs=20)
 fp16 = dict(loss_scale=512.)

external/model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/template.yaml

@@ -32,10 +32,10 @@ hyper_parameters:
       num_workers:
         default_value: 0
       learning_rate:
-        default_value: 0.016
+        default_value: 0.0058
         auto_hpo_state: POSSIBLE
       learning_rate_warmup_iters:
-        default_value: 100
+        default_value: 10
       num_iters:
         default_value: 90
     nncf_optimization:

external/model-preparation-algorithm/configs/detection/configuration.yaml

@@ -119,6 +119,85 @@ learning_parameters:
     value: 0
     visible_in_ui: true
     warning: null
+  enable_early_stopping:
+    affects_outcome_of: TRAINING
+    default_value: true
+    description: Early exit from training when validation accuracy isn't changed or decreased for several epochs.
+    editable: true
+    header: Enable early stopping of the training
+    type: BOOLEAN
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    visible_in_ui: true
+    warning: null
+  early_stop_start:
+    affects_outcome_of: TRAINING
+    default_value: 3
+    editable: true
+    header: Start epoch for early stopping
+    max_value: 1000
+    min_value: 0
+    type: INTEGER
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    value: 3
+    visible_in_ui: false
+  early_stop_patience:
+    affects_outcome_of: TRAINING
+    default_value: 10
+    description: Training will stop if the model does not improve within the number of epochs of patience.
+    editable: true
+    header: Patience for early stopping
+    max_value: 50
+    min_value: 0
+    type: INTEGER
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    value: 10
+    visible_in_ui: true
+    warning: This is applied exclusively when early stopping is enabled.
+  early_stop_iteration_patience:
+    affects_outcome_of: TRAINING
+    default_value: 0
+    description:
+      Training will stop if the model does not improve within the number of iterations of patience. 
+      This ensures the model is trained enough with the number of iterations of patience before early stopping.
+    editable: true
+    header: Iteration patience for early stopping
+    max_value: 1000
+    min_value: 0
+    type: INTEGER
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    value: 0
+    visible_in_ui: true
+    warning: This is applied exclusively when early stopping is enabled.
+  use_adaptive_interval:
+    affects_outcome_of: TRAINING
+    default_value: true
+    description: Depending on the size of iteration per epoch, adaptively update the validation interval and related values.
+    editable: true
+    header: Use adaptive validation interval
+    type: BOOLEAN
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    visible_in_ui: true
+    warning: This will automatically control the patience and interval when early stopping is enabled.
   type: PARAMETER_GROUP
   visible_in_ui: true
 postprocessing:

external/model-preparation-algorithm/configs/instance-segmentation/configuration.yaml

@@ -119,6 +119,85 @@ learning_parameters:
     value: 0
     visible_in_ui: true
     warning: null
+  enable_early_stopping:
+    affects_outcome_of: TRAINING
+    default_value: true
+    description: Early exit from training when validation accuracy isn't changed or decreased for several epochs.
+    editable: true
+    header: Enable early stopping of the training
+    type: BOOLEAN
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    visible_in_ui: true
+    warning: null
+  early_stop_start:
+    affects_outcome_of: TRAINING
+    default_value: 3
+    editable: true
+    header: Start epoch for early stopping
+    max_value: 1000
+    min_value: 0
+    type: INTEGER
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    value: 3
+    visible_in_ui: false
+  early_stop_patience:
+    affects_outcome_of: TRAINING
+    default_value: 10
+    description: Training will stop if the model does not improve within the number of epochs of patience.
+    editable: true
+    header: Patience for early stopping
+    max_value: 50
+    min_value: 0
+    type: INTEGER
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    value: 10
+    visible_in_ui: true
+    warning: This is applied exclusively when early stopping is enabled.
+  early_stop_iteration_patience:
+    affects_outcome_of: TRAINING
+    default_value: 0
+    description:
+      Training will stop if the model does not improve within the number of iterations of patience. 
+      This ensures the model is trained enough with the number of iterations of patience before early stopping.
+    editable: true
+    header: Iteration patience for early stopping
+    max_value: 1000
+    min_value: 0
+    type: INTEGER
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    value: 0
+    visible_in_ui: true
+    warning: This is applied exclusively when early stopping is enabled.
+  use_adaptive_interval:
+    affects_outcome_of: TRAINING
+    default_value: true
+    description: Depending on the size of iteration per epoch, adaptively update the validation interval and related values.
+    editable: true
+    header: Use adaptive validation interval
+    type: BOOLEAN
+    ui_rules:
+      action: DISABLE_EDITING
+      operator: AND
+      rules: []
+      type: UI_RULES
+    visible_in_ui: true
+    warning: This will automatically control the patience and interval when early stopping is enabled.
   type: PARAMETER_GROUP
   visible_in_ui: true
 postprocessing: