Add aliases to presets, improve preset documentation (open-mmlab#1593)

core/src/autogluon/core/utils/decorators.py

@@ -1,5 +1,6 @@
 import functools
 import logging
+from typing import Dict
 
 logger = logging.getLogger(__name__)
 
@@ -31,23 +32,47 @@ def _call(*args, **kwargs):
     return _unpack_inner
 
 
-def _apply_presets(preset_dict, *args, **kwargs):
+def _apply_presets(preset_dict: Dict[str, dict], presets_alias: Dict[str, str] = None, *args, **kwargs):
     """
     Pair with `unpack` to alter input arguments with preset values.
+
+    Parameters
+    ----------
+    preset_dict : Dict[str, dict]
+        Dictionary of preset keys that map to dictionaries of key-word values.
+    presets_alias : Dict[str, str], optional
+        Dictionary of aliases of the presets in preset_dict.
+        Aliases will be remapped to the original preset in preset_dict.
+    presets : str or list, optional
+        List of preset keys (and/or aliases) to apply.
+        If str, then it is converted to a 1 element long list.
+        presets are applied from first-to-last.
+        If a key-word is specified in multiple presets in the list, the value will be set to the value of the last preset with that key-word.
+    *args, **kwargs:
+        The original args and kwargs (including presets as a potential kwarg).
+        args and kwargs take priority over presets, and if specified in the input will not be overwritten.
+        Presets will add new key-values to kwargs if the key did not previously exist.
+
+    Returns
+    -------
+    (*args, **kwargs) with kwargs updated based on specified presets.
     """
-    if 'presets' in kwargs:
-        presets = kwargs['presets']
-        if presets is None:
-            return kwargs
+    presets = kwargs.get('presets', None)
+    if presets is not None:
         if not isinstance(presets, list):
             presets = [presets]
         preset_kwargs = {}
         for preset in presets:
             if isinstance(preset, str):
-                preset_orig = preset
-                preset = preset_dict.get(preset, None)
+                preset_og = preset
+                preset = preset_dict.get(preset_og, None)
+                if preset is None and presets_alias is not None:
+                    preset = presets_alias.get(preset_og, None)
+                    if preset is not None:
+                        logger.log(20, f"Preset alias specified: '{preset_og}' maps to '{preset}'.")
+                        preset = preset_dict.get(preset, None)
                 if preset is None:
-                    raise ValueError(f'Preset \'{preset_orig}\' was not found. Valid presets: {list(preset_dict.keys())}')
+                    raise ValueError(f'Preset \'{preset_og}\' was not found. Valid presets: {list(preset_dict.keys())}')
             if isinstance(preset, dict):
                 for key in preset:
                     preset_kwargs[key] = preset[key]
@@ -59,6 +84,6 @@ def _apply_presets(preset_dict, *args, **kwargs):
     return args, kwargs
 
 
-def apply_presets(preset_dict):
+def apply_presets(preset_dict: Dict[str, dict], presets_alias: Dict[str, str] = None):
     """Used as a decorator"""
-    return unpack(_apply_presets, preset_dict)
+    return unpack(_apply_presets, preset_dict, presets_alias)

core/tests/unittests/utils/decorators/test_presets.py

@@ -0,0 +1,92 @@
+import unittest
+
+from autogluon.core.utils.decorators import apply_presets
+
+
+class TestPresets(unittest.TestCase):
+    def test_presets(self):
+        presets_dict = dict(
+            preset_1=dict(a=2, b=3)
+        )
+
+        @apply_presets(presets_dict, None)
+        def get_presets(**kwargs):
+            return kwargs
+
+        # assert no presets works
+        out = get_presets()
+        assert len(out) == 0
+
+        # assert no presets works with user-specified values
+        out = get_presets(a=5)
+        assert out['a'] == 5
+        assert len(out) == 1
+
+        # assert ValueError raised if unknown preset
+        self.assertRaises(ValueError, get_presets, presets='invalid_preset')
+
+        # assert presets == None works
+        out = get_presets(presets=None)
+        assert out['presets'] is None
+        assert len(out) == 1
+
+        # assert presets as str works
+        out = get_presets(presets='preset_1')
+        assert out['presets'] == 'preset_1'
+        assert out['a'] == 2
+        assert out['b'] == 3
+        assert len(out) == 3
+
+        # assert presets as list works
+        out = get_presets(presets=['preset_1'])
+        assert out['presets'] == ['preset_1']
+        assert out['a'] == 2
+        assert out['b'] == 3
+        assert len(out) == 3
+
+        # assert custom preset works
+        custom_preset = dict(a=4, c=7)
+        out = get_presets(presets=custom_preset)
+        assert out['presets'] == custom_preset
+        assert out['a'] == 4
+        assert out['c'] == 7
+        assert len(out) == 3
+
+        # assert that multiple presets can be specified, and later ones overwrite earlier ones in shared keys
+        out = get_presets(presets=['preset_1', custom_preset])
+        assert out['presets'] == ['preset_1', custom_preset]
+        assert out['a'] == 4
+        assert out['b'] == 3
+        assert out['c'] == 7
+        assert len(out) == 4
+
+        # assert ValueError raised if unknown preset in list of presets
+        self.assertRaises(ValueError, get_presets, presets=['preset_1', 'invalid_preset'])
+
+        # assert that multiple presets can be specified, and later ones overwrite earlier ones in shared keys, but user-specified keys override presets
+        out = get_presets(a=1, presets=['preset_1', custom_preset], d=None)
+        assert out['presets'] == ['preset_1', custom_preset]
+        assert out['a'] == 1
+        assert out['b'] == 3
+        assert out['c'] == 7
+        assert out['d'] is None
+        assert len(out) == 5
+
+        presets_alias_dict = dict(
+            preset_1_alias='preset_1',
+            preset_invalid_alias='invalid_preset',
+        )
+
+        @apply_presets(presets_dict, presets_alias_dict)
+        def get_presets(**kwargs):
+            return kwargs
+
+        # assert preset alias works
+        out = get_presets(presets='preset_1_alias')
+        assert out['presets'] == 'preset_1_alias'
+        assert out['a'] == 2
+        assert out['b'] == 3
+        assert len(out) == 3
+
+        # assert ValueError raised if alias points to invalid preset
+        self.assertRaises(ValueError, get_presets, presets='preset_invalid_alias')

docs/tutorials/tabular_prediction/tabular-indepth.md

@@ -392,7 +392,7 @@ Instead of trying to speed up a cumbersome trained model at prediction time, if
 One option is to specify more lightweight `presets`:
 
 ```{.python .input}
-presets = ['good_quality_faster_inference_only_refit', 'optimize_for_deployment']
+presets = ['good_quality', 'optimize_for_deployment']
 predictor_light = TabularPredictor(label=label, eval_metric=metric).fit(train_data, presets=presets, time_limit=30)
 ```
 

docs/tutorials/tabular_prediction/tabular-multilabel.md

@@ -269,4 +269,4 @@ In order to obtain the best predictions, you should generally add the following
 If you find that too much memory/disk is being used, try calling `MultilabelPredictor.fit()` with additional arguments discussed under ["If you encounter memory issues" in the In Depth Tutorial](tabular-indepth.html#if-you-encounter-memory-issues) or ["If you encounter disk space issues"](tabular-indepth.html#if-you-encounter-disk-space-issues).
 
 If you find inference too slow, you can try the strategies discussed under ["Accelerating Inference" in the In Depth Tutorial](tabular-indepth.html#accelerating-inference).
-In particular, simply try specifying the following preset in `MultilabelPredictor.fit()`: `presets = ['good_quality_faster_inference_only_refit', 'optimize_for_deployment']`
+In particular, simply try specifying the following preset in `MultilabelPredictor.fit()`: `presets = ['good_quality', 'optimize_for_deployment']`

docs/tutorials/tabular_prediction/tabular-quickstart.md

@@ -117,6 +117,23 @@ predictor.predict(test_data, model='LightGBM')
 
 Above the scores of predictive performance were based on a default evaluation metric (accuracy for binary classification). Performance in certain applications may be measured by different metrics than the ones AutoGluon optimizes for by default. If you know the metric that counts in your application, you should specify it as demonstrated in the next section.
 
+## Presets
+
+AutoGluon comes with a variety of presets that can be specified in the call to `.fit` via the `presets` argument. `medium_quality` is used by default to encourage initial prototyping, but for serious usage, the other presets should be used instead.
+
+| Preset                            | Model Quality                                          | Use Cases                                                                                                                                               | Fit Time (Ideal) | Inference Time (Relative to medium_quality) | Disk Usage |
+|:----------------------------------|:-------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------|:--------------------------------------------|:-----------|
+| best_quality                      | State-of-the-art (SOTA), much better than high_quality | When accuracy is what matters                                                                                                                           | 16x+             | 32x+                                        | 16x+       |
+| high_quality                      | Better than good_quality                               | When a very powerful, portable solution with fast inference is required: Large-scale batch inference                                                    | 16x              | 4x                                          | 2x         |
+| good_quality                      | Significantly better than medium_quality               | When a powerful, highly portable solution with very fast inference is required: Billion-scale batch inference, sub-100ms online-inference, edge-devices | 16x              | 2x                                          | 0.1x       |
+| medium_quality                    | Competitive with other top AutoML Frameworks           | Initial prototyping, establishing a performance baseline                                                                                                | 1x               | 1x                                          | 1x         |
+
+We recommend users to start with `medium_quality` to get a sense of the problem and identify any data related issues. If `medium_quality` is taking too long to train, consider subsampling the training data during this prototyping phase.  
+Once you are comfortable, next try `best_quality`. Make sure to specify at least 16x the `time_limit` value as used in `medium_quality`. Once finished, you should have a very powerful solution that is often stronger than `medium_quality`.  
+Make sure to consider holding out test data that AutoGluon never sees during training to ensure that the models are performing as expected in terms of performance.  
+Once you evaluate both `best_quality` and `medium_quality`, check if either satisfies your needs. If neither do, consider trying `high_quality` and/or `good_quality`.  
+If none of the presets satisfy requirements, refer to :ref:`sec_tabularadvanced` for more advanced AutoGluon options.
+
 ## Maximizing predictive performance
 
 **Note:** You should not call `fit()` with entirely default arguments if you are benchmarking AutoGluon-Tabular or hoping to maximize its accuracy!
@@ -131,7 +148,7 @@ predictor.leaderboard(test_data, silent=True)
 
 This command implements the following strategy to maximize accuracy:
 
-- Specify the argument `presets='best_quality'`, which allows AutoGluon to automatically construct powerful model ensembles based on [stacking/bagging](https://arxiv.org/abs/2003.06505), and will greatly improve the resulting predictions if granted sufficient training time. The default value of `presets` is `'medium_quality_faster_train'`, which produces *less* accurate models but facilitates faster prototyping. With `presets`, you can flexibly prioritize predictive accuracy vs. training/inference speed. For example, if you care less about predictive performance and want to quickly deploy a basic model, consider using: `presets=['good_quality_faster_inference_only_refit', 'optimize_for_deployment']`.
+- Specify the argument `presets='best_quality'`, which allows AutoGluon to automatically construct powerful model ensembles based on [stacking/bagging](https://arxiv.org/abs/2003.06505), and will greatly improve the resulting predictions if granted sufficient training time. The default value of `presets` is `'medium_quality'`, which produces *less* accurate models but facilitates faster prototyping. With `presets`, you can flexibly prioritize predictive accuracy vs. training/inference speed. For example, if you care less about predictive performance and want to quickly deploy a basic model, consider using: `presets=['good_quality', 'optimize_for_deployment']`.
 
 - Provide the parameter `eval_metric` to `TabularPredictor()` if you know what metric will be used to evaluate predictions in your application. Some other non-default metrics you might use include things like: `'f1'` (for binary classification), `'roc_auc'` (for binary classification), `'log_loss'` (for classification), `'mean_absolute_error'` (for regression), `'median_absolute_error'` (for regression).  You can also define your own custom metric function.  For more information refer to :ref:`sec_tabularcustommetric`
 
@@ -171,3 +188,7 @@ predictor_age.leaderboard(test_data, silent=True)
 **Data Formats:** AutoGluon can currently operate on data tables already loaded into Python as pandas DataFrames, or those stored in files of [CSV format](https://en.wikipedia.org/wiki/Comma-separated_values) or [Parquet format](https://databricks.com/glossary/what-is-parquet). If your data live in multiple tables, you will first need to join them into a single table whose rows correspond to statistically independent observations (datapoints) and columns correspond to different features (aka. variables/covariates).
 
 Refer to the [TabularPredictor documentation](../../api/autogluon.predictor.html#autogluon.tabular.TabularPredictor.fit) to see all of the available methods/options.
+
+## Advanced Usage
+
+For more advanced usage examples of AutoGluon, refer to :ref:`sec_tabularadvanced`

tabular/src/autogluon/tabular/configs/config_helper.py

@@ -112,7 +112,7 @@ def presets(self, presets: Union[str, list, dict]) -> ConfigBuilder:
         """
         List of preset configurations for various arguments in `fit()`. Can significantly impact predictive accuracy, memory-footprint, and inference latency of trained models, and various other properties of the returned `predictor`.
         It is recommended to specify presets and avoid specifying most other `fit()` arguments or model hyperparameters prior to becoming familiar with AutoGluon.
-        Available Presets: ['best_quality', 'high_quality_fast_inference_only_refit', 'good_quality_faster_inference_only_refit', 'medium_quality_faster_train', 'optimize_for_deployment', 'ignore_text']
+        Available Presets: ['best_quality', 'high_quality', 'good_quality', 'medium_quality', 'optimize_for_deployment', 'ignore_text']
         It is recommended to only use one `quality` based preset in a given call to `fit()` as they alter many of the same arguments and are not compatible with each-other.
         If there is an overlap in presets keys, the latter presets will override the earlier ones.
         """

tabular/src/autogluon/tabular/configs/presets_configs.py

@@ -3,25 +3,29 @@
 tabular_presets_dict = dict(
     # Best predictive accuracy with little consideration to inference time or disk usage. Achieve even better results by specifying a large time_limit value.
     # Recommended for applications that benefit from the best possible model accuracy.
+    # Aliases: best
     best_quality={'auto_stack': True},
 
     # High predictive accuracy with fast inference. ~10x-200x faster inference and ~10x-200x lower disk usage than `best_quality`.
     # Recommended for applications that require reasonable inference speed and/or model size.
-    high_quality_fast_inference_only_refit={'auto_stack': True, 'refit_full': True, 'set_best_to_refit_full': True, '_save_bag_folds': False},
+    # Aliases: high, high_quality_fast_inference_only_refit
+    high_quality={'auto_stack': True, 'refit_full': True, 'set_best_to_refit_full': True, '_save_bag_folds': False},
 
-    # Good predictive accuracy with very fast inference. ~4x faster inference and ~4x lower disk usage than `high_quality_fast_inference_only_refit`.
+    # Good predictive accuracy with very fast inference. ~4x faster inference and ~4x lower disk usage than `high_quality`.
     # Recommended for applications that require fast inference speed.
-    good_quality_faster_inference_only_refit={'auto_stack': True, 'refit_full': True, 'set_best_to_refit_full': True, '_save_bag_folds': False, 'hyperparameters': 'light'},
+    # Aliases: good, good_quality_faster_inference_only_refit
+    good_quality={'auto_stack': True, 'refit_full': True, 'set_best_to_refit_full': True, '_save_bag_folds': False, 'hyperparameters': 'light'},
 
-    # Medium predictive accuracy with very fast inference and very fast training time. ~20x faster training than `good_quality_faster_inference_only_refit`.
-    # This is the default preset in AutoGluon, but should generally only be used for quick prototyping, as `good_quality_faster_inference_only_refit` results in significantly better predictive accuracy and faster inference time.
-    medium_quality_faster_train={'auto_stack': False},
+    # Medium predictive accuracy with very fast inference and very fast training time. ~20x faster training than `good_quality`.
+    # This is the default preset in AutoGluon, but should generally only be used for quick prototyping, as `good_quality` results in significantly better predictive accuracy and faster inference time.
+    # Aliases: medium, medium_quality_faster_train
+    medium_quality={'auto_stack': False},
 
     # Optimizes result immediately for deployment by deleting unused models and removing training artifacts.
     # Often can reduce disk usage by ~2-4x with no negatives to model accuracy or inference speed.
     # This will disable numerous advanced functionality, but has no impact on inference.
     # Recommended for applications where the inner details of AutoGluon's training is not important and there is no intention of manually choosing between the final models.
-    # This preset pairs well with the other presets such as `good_quality_faster_inference_only_refit` to make a very compact final model.
+    # This preset pairs well with the other presets such as `good_quality` to make a very compact final model.
     # Identical to calling `predictor.delete_models(models_to_keep='best', dry_run=False)` and `predictor.save_space()` directly after `fit()`.
     optimize_for_deployment={'keep_only_best': True, 'save_space': True},
 
@@ -31,3 +35,15 @@
 
     # TODO: Consider HPO-enabled configs if training time doesn't matter but inference latency does.
 )
+
+
+# Alias preset name alternatives
+tabular_presets_alias = dict(
+    best='best_quality',
+    high='high_quality',
+    high_quality_fast_inference_only_refit='high_quality',
+    good='good_quality',
+    good_quality_faster_inference_only_refit='good_quality',
+    medium='medium_quality',
+    medium_quality_faster_train='medium_quality',
+)