From aaef74d7fc79e5ff46ff74d53d730e36817b9161 Mon Sep 17 00:00:00 2001 From: Palermo Date: Mon, 21 Jun 2021 15:31:04 +0100 Subject: [PATCH 01/35] Add max_depth parameter to ModelSummary --- CHANGELOG.md | 2 ++ pytorch_lightning/core/lightning.py | 5 ++-- pytorch_lightning/core/memory.py | 13 ++++++++- tests/core/test_memory.py | 42 +++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b5ddef3e0a33..1987886302fcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added +- Add `max_depth` parameter to ModelSummary when mode="full" ([#8062](https://github.com/PyTorchLightning/pytorch-lightning/pull/8062)) + - Add support for named parameter groups in `LearningRateMonitor` ([#7987](https://github.com/PyTorchLightning/pytorch-lightning/pull/7987)) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index df9a3a0d362f1..c74f9455090e8 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1610,11 +1610,12 @@ def tbptt_split_batch(self, batch, split_size): return splits - def summarize(self, mode: Optional[str] = ModelSummary.MODE_DEFAULT) -> Optional[ModelSummary]: + def summarize(self, mode: Optional[str] = ModelSummary.MODE_DEFAULT, + max_depth: Optional[int] = None) -> Optional[ModelSummary]: model_summary = None if mode in ModelSummary.MODES: - model_summary = ModelSummary(self, mode=mode) + model_summary = ModelSummary(self, mode=mode, max_depth=max_depth) log.info("\n" + str(model_summary)) elif mode is not None: raise MisconfigurationException(f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 255946ec0c211..7635fbcd404e7 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -137,6 +137,8 @@ class ModelSummary(object): - `top` (default): only the top-level modules will be recorded (the children of the root module) - `full`: summarizes all layers and their submodules in the root module + max_depth: Maximum depth of modules to show when mode="full" + The string representation of this summary prints a table with columns containing the name, type and number of parameters for each layer. @@ -187,9 +189,10 @@ class ModelSummary(object): MODE_DEFAULT = MODE_TOP MODES = [MODE_FULL, MODE_TOP] - def __init__(self, model, mode: str = MODE_DEFAULT): + def __init__(self, model, mode: str = MODE_DEFAULT, max_depth: Optional[int] = None): self._model = model self._mode = mode + self._max_depth = max_depth self._layer_summary = self.summarize() # 1 byte -> 8 bits # TODO: how do we compute precisin_megabytes in case of mixed precision? @@ -249,6 +252,14 @@ def summarize(self) -> Dict[str, LayerSummary]: self._forward_example_input() for layer in summary.values(): layer.detach_hook() + + if self._max_depth is not None: + if self._mode == "top": + warning_cache.warn("ModelSummary's max_depth parameter only works when mode='full'.") + # remove summary entries with depth > max_depth + for k in [k for k in summary.keys() if k.count(".") > self._max_depth]: + del summary[k] + return summary def _forward_example_input(self) -> None: diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index a094ddfc3d80a..02b65f3dcd7de 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -114,6 +114,36 @@ def forward(self, inp): return self.layer2(self.layer1(inp)) +class DeepNestedModel(LightningModule): + """ A model with deep nested layers. """ + def __init__(self): + super().__init__() + self.branch1 = nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 3) + ) + ) + ) + ) + ) + ) + self.branch2 = nn.Linear(5, 10) + self.head = UnorderedModel() + self.example_input_array = torch.rand(2, 5) + + def forward(self, inp): + return self.head(self.branch1(inp), self.branch2(inp)) + + def test_invalid_weights_summmary(): """ Test that invalid value for weights_summary raises an error. """ with pytest.raises(MisconfigurationException, match='`mode` can be None, .* got temp'): @@ -336,3 +366,15 @@ def test_lazy_model_summary(): # https://github.com/pytorch/pytorch/issues/58350 assert summary.total_parameters == 7 assert summary.trainable_parameters == 7 + + +def test_max_depth_0_equals_mode_top(): + assert str(DeepNestedModel().summarize(mode="top")) \ + == str(DeepNestedModel().summarize(mode="full", max_depth=0)) + + +@pytest.mark.parametrize('max_depth', [None, 0, 1, 3, 999]) +def test_max_depth_param(max_depth): + assert not any([l.count(".") > max_depth for l in + DeepNestedModel().summarize(mode="full", max_depth=max_depth).layer_names + if max_depth is not None]) From 9a97169bc396d5798d4383f7f282ba50042843ea Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 14:33:11 +0000 Subject: [PATCH 02/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/core/lightning.py | 3 ++- tests/core/test_memory.py | 26 ++++++++++---------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index c74f9455090e8..f7ebd3b28949d 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1610,7 +1610,8 @@ def tbptt_split_batch(self, batch, split_size): return splits - def summarize(self, mode: Optional[str] = ModelSummary.MODE_DEFAULT, + def summarize(self, + mode: Optional[str] = ModelSummary.MODE_DEFAULT, max_depth: Optional[int] = None) -> Optional[ModelSummary]: model_summary = None diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index 02b65f3dcd7de..dd7662ee6f82d 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -116,25 +116,18 @@ def forward(self, inp): class DeepNestedModel(LightningModule): """ A model with deep nested layers. """ + def __init__(self): super().__init__() self.branch1 = nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( nn.Linear(5, 5), nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 3) - ) - ) - ) - ) + nn.Linear(5, 5), + nn.Sequential(nn.Linear(5, 5), nn.Sequential(nn.Linear(5, 5), nn.Sequential(nn.Linear(5, 3)))) ) + ) ) self.branch2 = nn.Linear(5, 10) self.head = UnorderedModel() @@ -375,6 +368,7 @@ def test_max_depth_0_equals_mode_top(): @pytest.mark.parametrize('max_depth', [None, 0, 1, 3, 999]) def test_max_depth_param(max_depth): - assert not any([l.count(".") > max_depth for l in - DeepNestedModel().summarize(mode="full", max_depth=max_depth).layer_names - if max_depth is not None]) + assert not any([ + l.count(".") > max_depth + for l in DeepNestedModel().summarize(mode="full", max_depth=max_depth).layer_names if max_depth is not None + ]) From 27528211b62559ae504afa2ad870a37a8173d7ac Mon Sep 17 00:00:00 2001 From: Palermo Date: Mon, 21 Jun 2021 15:58:50 +0100 Subject: [PATCH 03/35] Better test coverage + fix pep8 on ModelSummary max_depth --- pytorch_lightning/core/lightning.py | 5 +++-- pytorch_lightning/core/memory.py | 7 ++++--- tests/core/test_memory.py | 25 ++++++++++++++++--------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index f7ebd3b28949d..13a129d329326 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1611,8 +1611,9 @@ def tbptt_split_batch(self, batch, split_size): return splits def summarize(self, - mode: Optional[str] = ModelSummary.MODE_DEFAULT, - max_depth: Optional[int] = None) -> Optional[ModelSummary]: + mode: str = ModelSummary.MODE_DEFAULT, + max_depth: Optional[int] = None + ) -> Optional[ModelSummary]: model_summary = None if mode in ModelSummary.MODES: diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 7635fbcd404e7..c2bcfcde20575 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -256,9 +256,10 @@ def summarize(self) -> Dict[str, LayerSummary]: if self._max_depth is not None: if self._mode == "top": warning_cache.warn("ModelSummary's max_depth parameter only works when mode='full'.") - # remove summary entries with depth > max_depth - for k in [k for k in summary.keys() if k.count(".") > self._max_depth]: - del summary[k] + else: + # remove summary entries with depth > max_depth + for k in [k for k in summary.keys() if k.count(".") > self._max_depth]: + del summary[k] return summary diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index dd7662ee6f82d..530ac79e01a2f 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -120,14 +120,22 @@ class DeepNestedModel(LightningModule): def __init__(self): super().__init__() self.branch1 = nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( nn.Linear(5, 5), nn.Sequential( - nn.Linear(5, 5), - nn.Sequential(nn.Linear(5, 5), nn.Sequential(nn.Linear(5, 5), nn.Sequential(nn.Linear(5, 3)))) + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 3) + ) + ) + ) + ) ) - ) ) self.branch2 = nn.Linear(5, 10) self.head = UnorderedModel() @@ -368,7 +376,6 @@ def test_max_depth_0_equals_mode_top(): @pytest.mark.parametrize('max_depth', [None, 0, 1, 3, 999]) def test_max_depth_param(max_depth): - assert not any([ - l.count(".") > max_depth - for l in DeepNestedModel().summarize(mode="full", max_depth=max_depth).layer_names if max_depth is not None - ]) + assert not any([l.count(".") > max_depth for l in + DeepNestedModel().summarize(mode="full", max_depth=max_depth).layer_names + if max_depth is not None]) From 52d1c233c2b684c358fb4a364086370298e99a53 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 15:03:23 +0000 Subject: [PATCH 04/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/core/lightning.py | 3 +-- tests/core/test_memory.py | 25 +++++++++---------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 13a129d329326..918c7dc8ab49e 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1612,8 +1612,7 @@ def tbptt_split_batch(self, batch, split_size): def summarize(self, mode: str = ModelSummary.MODE_DEFAULT, - max_depth: Optional[int] = None - ) -> Optional[ModelSummary]: + max_depth: Optional[int] = None) -> Optional[ModelSummary]: model_summary = None if mode in ModelSummary.MODES: diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index 530ac79e01a2f..dd7662ee6f82d 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -120,22 +120,14 @@ class DeepNestedModel(LightningModule): def __init__(self): super().__init__() self.branch1 = nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( nn.Linear(5, 5), nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 3) - ) - ) - ) - ) + nn.Linear(5, 5), + nn.Sequential(nn.Linear(5, 5), nn.Sequential(nn.Linear(5, 5), nn.Sequential(nn.Linear(5, 3)))) ) + ) ) self.branch2 = nn.Linear(5, 10) self.head = UnorderedModel() @@ -376,6 +368,7 @@ def test_max_depth_0_equals_mode_top(): @pytest.mark.parametrize('max_depth', [None, 0, 1, 3, 999]) def test_max_depth_param(max_depth): - assert not any([l.count(".") > max_depth for l in - DeepNestedModel().summarize(mode="full", max_depth=max_depth).layer_names - if max_depth is not None]) + assert not any([ + l.count(".") > max_depth + for l in DeepNestedModel().summarize(mode="full", max_depth=max_depth).layer_names if max_depth is not None + ]) From f5919dc881cf2af52f11ee26bdf91fbbd5014051 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 21 Jun 2021 17:40:22 +0200 Subject: [PATCH 05/35] Apply suggestions from code review --- pytorch_lightning/core/lightning.py | 2 +- tests/core/test_memory.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 918c7dc8ab49e..321ba94fa4038 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1612,7 +1612,7 @@ def tbptt_split_batch(self, batch, split_size): def summarize(self, mode: str = ModelSummary.MODE_DEFAULT, - max_depth: Optional[int] = None) -> Optional[ModelSummary]: + max_depth: Optional[int] = None,) -> Optional[ModelSummary]: model_summary = None if mode in ModelSummary.MODES: diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index dd7662ee6f82d..42cd121370232 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -362,8 +362,9 @@ def test_lazy_model_summary(): def test_max_depth_0_equals_mode_top(): - assert str(DeepNestedModel().summarize(mode="top")) \ - == str(DeepNestedModel().summarize(mode="full", max_depth=0)) + summary_top = DeepNestedModel().summarize(mode="top") + summary_full_0 = DeepNestedModel().summarize(mode="full", max_depth=0) + assert str(summary_top) == str(summary_full_0) @pytest.mark.parametrize('max_depth', [None, 0, 1, 3, 999]) From 8771707934e1e69bf99c5ee6ecaa932b2f09b5a7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 15:41:25 +0000 Subject: [PATCH 06/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/core/lightning.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 321ba94fa4038..8e761ddd59466 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1610,9 +1610,11 @@ def tbptt_split_batch(self, batch, split_size): return splits - def summarize(self, - mode: str = ModelSummary.MODE_DEFAULT, - max_depth: Optional[int] = None,) -> Optional[ModelSummary]: + def summarize( + self, + mode: str = ModelSummary.MODE_DEFAULT, + max_depth: Optional[int] = None, + ) -> Optional[ModelSummary]: model_summary = None if mode in ModelSummary.MODES: From 9d05d7f78ac2a08708c11f6333f7aa4ba1d15b0b Mon Sep 17 00:00:00 2001 From: Palermo Date: Mon, 21 Jun 2021 18:28:21 +0100 Subject: [PATCH 07/35] Removed mode parameter inside ModelSummary --- CHANGELOG.md | 2 +- pytorch_lightning/core/lightning.py | 12 ++++----- pytorch_lightning/core/memory.py | 38 ++++++++++------------------- tests/core/test_memory.py | 30 ++++++++++++++--------- 4 files changed, 37 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1987886302fcf..1792daf69f3fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added -- Add `max_depth` parameter to ModelSummary when mode="full" ([#8062](https://github.com/PyTorchLightning/pytorch-lightning/pull/8062)) +- Replaced `mode` parameter with `max_depth` in ModelSummary" ([#8062](https://github.com/PyTorchLightning/pytorch-lightning/pull/8062)) - Add support for named parameter groups in `LearningRateMonitor` ([#7987](https://github.com/PyTorchLightning/pytorch-lightning/pull/7987)) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 8e761ddd59466..4474e55b66085 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1610,18 +1610,16 @@ def tbptt_split_batch(self, batch, split_size): return splits - def summarize( - self, - mode: str = ModelSummary.MODE_DEFAULT, - max_depth: Optional[int] = None, - ) -> Optional[ModelSummary]: + def summarize(self, mode: str = "top") -> Optional[ModelSummary]: model_summary = None if mode in ModelSummary.MODES: - model_summary = ModelSummary(self, mode=mode, max_depth=max_depth) + max_depth = ModelSummary.MODES[mode] # map from mode interface to max_depth + model_summary = ModelSummary(self, max_depth=max_depth) log.info("\n" + str(model_summary)) elif mode is not None: - raise MisconfigurationException(f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") + raise MisconfigurationException( + f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") return model_summary diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index c2bcfcde20575..eeacec2fb423d 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -132,12 +132,7 @@ class ModelSummary(object): Args: model: The model to summarize (also referred to as the root module) - mode: Can be one of - - - `top` (default): only the top-level modules will be recorded (the children of the root module) - - `full`: summarizes all layers and their submodules in the root module - - max_depth: Maximum depth of modules to show when mode="full" + max_depth: Maximum depth of modules to show, or -1 to show all modules. The string representation of this summary prints a table with columns containing the name, type and number of parameters for each layer. @@ -162,7 +157,7 @@ class ModelSummary(object): ... return self.net(x) ... >>> model = LitModel() - >>> ModelSummary(model, mode='top') # doctest: +NORMALIZE_WHITESPACE + >>> ModelSummary(model, max_depth=0) # doctest: +NORMALIZE_WHITESPACE | Name | Type | Params | In sizes | Out sizes ------------------------------------------------------------ 0 | net | Sequential | 132 K | [10, 256] | [10, 512] @@ -171,7 +166,7 @@ class ModelSummary(object): 0 Non-trainable params 132 K Total params 0.530 Total estimated model params size (MB) - >>> ModelSummary(model, mode='full') # doctest: +NORMALIZE_WHITESPACE + >>> ModelSummary(model, max_depth=-1) # doctest: +NORMALIZE_WHITESPACE | Name | Type | Params | In sizes | Out sizes -------------------------------------------------------------- 0 | net | Sequential | 132 K | [10, 256] | [10, 512] @@ -184,14 +179,10 @@ class ModelSummary(object): 0.530 Total estimated model params size (MB) """ - MODE_TOP = "top" - MODE_FULL = "full" - MODE_DEFAULT = MODE_TOP - MODES = [MODE_FULL, MODE_TOP] + MODES = dict(top=0, full=-1) - def __init__(self, model, mode: str = MODE_DEFAULT, max_depth: Optional[int] = None): + def __init__(self, model, max_depth: int = -1): self._model = model - self._mode = mode self._max_depth = max_depth self._layer_summary = self.summarize() # 1 byte -> 8 bits @@ -201,12 +192,12 @@ def __init__(self, model, mode: str = MODE_DEFAULT, max_depth: Optional[int] = N @property def named_modules(self) -> List[Tuple[str, nn.Module]]: - if self._mode == ModelSummary.MODE_FULL: - mods = self._model.named_modules() - mods = list(mods)[1:] # do not include root module (LightningModule) - elif self._mode == ModelSummary.MODE_TOP: + if self._max_depth == 0: # the children are the top-level modules mods = self._model.named_children() + elif self._max_depth ==-1 or self._max_depth > 0: + mods = self._model.named_modules() + mods = list(mods)[1:] # do not include root module (LightningModule) else: mods = [] return list(mods) @@ -253,13 +244,10 @@ def summarize(self) -> Dict[str, LayerSummary]: for layer in summary.values(): layer.detach_hook() - if self._max_depth is not None: - if self._mode == "top": - warning_cache.warn("ModelSummary's max_depth parameter only works when mode='full'.") - else: - # remove summary entries with depth > max_depth - for k in [k for k in summary.keys() if k.count(".") > self._max_depth]: - del summary[k] + if self._max_depth >= 0: + # remove summary entries with depth > max_depth + for k in [k for k in summary.keys() if k.count(".") > self._max_depth]: + del summary[k] return summary diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index 42cd121370232..530ac79e01a2f 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -120,14 +120,22 @@ class DeepNestedModel(LightningModule): def __init__(self): super().__init__() self.branch1 = nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( nn.Linear(5, 5), nn.Sequential( - nn.Linear(5, 5), - nn.Sequential(nn.Linear(5, 5), nn.Sequential(nn.Linear(5, 5), nn.Sequential(nn.Linear(5, 3)))) + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( + nn.Linear(5, 3) + ) + ) + ) + ) ) - ) ) self.branch2 = nn.Linear(5, 10) self.head = UnorderedModel() @@ -362,14 +370,12 @@ def test_lazy_model_summary(): def test_max_depth_0_equals_mode_top(): - summary_top = DeepNestedModel().summarize(mode="top") - summary_full_0 = DeepNestedModel().summarize(mode="full", max_depth=0) - assert str(summary_top) == str(summary_full_0) + assert str(DeepNestedModel().summarize(mode="top")) \ + == str(DeepNestedModel().summarize(mode="full", max_depth=0)) @pytest.mark.parametrize('max_depth', [None, 0, 1, 3, 999]) def test_max_depth_param(max_depth): - assert not any([ - l.count(".") > max_depth - for l in DeepNestedModel().summarize(mode="full", max_depth=max_depth).layer_names if max_depth is not None - ]) + assert not any([l.count(".") > max_depth for l in + DeepNestedModel().summarize(mode="full", max_depth=max_depth).layer_names + if max_depth is not None]) From af2aaa4aa31349e07336884f99b2d898c7e7f5cc Mon Sep 17 00:00:00 2001 From: Palermo Date: Mon, 21 Jun 2021 18:32:52 +0100 Subject: [PATCH 08/35] Removed mode parameter inside ModelSummary --- pytorch_lightning/core/lightning.py | 2 +- tests/core/test_memory.py | 51 ++++++++++++++++++----------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 4474e55b66085..50c41e3a6d113 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1610,7 +1610,7 @@ def tbptt_split_batch(self, batch, split_size): return splits - def summarize(self, mode: str = "top") -> Optional[ModelSummary]: + def summarize(self, mode: Optional[str] = "top") -> Optional[ModelSummary]: model_summary = None if mode in ModelSummary.MODES: diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index 530ac79e01a2f..2f8621137e667 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -154,8 +154,8 @@ def test_invalid_weights_summmary(): Trainer(weights_summary='temp') -@pytest.mark.parametrize('mode', [ModelSummary.MODE_FULL, ModelSummary.MODE_TOP]) -def test_empty_model_summary_shapes(mode: ModelSummary): +@pytest.mark.parametrize('mode', ["full", "top"]) +def test_empty_model_summary_shapes(mode: str): """ Test that the summary works for models that have no submodules. """ model = EmptyModule() summary = model.summarize(mode=mode) @@ -165,7 +165,7 @@ def test_empty_model_summary_shapes(mode: ModelSummary): @RunIf(min_gpus=1) -@pytest.mark.parametrize('mode', [ModelSummary.MODE_FULL, ModelSummary.MODE_TOP]) +@pytest.mark.parametrize('mode', ["full", "top"]) @pytest.mark.parametrize(['device'], [ pytest.param(torch.device('cpu')), pytest.param(torch.device('cuda', 0)), @@ -208,18 +208,18 @@ def test_mixed_dtype_model_summary(): ] -@pytest.mark.parametrize('mode', [ModelSummary.MODE_FULL, ModelSummary.MODE_TOP]) -def test_hooks_removed_after_summarize(mode): +@pytest.mark.parametrize('max_depth', [-1, 0]) +def test_hooks_removed_after_summarize(max_depth): """ Test that all hooks were properly removed after summary, even ones that were not run. """ model = UnorderedModel() - summary = ModelSummary(model, mode=mode) + summary = ModelSummary(model, max_depth=max_depth) # hooks should be removed for _, layer in summary.summarize().items(): handle = layer._hook_handle assert handle.id not in handle.hooks_dict_ref() -@pytest.mark.parametrize('mode', [ModelSummary.MODE_FULL, ModelSummary.MODE_TOP]) +@pytest.mark.parametrize('mode', ["full", "top"]) def test_rnn_summary_shapes(mode): """ Test that the model summary works for RNNs. """ model = ParityModuleRNN() @@ -243,7 +243,7 @@ def test_rnn_summary_shapes(mode): ] -@pytest.mark.parametrize('mode', [ModelSummary.MODE_FULL, ModelSummary.MODE_TOP]) +@pytest.mark.parametrize('mode', ["full", "top"]) def test_summary_parameter_count(mode): """ Test that the summary counts the number of parameters in every submodule. """ model = UnorderedModel() @@ -257,7 +257,7 @@ def test_summary_parameter_count(mode): ] -@pytest.mark.parametrize('mode', [ModelSummary.MODE_FULL, ModelSummary.MODE_TOP]) +@pytest.mark.parametrize('mode', ["full", "top"]) def test_summary_layer_types(mode): """ Test that the summary displays the layer names correctly. """ model = UnorderedModel() @@ -271,7 +271,7 @@ def test_summary_layer_types(mode): ] -@pytest.mark.parametrize('mode', [ModelSummary.MODE_FULL, ModelSummary.MODE_TOP]) +@pytest.mark.parametrize('mode', ["full", "top"]) def test_summary_with_scripted_modules(mode): model = PartialScriptModel() summary = model.summarize(mode=mode) @@ -280,7 +280,7 @@ def test_summary_with_scripted_modules(mode): assert summary.out_sizes == [UNKNOWN_SIZE, [2, 2]] -@pytest.mark.parametrize('mode', [ModelSummary.MODE_FULL, ModelSummary.MODE_TOP]) +@pytest.mark.parametrize('mode', ["full", "top"]) @pytest.mark.parametrize(['example_input', 'expected_size'], [ pytest.param([], UNKNOWN_SIZE), pytest.param((1, 2, 3), [UNKNOWN_SIZE] * 3), @@ -314,7 +314,7 @@ def forward(self, *args, **kwargs): assert summary.in_sizes == [expected_size] -@pytest.mark.parametrize('mode', [ModelSummary.MODE_FULL, ModelSummary.MODE_TOP]) +@pytest.mark.parametrize('mode', ["full", "top"]) def test_model_size(mode): """ Test model size is calculated correctly. """ model = PreCalculatedModel() @@ -322,7 +322,7 @@ def test_model_size(mode): assert model.pre_calculated_model_size == summary.model_size -@pytest.mark.parametrize('mode', [ModelSummary.MODE_FULL, ModelSummary.MODE_TOP]) +@pytest.mark.parametrize('mode', ["full", "top"]) def test_empty_model_size(mode): """ Test empty model size is zero. """ model = EmptyModule() @@ -369,13 +369,24 @@ def test_lazy_model_summary(): assert summary.trainable_parameters == 7 -def test_max_depth_0_equals_mode_top(): - assert str(DeepNestedModel().summarize(mode="top")) \ - == str(DeepNestedModel().summarize(mode="full", max_depth=0)) +def test_max_depth_equals_mode_interface(): + """Test model.summarize(full/top) interface matches ModelSummary mapping""" + model = DeepNestedModel() + summary_top = model.summarize(mode="top") + summary_0 = ModelSummary(model, max_depth=0) + assert str(summary_top) == str(summary_0) -@pytest.mark.parametrize('max_depth', [None, 0, 1, 3, 999]) + summary_full = model.summarize(mode="full") + summary_minus1 = ModelSummary(model, max_depth=-1) + assert str(summary_full) == str(summary_minus1) + + +@pytest.mark.parametrize('max_depth', [-1, 0, 1, 3, 999]) def test_max_depth_param(max_depth): - assert not any([l.count(".") > max_depth for l in - DeepNestedModel().summarize(mode="full", max_depth=max_depth).layer_names - if max_depth is not None]) + """Check if only modules with desired depth are shown""" + model = DeepNestedModel() + summary = ModelSummary(model, max_depth=max_depth) + for lname in summary.layer_names: + if max_depth >= 0: + assert lname.count(".") <= max_depth From 658140fab6b37c7b826ef61254981556fc22853a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 17:34:15 +0000 Subject: [PATCH 09/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/core/lightning.py | 5 ++--- pytorch_lightning/core/memory.py | 2 +- tests/core/test_memory.py | 18 +++++------------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 50c41e3a6d113..aa1181a651a85 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1614,12 +1614,11 @@ def summarize(self, mode: Optional[str] = "top") -> Optional[ModelSummary]: model_summary = None if mode in ModelSummary.MODES: - max_depth = ModelSummary.MODES[mode] # map from mode interface to max_depth + max_depth = ModelSummary.MODES[mode] # map from mode interface to max_depth model_summary = ModelSummary(self, max_depth=max_depth) log.info("\n" + str(model_summary)) elif mode is not None: - raise MisconfigurationException( - f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") + raise MisconfigurationException(f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") return model_summary diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index eeacec2fb423d..472b20c82949f 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -195,7 +195,7 @@ def named_modules(self) -> List[Tuple[str, nn.Module]]: if self._max_depth == 0: # the children are the top-level modules mods = self._model.named_children() - elif self._max_depth ==-1 or self._max_depth > 0: + elif self._max_depth == -1 or self._max_depth > 0: mods = self._model.named_modules() mods = list(mods)[1:] # do not include root module (LightningModule) else: diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index 2f8621137e667..cf243ef1b0e8f 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -120,22 +120,14 @@ class DeepNestedModel(LightningModule): def __init__(self): super().__init__() self.branch1 = nn.Sequential( + nn.Linear(5, 5), + nn.Sequential( nn.Linear(5, 5), nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 5), - nn.Sequential( - nn.Linear(5, 3) - ) - ) - ) - ) + nn.Linear(5, 5), + nn.Sequential(nn.Linear(5, 5), nn.Sequential(nn.Linear(5, 5), nn.Sequential(nn.Linear(5, 3)))) ) + ) ) self.branch2 = nn.Linear(5, 10) self.head = UnorderedModel() From 1afd7227ca07ce4f5f6b4836a0e6b3f152293de3 Mon Sep 17 00:00:00 2001 From: Palermo Date: Mon, 21 Jun 2021 19:17:35 +0100 Subject: [PATCH 10/35] Added deprecation warning to PL.summarize(mode=) --- pytorch_lightning/core/lightning.py | 26 +++++++++++++++++++------- pytorch_lightning/trainer/trainer.py | 3 ++- tests/core/test_memory.py | 6 +++--- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 50c41e3a6d113..4d9b5202e0b39 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1610,16 +1610,28 @@ def tbptt_split_batch(self, batch, split_size): return splits - def summarize(self, mode: Optional[str] = "top") -> Optional[ModelSummary]: + def summarize( + self, + mode: Optional[str] = "top", + max_depth: Optional[int] = None + ) -> Optional[ModelSummary]: model_summary = None - if mode in ModelSummary.MODES: - max_depth = ModelSummary.MODES[mode] # map from mode interface to max_depth + if max_depth is None: + rank_zero_deprecation( + "The use of model.summarize(mode) argument will be " + "deprecated in a future release, in favour of 'max_depth'.") + + if mode in ModelSummary.MODES: + # temporary map from mode interface to max_depth + max_depth = ModelSummary.MODES[mode] + model_summary = ModelSummary(self, max_depth=max_depth) + log.info("\n" + str(model_summary)) + elif mode is not None: + raise MisconfigurationException( + f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") + else: model_summary = ModelSummary(self, max_depth=max_depth) - log.info("\n" + str(model_summary)) - elif mode is not None: - raise MisconfigurationException( - f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") return model_summary diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 01b823755156a..ac6af1d67c55d 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -941,7 +941,8 @@ def _pre_training_routine(self): # print model summary if self.is_global_zero and self.weights_summary is not None and not self.testing: - ref_model.summarize(mode=self.weights_summary) + max_depth = ModelSummary.MODES[self.weights_summary] + ref_model.summarize(max_depth=max_depth) # on pretrain routine end self.on_pretrain_routine_end() diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index 2f8621137e667..add09a00a340e 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -370,15 +370,15 @@ def test_lazy_model_summary(): def test_max_depth_equals_mode_interface(): - """Test model.summarize(full/top) interface matches ModelSummary mapping""" + """Test model.summarize(full/top) interface mapping matches max_depth""" model = DeepNestedModel() summary_top = model.summarize(mode="top") - summary_0 = ModelSummary(model, max_depth=0) + summary_0 = model.summarize(max_depth=0) assert str(summary_top) == str(summary_0) summary_full = model.summarize(mode="full") - summary_minus1 = ModelSummary(model, max_depth=-1) + summary_minus1 = model.summarize(max_depth=-1) assert str(summary_full) == str(summary_minus1) From a1ec6cc2eb32c45033132ab669f60a2502420cba Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 18:19:06 +0000 Subject: [PATCH 11/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/core/lightning.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 4d9b5202e0b39..604d019e8ee23 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1610,17 +1610,14 @@ def tbptt_split_batch(self, batch, split_size): return splits - def summarize( - self, - mode: Optional[str] = "top", - max_depth: Optional[int] = None - ) -> Optional[ModelSummary]: + def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None) -> Optional[ModelSummary]: model_summary = None if max_depth is None: rank_zero_deprecation( - "The use of model.summarize(mode) argument will be " - "deprecated in a future release, in favour of 'max_depth'.") + "The use of model.summarize(mode) argument will be " + "deprecated in a future release, in favour of 'max_depth'." + ) if mode in ModelSummary.MODES: # temporary map from mode interface to max_depth @@ -1628,8 +1625,7 @@ def summarize( model_summary = ModelSummary(self, max_depth=max_depth) log.info("\n" + str(model_summary)) elif mode is not None: - raise MisconfigurationException( - f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") + raise MisconfigurationException(f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") else: model_summary = ModelSummary(self, max_depth=max_depth) From 2b67abba5abd388f3ee69785faeacf3df935cd1f Mon Sep 17 00:00:00 2001 From: Palermo Date: Thu, 24 Jun 2021 16:38:50 +0100 Subject: [PATCH 12/35] Improve deprecation of `mode` param in ModelSummary --- pytorch_lightning/core/lightning.py | 9 ++++----- pytorch_lightning/core/memory.py | 23 +++++++++++++++++++---- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 4d9b5202e0b39..57c662a16d92f 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1617,13 +1617,12 @@ def summarize( ) -> Optional[ModelSummary]: model_summary = None + # temporary mapping from mode to max_depth if max_depth is None: - rank_zero_deprecation( - "The use of model.summarize(mode) argument will be " - "deprecated in a future release, in favour of 'max_depth'.") - if mode in ModelSummary.MODES: - # temporary map from mode interface to max_depth + rank_zero_deprecation( + "The use of `model.summarize(mode)` argument is deprecated in v1.4 " + "and will be removed in v1.5, use `max_depth` argument instead.") max_depth = ModelSummary.MODES[mode] model_summary = ModelSummary(self, max_depth=max_depth) log.info("\n" + str(model_summary)) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 472b20c82949f..f139693a9bed9 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -131,8 +131,10 @@ class ModelSummary(object): Generates a summary of all layers in a :class:`~pytorch_lightning.core.lightning.LightningModule`. Args: - model: The model to summarize (also referred to as the root module) - max_depth: Maximum depth of modules to show, or -1 to show all modules. + model: The model to summarize (also referred to as the root module). + mode: deprecated. Please use `max_depth`. + max_depth: Maximum depth of modules to show, starting on 0 (for modules connected root), + or -1 to show all modules. The string representation of this summary prints a table with columns containing the name, type and number of parameters for each layer. @@ -179,10 +181,23 @@ class ModelSummary(object): 0.530 Total estimated model params size (MB) """ - MODES = dict(top=0, full=-1) + MODES = dict(top=0, full=-1) # temporary mapping from mode to max_depth - def __init__(self, model, max_depth: int = -1): + def __init__(self, model, mode: Optional[str] = "top", max_depth: Optional[int] = None): self._model = model + + # temporary mapping from mode to max_depth + if max_depth is None: + if mode in ModelSummary.MODES: + from pytorch_lightning.utilities import rank_zero_deprecation + rank_zero_deprecation( + "The use of `mode` argument is deprecated in v1.4 " + "and will be removed in v1.5, use `max_depth` argument instead.") + max_depth = ModelSummary.MODES[mode] + else: + raise MisconfigurationException( + f"`mode` can be {', '.join(ModelSummary.MODES)}, got {mode}.") + self._max_depth = max_depth self._layer_summary = self.summarize() # 1 byte -> 8 bits From a1c48c0779230e263a312be286de527c9ba6ce9e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 24 Jun 2021 15:40:38 +0000 Subject: [PATCH 13/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/core/lightning.py | 14 +++++--------- pytorch_lightning/core/memory.py | 10 +++++----- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 57c662a16d92f..b02fa8d6338da 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1610,25 +1610,21 @@ def tbptt_split_batch(self, batch, split_size): return splits - def summarize( - self, - mode: Optional[str] = "top", - max_depth: Optional[int] = None - ) -> Optional[ModelSummary]: + def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None) -> Optional[ModelSummary]: model_summary = None # temporary mapping from mode to max_depth if max_depth is None: if mode in ModelSummary.MODES: rank_zero_deprecation( - "The use of `model.summarize(mode)` argument is deprecated in v1.4 " - "and will be removed in v1.5, use `max_depth` argument instead.") + "The use of `model.summarize(mode)` argument is deprecated in v1.4 " + "and will be removed in v1.5, use `max_depth` argument instead." + ) max_depth = ModelSummary.MODES[mode] model_summary = ModelSummary(self, max_depth=max_depth) log.info("\n" + str(model_summary)) elif mode is not None: - raise MisconfigurationException( - f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") + raise MisconfigurationException(f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") else: model_summary = ModelSummary(self, max_depth=max_depth) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index f139693a9bed9..20a4375e97bf0 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -181,7 +181,7 @@ class ModelSummary(object): 0.530 Total estimated model params size (MB) """ - MODES = dict(top=0, full=-1) # temporary mapping from mode to max_depth + MODES = dict(top=0, full=-1) # temporary mapping from mode to max_depth def __init__(self, model, mode: Optional[str] = "top", max_depth: Optional[int] = None): self._model = model @@ -191,12 +191,12 @@ def __init__(self, model, mode: Optional[str] = "top", max_depth: Optional[int] if mode in ModelSummary.MODES: from pytorch_lightning.utilities import rank_zero_deprecation rank_zero_deprecation( - "The use of `mode` argument is deprecated in v1.4 " - "and will be removed in v1.5, use `max_depth` argument instead.") + "The use of `mode` argument is deprecated in v1.4 " + "and will be removed in v1.5, use `max_depth` argument instead." + ) max_depth = ModelSummary.MODES[mode] else: - raise MisconfigurationException( - f"`mode` can be {', '.join(ModelSummary.MODES)}, got {mode}.") + raise MisconfigurationException(f"`mode` can be {', '.join(ModelSummary.MODES)}, got {mode}.") self._max_depth = max_depth self._layer_summary = self.summarize() From fc78dc73125279543c7db4031333dfa80da44a0e Mon Sep 17 00:00:00 2001 From: Palermo Date: Thu, 24 Jun 2021 16:44:35 +0100 Subject: [PATCH 14/35] Add missing import --- pytorch_lightning/core/memory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index f139693a9bed9..52829c4eca002 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -195,6 +195,7 @@ def __init__(self, model, mode: Optional[str] = "top", max_depth: Optional[int] "and will be removed in v1.5, use `max_depth` argument instead.") max_depth = ModelSummary.MODES[mode] else: + from pytorch_lightning.utilities.exceptions import MisconfigurationException raise MisconfigurationException( f"`mode` can be {', '.join(ModelSummary.MODES)}, got {mode}.") From 32df3e78c23dcf90c216410aca6df576fa0c2076 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 24 Jun 2021 15:45:55 +0000 Subject: [PATCH 15/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/core/memory.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 371932c2204d9..301f48a32b799 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -191,15 +191,13 @@ def __init__(self, model, mode: Optional[str] = "top", max_depth: Optional[int] if mode in ModelSummary.MODES: from pytorch_lightning.utilities import rank_zero_deprecation rank_zero_deprecation( - "The use of `mode` argument is deprecated in v1.4 " - "and will be removed in v1.5, use `max_depth` argument instead." + "The use of `mode` argument is deprecated in v1.4 " + "and will be removed in v1.5, use `max_depth` argument instead." ) max_depth = ModelSummary.MODES[mode] else: from pytorch_lightning.utilities.exceptions import MisconfigurationException - raise MisconfigurationException( - f"`mode` can be {', '.join(ModelSummary.MODES)}, got {mode}." - ) + raise MisconfigurationException(f"`mode` can be {', '.join(ModelSummary.MODES)}, got {mode}.") self._max_depth = max_depth self._layer_summary = self.summarize() From 2f9e770e10d5b72739c4e7a065dcedfa81627c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 29 Jun 2021 11:59:43 +0200 Subject: [PATCH 16/35] update changelog --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eddcabe4eac78..216f71405d08d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,6 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added -- Replaced `mode` parameter with `max_depth` in ModelSummary" ([#8062](https://github.com/PyTorchLightning/pytorch-lightning/pull/8062)) - - Add support for named parameter groups in `LearningRateMonitor` ([#7987](https://github.com/PyTorchLightning/pytorch-lightning/pull/7987)) @@ -123,6 +121,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added XLA Profiler ([#8014](https://github.com/PyTorchLightning/pytorch-lightning/pull/8014)) +- Added `max_depth` parameter in `ModelSummary` ([#8062](https://github.com/PyTorchLightning/pytorch-lightning/pull/8062)) + + ### Changed @@ -264,6 +265,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Deprecated the `Trainer.train_loop` property in favor of `Trainer.fit_loop` ([#8025](https://github.com/PyTorchLightning/pytorch-lightning/pull/8025)) +- Deprecated `mode` parameter in `ModelSummary` in favor of `max_depth` ([#8062](https://github.com/PyTorchLightning/pytorch-lightning/pull/8062)) + + ### Removed - Removed `ProfilerConnector` ([#7654](https://github.com/PyTorchLightning/pytorch-lightning/pull/7654)) From e0b1d54df650af3891a068b9fa943dc5216f53fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 29 Jun 2021 12:02:56 +0200 Subject: [PATCH 17/35] update deprecation message --- pytorch_lightning/core/lightning.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index b7a9354d8eca5..dbdfbca5e62f1 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1649,8 +1649,8 @@ def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None if max_depth is None: if mode in ModelSummary.MODES: rank_zero_deprecation( - "The use of `model.summarize(mode)` argument is deprecated in v1.4 " - "and will be removed in v1.5, use `max_depth` argument instead." + "Argument `mode` in `LightningModule.summarize` is deprecated in v1.4" + " and will be removed in v1.5. Use `max_depth` instead." ) max_depth = ModelSummary.MODES[mode] model_summary = ModelSummary(self, max_depth=max_depth) From 6af47fd10a7b3433b2d7f3750fc134f443179692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 29 Jun 2021 12:09:56 +0200 Subject: [PATCH 18/35] update docs and deprecation --- pytorch_lightning/core/memory.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 301f48a32b799..e94450f1b659b 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -132,9 +132,17 @@ class ModelSummary(object): Args: model: The model to summarize (also referred to as the root module). - mode: deprecated. Please use `max_depth`. - max_depth: Maximum depth of modules to show, starting on 0 (for modules connected root), - or -1 to show all modules. + mode: Can be one of + + - `top` (default): only the top-level modules will be recorded (the children of the root module) + - `full`: summarizes all layers and their submodules in the root module + + .. deprecated: v1.4 + This parameter was deprecated in v1.4 in favor of `max_depth` and will be removed in v1.6. + + + max_depth: Maximum depth of modules to show, starting from 0 (for child modules of the given model), + or -1 to show all modules. Defaults to 0. The string representation of this summary prints a table with columns containing the name, type and number of parameters for each layer. @@ -181,7 +189,7 @@ class ModelSummary(object): 0.530 Total estimated model params size (MB) """ - MODES = dict(top=0, full=-1) # temporary mapping from mode to max_depth + MODES = dict(top=0, full=-1) # TODO: remove in v1.6 def __init__(self, model, mode: Optional[str] = "top", max_depth: Optional[int] = None): self._model = model @@ -191,8 +199,8 @@ def __init__(self, model, mode: Optional[str] = "top", max_depth: Optional[int] if mode in ModelSummary.MODES: from pytorch_lightning.utilities import rank_zero_deprecation rank_zero_deprecation( - "The use of `mode` argument is deprecated in v1.4 " - "and will be removed in v1.5, use `max_depth` argument instead." + "Argument `mode` in `ModelSummary` is deprecated in v1.4" + " and will be removed in v1.5. Use `max_depth` instead." ) max_depth = ModelSummary.MODES[mode] else: From d3f49eb7c1a797c1e3418fa80e176abc7df06791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 29 Jun 2021 12:18:05 +0200 Subject: [PATCH 19/35] update test --- tests/core/test_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index c5a459eb5bf8c..a113d6ccad319 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -376,7 +376,7 @@ def test_max_depth_equals_mode_interface(): @pytest.mark.parametrize('max_depth', [-1, 0, 1, 3, 999]) def test_max_depth_param(max_depth): - """Check if only modules with desired depth are shown""" + """Test that only the modules up to the desired depth are shown""" model = DeepNestedModel() summary = ModelSummary(model, max_depth=max_depth) for lname in summary.layer_names: From 21f222f935222d2c990441a3ff25024307a852ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 29 Jun 2021 12:21:53 +0200 Subject: [PATCH 20/35] add deprecation test --- tests/deprecated_api/test_remove_1-4.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/deprecated_api/test_remove_1-4.py b/tests/deprecated_api/test_remove_1-4.py index 23df12586d328..868852eaa6d70 100644 --- a/tests/deprecated_api/test_remove_1-4.py +++ b/tests/deprecated_api/test_remove_1-4.py @@ -16,6 +16,7 @@ import pytest from pytorch_lightning import Trainer +from pytorch_lightning.core.memory import ModelSummary from tests.deprecated_api import _soft_unimport_module from tests.helpers import BoringModel @@ -79,3 +80,12 @@ def test_v1_4_0_deprecated_hpc_load(tmpdir): checkpoint_path = trainer.checkpoint_connector.get_max_ckpt_path_from_folder(str(tmpdir)) with pytest.deprecated_call(match=r"`CheckpointConnector.hpc_load\(\)` was deprecated in v1.4"): trainer.checkpoint_connector.hpc_load(checkpoint_path) + + +def test_v1_4_0_deprecated_model_summary_mode(tmpdir): + model = BoringModel() + with pytest.deprecated_call(match="Argument `mode` in `ModelSummary` is deprecated in v1.4"): + ModelSummary(model, mode="top") + + with pytest.deprecated_call(match="Argument `mode` in `LightningModule.summarize` is deprecated in v1.4"): + model.summarize(mode="top") From f0a07a64dcb39699e067dff967cc886eb8b14621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 29 Jun 2021 12:23:27 +0200 Subject: [PATCH 21/35] rm notebooks --- _notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_notebooks b/_notebooks index 3321b468e7816..29aea106edefc 160000 --- a/_notebooks +++ b/_notebooks @@ -1 +1 @@ -Subproject commit 3321b468e78167aaf056894e92ed6d649c76e89e +Subproject commit 29aea106edefc9d1904c0c17223a8ac2b15c48e7 From 32518a50b7f6bdd09b52a161bcf7b504e4e0471e Mon Sep 17 00:00:00 2001 From: Palermo Date: Tue, 29 Jun 2021 14:50:31 +0100 Subject: [PATCH 22/35] Change ModelSummary max_depth idx logic: -idx now start at 1 -max_depth=1: children modules of root ("mode=top") -max_depth=-1: all modules ("mode=full") -max_depth=0: no modules summary -added exception to invalid values --- pytorch_lightning/core/memory.py | 19 +++++++++++-------- tests/core/test_memory.py | 12 ++++++++++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index e94450f1b659b..8b895c6d7f31a 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -141,8 +141,8 @@ class ModelSummary(object): This parameter was deprecated in v1.4 in favor of `max_depth` and will be removed in v1.6. - max_depth: Maximum depth of modules to show, starting from 0 (for child modules of the given model), - or -1 to show all modules. Defaults to 0. + max_depth: Maximum depth of modules to show. Use -1 to show all modules or 0 to show no + summary. Defaults to 1. The string representation of this summary prints a table with columns containing the name, type and number of parameters for each layer. @@ -167,7 +167,7 @@ class ModelSummary(object): ... return self.net(x) ... >>> model = LitModel() - >>> ModelSummary(model, max_depth=0) # doctest: +NORMALIZE_WHITESPACE + >>> ModelSummary(model, max_depth=1) # doctest: +NORMALIZE_WHITESPACE | Name | Type | Params | In sizes | Out sizes ------------------------------------------------------------ 0 | net | Sequential | 132 K | [10, 256] | [10, 512] @@ -189,7 +189,7 @@ class ModelSummary(object): 0.530 Total estimated model params size (MB) """ - MODES = dict(top=0, full=-1) # TODO: remove in v1.6 + MODES = dict(top=1, full=-1) # TODO: remove in v1.6 def __init__(self, model, mode: Optional[str] = "top", max_depth: Optional[int] = None): self._model = model @@ -217,13 +217,16 @@ def __init__(self, model, mode: Optional[str] = "top", max_depth: Optional[int] @property def named_modules(self) -> List[Tuple[str, nn.Module]]: if self._max_depth == 0: + mods = [] + elif self._max_depth == 1: # the children are the top-level modules mods = self._model.named_children() - elif self._max_depth == -1 or self._max_depth > 0: + elif self._max_depth == -1 or self._max_depth > 1: mods = self._model.named_modules() mods = list(mods)[1:] # do not include root module (LightningModule) else: - mods = [] + raise ValueError(f"Invalid value for max_depth encountered. " + f"Expected -1, 0 or >0, but got {self._max_depth}.") return list(mods) @property @@ -268,9 +271,9 @@ def summarize(self) -> Dict[str, LayerSummary]: for layer in summary.values(): layer.detach_hook() - if self._max_depth >= 0: + if self._max_depth >= 1: # remove summary entries with depth > max_depth - for k in [k for k in summary.keys() if k.count(".") > self._max_depth]: + for k in [k for k in summary.keys() if k.count(".") >= self._max_depth]: del summary[k] return summary diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index a113d6ccad319..99ec676c7814a 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -366,7 +366,7 @@ def test_max_depth_equals_mode_interface(): model = DeepNestedModel() summary_top = model.summarize(mode="top") - summary_0 = model.summarize(max_depth=0) + summary_0 = model.summarize(max_depth=1) assert str(summary_top) == str(summary_0) summary_full = model.summarize(mode="full") @@ -381,4 +381,12 @@ def test_max_depth_param(max_depth): summary = ModelSummary(model, max_depth=max_depth) for lname in summary.layer_names: if max_depth >= 0: - assert lname.count(".") <= max_depth + assert lname.count(".") < max_depth + + +@pytest.mark.parametrize('max_depth', [-99, -2]) +def test_raise_invalid_max_depth_value(max_depth): + with pytest.raises(ValueError, + match=f"Invalid value for max_depth encountered. " + f"Expected -1, 0 or >0, but got {max_depth}."): + model = DeepNestedModel().summarize(max_depth=max_depth) From 5551f54e0a5929ba9fb98f11f0f77ef99305ef93 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 13:50:49 +0000 Subject: [PATCH 23/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/core/memory.py | 6 ++++-- tests/core/test_memory.py | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 8b895c6d7f31a..0ac3150d376a2 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -225,8 +225,10 @@ def named_modules(self) -> List[Tuple[str, nn.Module]]: mods = self._model.named_modules() mods = list(mods)[1:] # do not include root module (LightningModule) else: - raise ValueError(f"Invalid value for max_depth encountered. " - f"Expected -1, 0 or >0, but got {self._max_depth}.") + raise ValueError( + f"Invalid value for max_depth encountered. " + f"Expected -1, 0 or >0, but got {self._max_depth}." + ) return list(mods) @property diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index 99ec676c7814a..75f85d1e8597c 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -386,7 +386,8 @@ def test_max_depth_param(max_depth): @pytest.mark.parametrize('max_depth', [-99, -2]) def test_raise_invalid_max_depth_value(max_depth): - with pytest.raises(ValueError, - match=f"Invalid value for max_depth encountered. " - f"Expected -1, 0 or >0, but got {max_depth}."): + with pytest.raises( + ValueError, match=f"Invalid value for max_depth encountered. " + f"Expected -1, 0 or >0, but got {max_depth}." + ): model = DeepNestedModel().summarize(max_depth=max_depth) From 55188a81b815a0c9e3fff49302cc020d8c7a0dfc Mon Sep 17 00:00:00 2001 From: Palermo Date: Tue, 29 Jun 2021 14:54:07 +0100 Subject: [PATCH 24/35] fix pep8 issue --- tests/core/test_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index 75f85d1e8597c..17c1b660f3743 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -390,4 +390,4 @@ def test_raise_invalid_max_depth_value(max_depth): ValueError, match=f"Invalid value for max_depth encountered. " f"Expected -1, 0 or >0, but got {max_depth}." ): - model = DeepNestedModel().summarize(max_depth=max_depth) + DeepNestedModel().summarize(max_depth=max_depth) From e0d8b7b46bf9e0d9c9f3f7f2e933a385981c6dfd Mon Sep 17 00:00:00 2001 From: Palermo Date: Tue, 29 Jun 2021 15:33:12 +0100 Subject: [PATCH 25/35] improve deprecation warnings + default values --- pytorch_lightning/core/lightning.py | 7 ++++--- pytorch_lightning/core/memory.py | 13 +++++++------ tests/core/test_memory.py | 5 +---- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index dbdfbca5e62f1..73a193510d246 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1648,11 +1648,12 @@ def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None # temporary mapping from mode to max_depth if max_depth is None: if mode in ModelSummary.MODES: + max_depth = ModelSummary.MODES[mode] rank_zero_deprecation( - "Argument `mode` in `LightningModule.summarize` is deprecated in v1.4" - " and will be removed in v1.5. Use `max_depth` instead." + f"Argument `mode` in `LightningModule.summarize` is deprecated in v1.4" + f" and will be removed in v1.5. Use `max_depth` with a value of {max_depth} " + f"to replicate `mode={mode}` behaviour." ) - max_depth = ModelSummary.MODES[mode] model_summary = ModelSummary(self, max_depth=max_depth) log.info("\n" + str(model_summary)) elif mode is not None: diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 0ac3150d376a2..3fa9746df9903 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -191,18 +191,19 @@ class ModelSummary(object): MODES = dict(top=1, full=-1) # TODO: remove in v1.6 - def __init__(self, model, mode: Optional[str] = "top", max_depth: Optional[int] = None): + def __init__(self, model, mode: Optional[str] = None, max_depth: Optional[int] = 1): self._model = model # temporary mapping from mode to max_depth - if max_depth is None: + if max_depth is None or mode is not None: if mode in ModelSummary.MODES: + max_depth = ModelSummary.MODES[mode] from pytorch_lightning.utilities import rank_zero_deprecation rank_zero_deprecation( - "Argument `mode` in `ModelSummary` is deprecated in v1.4" - " and will be removed in v1.5. Use `max_depth` instead." + f"Argument `mode` in `ModelSummary` is deprecated in v1.4 " + f"and will be removed in v1.5. Use `max_depth` with a value " + f"of {max_depth} to replicate `mode={mode}` behaviour." ) - max_depth = ModelSummary.MODES[mode] else: from pytorch_lightning.utilities.exceptions import MisconfigurationException raise MisconfigurationException(f"`mode` can be {', '.join(ModelSummary.MODES)}, got {mode}.") @@ -275,7 +276,7 @@ def summarize(self) -> Dict[str, LayerSummary]: if self._max_depth >= 1: # remove summary entries with depth > max_depth - for k in [k for k in summary.keys() if k.count(".") >= self._max_depth]: + for k in [k for k in summary if k.count(".") >= self._max_depth]: del summary[k] return summary diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index 17c1b660f3743..3adf5efcd172d 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -386,8 +386,5 @@ def test_max_depth_param(max_depth): @pytest.mark.parametrize('max_depth', [-99, -2]) def test_raise_invalid_max_depth_value(max_depth): - with pytest.raises( - ValueError, match=f"Invalid value for max_depth encountered. " - f"Expected -1, 0 or >0, but got {max_depth}." - ): + with pytest.raises(ValueError, match=f"Invalid value for max_depth encountered"): DeepNestedModel().summarize(max_depth=max_depth) From 62a580ce771409f7691741014bda4c19cd5d097c Mon Sep 17 00:00:00 2001 From: Palermo Date: Tue, 29 Jun 2021 15:43:07 +0100 Subject: [PATCH 26/35] pep8 fix --- tests/core/test_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index 3adf5efcd172d..aeff456e04ca4 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -386,5 +386,5 @@ def test_max_depth_param(max_depth): @pytest.mark.parametrize('max_depth', [-99, -2]) def test_raise_invalid_max_depth_value(max_depth): - with pytest.raises(ValueError, match=f"Invalid value for max_depth encountered"): + with pytest.raises(ValueError, match="Invalid value for max_depth encountered"): DeepNestedModel().summarize(max_depth=max_depth) From 6792f3b1bd7b9fceb0a283dc49ffdf2d67ed347f Mon Sep 17 00:00:00 2001 From: Palermo Date: Wed, 30 Jun 2021 23:02:47 +0100 Subject: [PATCH 27/35] added missing model_summary log info --- pytorch_lightning/core/lightning.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 73a193510d246..35f5eb692c9ca 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1660,6 +1660,7 @@ def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None raise MisconfigurationException(f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") else: model_summary = ModelSummary(self, max_depth=max_depth) + log.info("\n" + str(model_summary)) return model_summary From 30f2bb77681c711c2faa97747b96052811203bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 1 Jul 2021 00:39:34 +0200 Subject: [PATCH 28/35] move deprecation test --- tests/deprecated_api/test_remove_1-4.py | 10 ---------- tests/deprecated_api/test_remove_1-6.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/deprecated_api/test_remove_1-4.py b/tests/deprecated_api/test_remove_1-4.py index 868852eaa6d70..23df12586d328 100644 --- a/tests/deprecated_api/test_remove_1-4.py +++ b/tests/deprecated_api/test_remove_1-4.py @@ -16,7 +16,6 @@ import pytest from pytorch_lightning import Trainer -from pytorch_lightning.core.memory import ModelSummary from tests.deprecated_api import _soft_unimport_module from tests.helpers import BoringModel @@ -80,12 +79,3 @@ def test_v1_4_0_deprecated_hpc_load(tmpdir): checkpoint_path = trainer.checkpoint_connector.get_max_ckpt_path_from_folder(str(tmpdir)) with pytest.deprecated_call(match=r"`CheckpointConnector.hpc_load\(\)` was deprecated in v1.4"): trainer.checkpoint_connector.hpc_load(checkpoint_path) - - -def test_v1_4_0_deprecated_model_summary_mode(tmpdir): - model = BoringModel() - with pytest.deprecated_call(match="Argument `mode` in `ModelSummary` is deprecated in v1.4"): - ModelSummary(model, mode="top") - - with pytest.deprecated_call(match="Argument `mode` in `LightningModule.summarize` is deprecated in v1.4"): - model.summarize(mode="top") diff --git a/tests/deprecated_api/test_remove_1-6.py b/tests/deprecated_api/test_remove_1-6.py index 5fe6bb210dab7..2ac4b110d82ae 100644 --- a/tests/deprecated_api/test_remove_1-6.py +++ b/tests/deprecated_api/test_remove_1-6.py @@ -16,6 +16,7 @@ from pytorch_lightning import Trainer from pytorch_lightning.callbacks.early_stopping import EarlyStopping +from pytorch_lightning.core.memory import ModelSummary from pytorch_lightning.plugins.training_type import DDPPlugin, DDPSpawnPlugin from pytorch_lightning.utilities.distributed import rank_zero_deprecation, rank_zero_warn from pytorch_lightning.utilities.model_helpers import is_overridden @@ -249,3 +250,12 @@ def test_v1_6_0_ddp_plugin_task_idx(): plugin = DDPPlugin() with pytest.deprecated_call(match='Use `DDPPlugin.local_rank` instead'): _ = plugin.task_idx + + +def test_v1_6_0_deprecated_model_summary_mode(tmpdir): + model = BoringModel() + with pytest.deprecated_call(match="Argument `mode` in `ModelSummary` is deprecated in v1.4"): + ModelSummary(model, mode="top") + + with pytest.deprecated_call(match="Argument `mode` in `LightningModule.summarize` is deprecated in v1.4"): + model.summarize(mode="top") From 4d638ec6820d0f21bf123d98c289e5439ab293dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 1 Jul 2021 00:46:34 +0200 Subject: [PATCH 29/35] compact deprecation message --- pytorch_lightning/core/lightning.py | 3 +-- pytorch_lightning/core/memory.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 35f5eb692c9ca..d6ef30eae36d1 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1651,8 +1651,7 @@ def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None max_depth = ModelSummary.MODES[mode] rank_zero_deprecation( f"Argument `mode` in `LightningModule.summarize` is deprecated in v1.4" - f" and will be removed in v1.5. Use `max_depth` with a value of {max_depth} " - f"to replicate `mode={mode}` behaviour." + f" and will be removed in v1.6. Use `{max_depth=}` to replicate the `{mode=}` behavior." ) model_summary = ModelSummary(self, max_depth=max_depth) log.info("\n" + str(model_summary)) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 3fa9746df9903..9de913d030977 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -201,8 +201,7 @@ def __init__(self, model, mode: Optional[str] = None, max_depth: Optional[int] = from pytorch_lightning.utilities import rank_zero_deprecation rank_zero_deprecation( f"Argument `mode` in `ModelSummary` is deprecated in v1.4 " - f"and will be removed in v1.5. Use `max_depth` with a value " - f"of {max_depth} to replicate `mode={mode}` behaviour." + f" and will be removed in v1.6. Use `{max_depth=}` to replicate the `{mode=}` behavior." ) else: from pytorch_lightning.utilities.exceptions import MisconfigurationException From 5cf8f0593ed5dd5a5049d6f12e81536c773858b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 1 Jul 2021 00:47:00 +0200 Subject: [PATCH 30/35] move log.info to the end of method --- pytorch_lightning/core/lightning.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index d6ef30eae36d1..62700fb3f0977 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1654,13 +1654,12 @@ def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None f" and will be removed in v1.6. Use `{max_depth=}` to replicate the `{mode=}` behavior." ) model_summary = ModelSummary(self, max_depth=max_depth) - log.info("\n" + str(model_summary)) elif mode is not None: raise MisconfigurationException(f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") else: model_summary = ModelSummary(self, max_depth=max_depth) - log.info("\n" + str(model_summary)) + log.info("\n" + str(model_summary)) return model_summary def freeze(self) -> None: From ad01312f98690bd1604b3114c7ad08a26eea7fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 1 Jul 2021 01:21:29 +0200 Subject: [PATCH 31/35] update changelog --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37a92de6dc5bd..d704e5ae7e9ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,9 +126,6 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added `max_depth` parameter in `ModelSummary` ([#8062](https://github.com/PyTorchLightning/pytorch-lightning/pull/8062)) -- Added `should_raise_exception` parameter to `parse_gpu_ids`, `parse_tpu_cores` and `_sanitize_gpu_ids` utility functions ([#8194](https://github.com/PyTorchLightning/pytorch-lightning/pull/8194)) - - ### Changed From 7dea235d7bf31baa98834ac6ac9cb7149ad015aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 1 Jul 2021 00:46:34 +0200 Subject: [PATCH 32/35] Revert "compact deprecation message" This reverts commit 4d638ec6820d0f21bf123d98c289e5439ab293dd. --- pytorch_lightning/core/lightning.py | 3 ++- pytorch_lightning/core/memory.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 62700fb3f0977..970ae1123e152 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1651,7 +1651,8 @@ def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None max_depth = ModelSummary.MODES[mode] rank_zero_deprecation( f"Argument `mode` in `LightningModule.summarize` is deprecated in v1.4" - f" and will be removed in v1.6. Use `{max_depth=}` to replicate the `{mode=}` behavior." + f" and will be removed in v1.5. Use `max_depth` with a value of {max_depth} " + f"to replicate `mode={mode}` behaviour." ) model_summary = ModelSummary(self, max_depth=max_depth) elif mode is not None: diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 9de913d030977..3fa9746df9903 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -201,7 +201,8 @@ def __init__(self, model, mode: Optional[str] = None, max_depth: Optional[int] = from pytorch_lightning.utilities import rank_zero_deprecation rank_zero_deprecation( f"Argument `mode` in `ModelSummary` is deprecated in v1.4 " - f" and will be removed in v1.6. Use `{max_depth=}` to replicate the `{mode=}` behavior." + f"and will be removed in v1.5. Use `max_depth` with a value " + f"of {max_depth} to replicate `mode={mode}` behaviour." ) else: from pytorch_lightning.utilities.exceptions import MisconfigurationException From 224ef45ff544f20f360b058d7d207d64f39028a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 1 Jul 2021 02:14:05 +0200 Subject: [PATCH 33/35] compact deprecation message --- pytorch_lightning/core/lightning.py | 3 +-- pytorch_lightning/core/memory.py | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 970ae1123e152..b7ce88eace5fd 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1651,8 +1651,7 @@ def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None max_depth = ModelSummary.MODES[mode] rank_zero_deprecation( f"Argument `mode` in `LightningModule.summarize` is deprecated in v1.4" - f" and will be removed in v1.5. Use `max_depth` with a value of {max_depth} " - f"to replicate `mode={mode}` behaviour." + f" and will be removed in v1.6. Use `max_depth={max_depth}` to replicate `mode={mode}` behavior." ) model_summary = ModelSummary(self, max_depth=max_depth) elif mode is not None: diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 3fa9746df9903..c35be84af7c9f 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -200,9 +200,8 @@ def __init__(self, model, mode: Optional[str] = None, max_depth: Optional[int] = max_depth = ModelSummary.MODES[mode] from pytorch_lightning.utilities import rank_zero_deprecation rank_zero_deprecation( - f"Argument `mode` in `ModelSummary` is deprecated in v1.4 " - f"and will be removed in v1.5. Use `max_depth` with a value " - f"of {max_depth} to replicate `mode={mode}` behaviour." + f"Argument `mode` in `ModelSummary` is deprecated in v1.4" + f" and will be removed in v1.6. Use `max_depth={max_depth}` to replicate `mode={mode}` behaviour." ) else: from pytorch_lightning.utilities.exceptions import MisconfigurationException From 14d4d330cad19970fe33558ae395f8570d29cbd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 1 Jul 2021 02:15:49 +0200 Subject: [PATCH 34/35] update deprecated tag --- pytorch_lightning/core/memory.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index c35be84af7c9f..2430978ec06d2 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -137,10 +137,9 @@ class ModelSummary(object): - `top` (default): only the top-level modules will be recorded (the children of the root module) - `full`: summarizes all layers and their submodules in the root module - .. deprecated: v1.4 + .. deprecated:: v1.4 This parameter was deprecated in v1.4 in favor of `max_depth` and will be removed in v1.6. - max_depth: Maximum depth of modules to show. Use -1 to show all modules or 0 to show no summary. Defaults to 1. From a479cfa9aa62f4bada41f01cc9749487c01ddced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 1 Jul 2021 02:26:00 +0200 Subject: [PATCH 35/35] handle max_depth validation at init update test --- pytorch_lightning/core/memory.py | 10 ++++------ tests/core/test_memory.py | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 2430978ec06d2..bba42d6997be3 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -206,6 +206,9 @@ def __init__(self, model, mode: Optional[str] = None, max_depth: Optional[int] = from pytorch_lightning.utilities.exceptions import MisconfigurationException raise MisconfigurationException(f"`mode` can be {', '.join(ModelSummary.MODES)}, got {mode}.") + if not isinstance(max_depth, int) or max_depth < -1: + raise ValueError(f"`max_depth` can be -1, 0 or > 0, got {max_depth}.") + self._max_depth = max_depth self._layer_summary = self.summarize() # 1 byte -> 8 bits @@ -220,14 +223,9 @@ def named_modules(self) -> List[Tuple[str, nn.Module]]: elif self._max_depth == 1: # the children are the top-level modules mods = self._model.named_children() - elif self._max_depth == -1 or self._max_depth > 1: + else: mods = self._model.named_modules() mods = list(mods)[1:] # do not include root module (LightningModule) - else: - raise ValueError( - f"Invalid value for max_depth encountered. " - f"Expected -1, 0 or >0, but got {self._max_depth}." - ) return list(mods) @property diff --git a/tests/core/test_memory.py b/tests/core/test_memory.py index aeff456e04ca4..96e1bfaec14cb 100644 --- a/tests/core/test_memory.py +++ b/tests/core/test_memory.py @@ -384,7 +384,7 @@ def test_max_depth_param(max_depth): assert lname.count(".") < max_depth -@pytest.mark.parametrize('max_depth', [-99, -2]) +@pytest.mark.parametrize('max_depth', [-99, -2, "invalid"]) def test_raise_invalid_max_depth_value(max_depth): - with pytest.raises(ValueError, match="Invalid value for max_depth encountered"): + with pytest.raises(ValueError, match=f"`max_depth` can be -1, 0 or > 0, got {max_depth}"): DeepNestedModel().summarize(max_depth=max_depth)