fix duplicate console logging bug / properly configure logging

CHANGELOG.md

@@ -33,6 +33,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Fixed `val_check_interval` with `fast_dev_run` ([#5540](https://github.com/PyTorchLightning/pytorch-lightning/pull/5540))
 
 
+- Fixed duplicate logs appearing in console when using the python logging module ([#5509](https://github.com/PyTorchLightning/pytorch-lightning/pull/5509))
+
+
 ## [1.1.4] - 2021-01-12
 
 ### Added

docs/source/logging.rst

@@ -259,13 +259,19 @@ Configure console logging
 *************************
 
 Lightning logs useful information about the training process and user warnings to the console.
-You can retrieve the Lightning logger and change it to your liking. For example, increase the logging level
-to see fewer messages like so:
+You can retrieve the Lightning logger and change it to your liking. For example, adjust the logging level
+or redirect output for certain modules to log files:
 
 .. code-block:: python
 
     import logging
-    logging.getLogger("lightning").setLevel(logging.ERROR)
+
+    # configure logging at the root level of lightning
+    logging.getLogger("pytorch_lightning").setLevel(logging.ERROR)
+
+    # configure logging on module level, redirect to file
+    logger = logging.getLogger("pytorch_lightning.core")
+    logger.addHandler(logging.FileHandler("core.log"))
 
 Read more about custom Python logging `here <https://docs.python.org/3/library/logging.html>`_.
 

pl_examples/domain_templates/computer_vision_fine_tuning.py

@@ -40,6 +40,7 @@
 """
 
 import argparse
+import logging
 from collections import OrderedDict
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -58,8 +59,8 @@
 
 import pytorch_lightning as pl
 from pl_examples import cli_lightning_logo
-from pytorch_lightning import _logger as log
 
+log = logging.getLogger(__name__)
 BN_TYPES = (torch.nn.BatchNorm1d, torch.nn.BatchNorm2d, torch.nn.BatchNorm3d)
 DATA_URL = "https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip"
 

pytorch_lightning/__init__.py

@@ -2,6 +2,7 @@
 
 import logging as python_logging
 import os
+import sys
 import time
 
 _this_year = time.strftime("%Y")
@@ -37,11 +38,16 @@
 - https://pytorch-lightning.readthedocs.io/en/latest
 - https://pytorch-lightning.readthedocs.io/en/stable
 """
-
-_logger = python_logging.getLogger("lightning")
-_logger.addHandler(python_logging.StreamHandler())
+_root_logger = python_logging.getLogger()
+_logger = python_logging.getLogger(__name__)
 _logger.setLevel(python_logging.INFO)
 
+# if root logger has handlers, propagate messages up and let root logger process them
+if not _root_logger.hasHandlers():
+    _logger.addHandler(python_logging.StreamHandler())
+    _logger.propagate = False
+
+
 PACKAGE_ROOT = os.path.dirname(__file__)
 PROJECT_ROOT = os.path.dirname(PACKAGE_ROOT)
 
@@ -53,9 +59,7 @@
 except NameError:
     __LIGHTNING_SETUP__: bool = False
 
-if __LIGHTNING_SETUP__:
-    import sys  # pragma: no-cover
-
+if __LIGHTNING_SETUP__:  # pragma: no-cover
     sys.stdout.write(f'Partial import of `{__name__}` during the build process.\n')  # pragma: no-cover
     # We are not importing the rest of the lightning during the build process, as it may not be compiled yet
 else:

pytorch_lightning/accelerators/accelerator_connector.py

@@ -11,12 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
 import os
 
 import torch
 
 from pytorch_lightning.utilities import HOROVOD_AVAILABLE
-from pytorch_lightning import _logger as log
+
 from pytorch_lightning import accelerators
 from pytorch_lightning.accelerators.accelerator import Accelerator
 from pytorch_lightning.cluster_environments.slurm_environment import SLURMEnvironment
@@ -28,6 +29,8 @@
 if HOROVOD_AVAILABLE:
     import horovod.torch as hvd
 
+log = logging.getLogger(__name__)
+
 
 class AcceleratorConnector:
 

pytorch_lightning/accelerators/ddp2_accelerator.py

@@ -11,14 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License
+import logging
 import os
 from typing import Any, List, Optional, Union
 
 import torch
 import torch.distributed as torch_distrib
 from torch.nn.parallel import DistributedDataParallel
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.accelerators.accelerator import Accelerator, ReduceOp
 from pytorch_lightning.cluster_environments import ClusterEnvironment
 from pytorch_lightning.core.lightning import LightningModule
@@ -29,6 +29,7 @@
 from pytorch_lightning.utilities import AMPType, HYDRA_AVAILABLE
 from pytorch_lightning.utilities.distributed import all_gather_ddp_if_available, rank_zero_only, sync_ddp_if_available
 
+log = logging.getLogger(__name__)
 if HYDRA_AVAILABLE:
     from hydra.core.hydra_config import HydraConfig
     from hydra.utils import get_original_cwd, to_absolute_path

pytorch_lightning/accelerators/ddp_accelerator.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License
+import logging
 import os
 from os.path import abspath
 import subprocess
@@ -23,7 +24,6 @@
 import torch.distributed as torch_distrib
 from torch.nn.parallel import DistributedDataParallel
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.accelerators.accelerator import Accelerator, ReduceOp
 from pytorch_lightning.cluster_environments import ClusterEnvironment
 from pytorch_lightning.core.lightning import LightningModule
@@ -40,6 +40,7 @@
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 from pytorch_lightning.utilities.seed import seed_everything
 
+log = logging.getLogger(__name__)
 if HYDRA_AVAILABLE:
     from hydra.core.hydra_config import HydraConfig
     from hydra.utils import get_original_cwd, to_absolute_path

pytorch_lightning/accelerators/ddp_cpu_hpc_accelerator.py

@@ -13,7 +13,6 @@
 # limitations under the License
 from typing import Optional
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.accelerators.ddp_hpc_accelerator import DDPHPCAccelerator
 from pytorch_lightning.cluster_environments import ClusterEnvironment
 from pytorch_lightning.plugins.ddp_plugin import DDPPlugin

pytorch_lightning/accelerators/ddp_cpu_spawn_accelerator.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License
+import logging
 import os
 from typing import Any, List, Optional, Union
 
@@ -19,7 +20,6 @@
 import torch.multiprocessing as mp
 from torch.nn.parallel import DistributedDataParallel
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.accelerators.accelerator import Accelerator, ReduceOp
 from pytorch_lightning.cluster_environments import ClusterEnvironment
 from pytorch_lightning.core.lightning import LightningModule
@@ -35,6 +35,7 @@
     sync_ddp_if_available,
 )
 
+log = logging.getLogger(__name__)
 if HYDRA_AVAILABLE:
     from hydra.core.hydra_config import HydraConfig
     from hydra.utils import get_original_cwd, to_absolute_path

pytorch_lightning/accelerators/ddp_hpc_accelerator.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License
+import logging
 import os
 from typing import Any, List, Optional, Union
 
@@ -19,7 +20,6 @@
 import torch.distributed as torch_distrib
 from torch.nn.parallel import DistributedDataParallel
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.accelerators.accelerator import Accelerator, ReduceOp
 from pytorch_lightning.cluster_environments import ClusterEnvironment
 from pytorch_lightning.core.lightning import LightningModule
@@ -29,6 +29,7 @@
 from pytorch_lightning.utilities import AMPType, HYDRA_AVAILABLE
 from pytorch_lightning.utilities.distributed import all_gather_ddp_if_available, rank_zero_only, sync_ddp_if_available
 
+log = logging.getLogger(__name__)
 if HYDRA_AVAILABLE:
     from hydra.core.hydra_config import HydraConfig
     from hydra.utils import get_original_cwd, to_absolute_path

pytorch_lightning/accelerators/ddp_spawn_accelerator.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License
+import logging
 import os
 import re
 from typing import Any, List, Optional, Union
@@ -20,7 +21,6 @@
 import torch.multiprocessing as mp
 from torch.nn.parallel import DistributedDataParallel
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.accelerators.accelerator import Accelerator, ReduceOp
 from pytorch_lightning.cluster_environments import ClusterEnvironment
 from pytorch_lightning.core.lightning import LightningModule
@@ -39,6 +39,7 @@
 )
 from pytorch_lightning.utilities.seed import seed_everything
 
+log = logging.getLogger(__name__)
 if HYDRA_AVAILABLE:
     from hydra.core.hydra_config import HydraConfig
     from hydra.utils import get_original_cwd, to_absolute_path

pytorch_lightning/accelerators/dp_accelerator.py

@@ -16,7 +16,6 @@
 import torch
 from torch import optim
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.accelerators.accelerator import Accelerator
 from pytorch_lightning.cluster_environments import ClusterEnvironment
 from pytorch_lightning.core.lightning import LightningModule

pytorch_lightning/accelerators/gpu_accelerator.py

@@ -15,7 +15,6 @@
 
 import torch
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.accelerators.accelerator import Accelerator, ReduceOp
 from pytorch_lightning.cluster_environments import ClusterEnvironment
 from pytorch_lightning.distributed.dist import LightningDistributed

pytorch_lightning/accelerators/horovod_accelerator.py

@@ -17,7 +17,6 @@
 import torch
 from torch.optim.lr_scheduler import _LRScheduler
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.accelerators.accelerator import Accelerator, ReduceOp
 from pytorch_lightning.cluster_environments import ClusterEnvironment
 from pytorch_lightning.utilities import AMPType, HOROVOD_AVAILABLE

pytorch_lightning/accelerators/tpu_accelerator.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import io
+import logging
 import os
 import re
 from typing import Any, Callable, Optional, Union
@@ -20,7 +21,6 @@
 import torch.multiprocessing as mp
 from torch.optim import Optimizer
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.accelerators.accelerator import Accelerator, ReduceOp
 from pytorch_lightning.cluster_environments import ClusterEnvironment
 from pytorch_lightning.core import LightningModule
@@ -35,6 +35,7 @@
 from pytorch_lightning.utilities.cloud_io import atomic_save
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 
+log = logging.getLogger(__name__)
 if TPU_AVAILABLE:
     import torch_xla
     import torch_xla.core.xla_model as xm

pytorch_lightning/callbacks/early_stopping.py

@@ -25,7 +25,6 @@
 import numpy as np
 import torch
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.callbacks.base import Callback
 from pytorch_lightning.metrics.metric import Metric
 from pytorch_lightning.utilities import rank_zero_info, rank_zero_warn, TPU_AVAILABLE

pytorch_lightning/callbacks/model_checkpoint.py

@@ -19,7 +19,7 @@
 Automatically save model checkpoints during training.
 
 """
-
+import logging
 import numbers
 import os
 import re
@@ -31,13 +31,14 @@
 import torch
 import yaml
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.callbacks.base import Callback
 from pytorch_lightning.metrics.metric import Metric
 from pytorch_lightning.utilities import rank_zero_info, rank_zero_only, rank_zero_warn
 from pytorch_lightning.utilities.cloud_io import get_filesystem
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 
+log = logging.getLogger(__name__)
+
 
 class ModelCheckpoint(Callback):
     r"""

pytorch_lightning/cluster_environments/slurm_environment.py

@@ -11,12 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import logging
 import os
 import re
-from pytorch_lightning import _logger as log
+
 from pytorch_lightning.cluster_environments.cluster_environment import ClusterEnvironment
 
+log = logging.getLogger(__name__)
+
 
 class SLURMEnvironment(ClusterEnvironment):
 

pytorch_lightning/cluster_environments/torchelastic_environment.py

@@ -11,12 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import logging
 import os
-from pytorch_lightning import _logger as log
+
 from pytorch_lightning.utilities import rank_zero_warn
 from pytorch_lightning.cluster_environments.cluster_environment import ClusterEnvironment
 
+log = logging.getLogger(__name__)
+
 
 class TorchElasticEnvironment(ClusterEnvironment):
 

pytorch_lightning/core/lightning.py

@@ -17,6 +17,7 @@
 import collections
 import copy
 import inspect
+import logging
 import os
 import re
 import tempfile
@@ -30,7 +31,6 @@
 from torch.nn import Module
 from torch.optim.optimizer import Optimizer
 
-from pytorch_lightning import _logger as log
 from pytorch_lightning.core.grads import GradInformation
 from pytorch_lightning.core.hooks import CheckpointHooks, DataHooks, ModelHooks
 from pytorch_lightning.core.memory import ModelSummary
@@ -42,6 +42,7 @@
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 from pytorch_lightning.utilities.parsing import AttributeDict, collect_init_args, get_init_args
 
+log = logging.getLogger(__name__)
 if TPU_AVAILABLE:
     import torch_xla.core.xla_model as xm