diff --git a/.changes/unreleased/Under the Hood-20230724-150654.yaml b/.changes/unreleased/Under the Hood-20230724-150654.yaml new file mode 100644 index 00000000000..bb78d3b1f1a --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230724-150654.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: A way to control maxBytes for a single dbt.log file +time: 2023-07-24T15:06:54.263822-07:00 +custom: + Author: ChenyuLInx + Issue: "8199" diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 9bc42164490..773362ebf40 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -132,6 +132,7 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: @p.enable_legacy_logger @p.fail_fast @p.log_cache_events +@p.log_file_max_bytes @p.log_format @p.log_format_file @p.log_level diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 9bff0f11d41..ff0f930b19c 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -171,6 +171,15 @@ default=True, ) +log_file_max_bytes = click.option( + "--log-file-max-bytes", + envvar="DBT_LOG_FILE_MAX_BYTES", + help="Configure the max file size in bytes for a single dbt.log file, before rolling over. 0 means no limit.", + default=10 * 1024 * 1024, # 10mb + type=click.INT, + hidden=True, +) + log_path = click.option( "--log-path", envvar="DBT_LOG_PATH", diff --git a/core/dbt/events/eventmgr.py b/core/dbt/events/eventmgr.py index 2f787ea4096..207816ba322 100644 --- a/core/dbt/events/eventmgr.py +++ b/core/dbt/events/eventmgr.py @@ -80,6 +80,7 @@ class LoggerConfig: use_colors: bool = False output_stream: Optional[TextIO] = None output_file_name: Optional[str] = None + output_file_max_bytes: Optional[int] = 10 * 1024 * 1024 # 10 mb logger: Optional[Any] = None @@ -100,7 +101,7 @@ def __init__(self, event_manager: "EventManager", config: LoggerConfig) -> None: file_handler = RotatingFileHandler( filename=str(config.output_file_name), encoding="utf8", - maxBytes=10 * 1024 * 1024, # 10 mb + maxBytes=config.output_file_max_bytes, # type: ignore backupCount=5, ) self._python_logger = self._get_python_log_for_handler(file_handler) diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index c60f615aa95..3d913a10338 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -64,7 +64,11 @@ def setup_event_logger(flags, callbacks: List[Callable[[EventMsg], None]] = []) log_level_file = EventLevel.DEBUG if flags.DEBUG else EventLevel(flags.LOG_LEVEL_FILE) EVENT_MANAGER.add_logger( _get_logfile_config( - log_file, flags.USE_COLORS_FILE, log_file_format, log_level_file + log_file, + flags.USE_COLORS_FILE, + log_file_format, + log_level_file, + flags.LOG_FILE_MAX_BYTES, ) ) @@ -122,7 +126,11 @@ def _stdout_filter( def _get_logfile_config( - log_path: str, use_colors: bool, line_format: LineFormat, level: EventLevel + log_path: str, + use_colors: bool, + line_format: LineFormat, + level: EventLevel, + log_file_max_bytes: int, ) -> LoggerConfig: return LoggerConfig( name="file_log", @@ -132,6 +140,7 @@ def _get_logfile_config( scrubber=env_scrubber, filter=partial(_logfile_filter, bool(get_flags().LOG_CACHE_EVENTS), line_format), output_file_name=log_path, + output_file_max_bytes=log_file_max_bytes, ) diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index 23243caef58..e5e8c7d2426 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -483,6 +483,7 @@ def project( DEBUG=False, LOG_CACHE_EVENTS=False, QUIET=False, + LOG_FILE_MAX_BYTES=1000000, ) setup_event_logger(log_flags) orig_cwd = os.getcwd() diff --git a/test/unit/test_graph.py b/test/unit/test_graph.py index 5bee5613236..fe1dc7e868a 100644 --- a/test/unit/test_graph.py +++ b/test/unit/test_graph.py @@ -18,6 +18,7 @@ from dbt.contracts.files import SourceFile, FileHash, FilePath from dbt.contracts.graph.manifest import MacroManifest, ManifestStateCheck from dbt.graph import NodeSelector, parse_difference +from dbt.events.functions import setup_event_logger try: from queue import Empty @@ -140,6 +141,7 @@ def get_config(self, extra_cfg=None): config = config_from_parts_or_dicts(project=cfg, profile=self.profile) dbt.flags.set_from_args(Namespace(), config) + setup_event_logger(dbt.flags.get_flags()) object.__setattr__(dbt.flags.get_flags(), "PARTIAL_PARSE", False) return config diff --git a/tests/unit/test_cli_flags.py b/tests/unit/test_cli_flags.py index b5a41099c5a..12ffafe5be5 100644 --- a/tests/unit/test_cli_flags.py +++ b/tests/unit/test_cli_flags.py @@ -54,6 +54,11 @@ def test_log_path_default(self, run_context): assert hasattr(flags, "LOG_PATH") assert getattr(flags, "LOG_PATH") == Path("logs") + def test_log_file_max_size_default(self, run_context): + flags = Flags(run_context) + assert hasattr(flags, "LOG_FILE_MAX_BYTES") + assert getattr(flags, "LOG_FILE_MAX_BYTES") == 10 * 1024 * 1024 + @pytest.mark.parametrize( "set_stats_param,do_not_track,expected_anonymous_usage_stats", [ diff --git a/tests/unit/test_functions.py b/tests/unit/test_functions.py index 20f419218ed..30112d8507a 100644 --- a/tests/unit/test_functions.py +++ b/tests/unit/test_functions.py @@ -2,7 +2,7 @@ import pytest import dbt.flags as flags -from dbt.events.functions import msg_to_dict, warn_or_error +from dbt.events.functions import msg_to_dict, warn_or_error, setup_event_logger from dbt.events.types import InfoLevel, NoNodesForSelectionCriteria from dbt.exceptions import EventCompilationError @@ -59,3 +59,13 @@ def __init__(self): assert ( False ), f"We expect `msg_to_dict` to gracefully handle exceptions, but it raised {exc}" + + +def test_setup_event_logger_specify_max_bytes(mocker): + patched_file_handler = mocker.patch("dbt.events.eventmgr.RotatingFileHandler") + args = Namespace(log_file_max_bytes=1234567) + flags.set_from_args(args, {}) + setup_event_logger(flags.get_flags()) + patched_file_handler.assert_called_once_with( + filename="logs/dbt.log", encoding="utf8", maxBytes=1234567, backupCount=5 + )