Add --target-path as a CLI option. (#5402)

.changes/unreleased/Features-20220622-192329.yaml

@@ -0,0 +1,8 @@
+kind: Features
+body: 'Allow customizing `target-path` and `log-path` through
+  environment variables and CLI flags.'
+time: 2022-06-22T19:23:29.748577478+03:00
+custom:
+  Author: isidentical
+  Issue: "5399"
+  PR: "5402"

core/dbt/config/project.py

@@ -15,7 +15,7 @@
 import hashlib
 import os
 
-from dbt import deprecations
+from dbt import flags, deprecations
 from dbt.clients.system import resolve_path_from_base
 from dbt.clients.system import path_exists
 from dbt.clients.system import load_file_contents
@@ -142,6 +142,13 @@ def _all_source_paths(
 T = TypeVar("T")
 
 
+def flag_or(flag: Optional[T], value: Optional[T], default: T) -> T:
+    if flag is None:
+        return value_or(value, default)
+    else:
+        return flag
+
+
 def value_or(value: Optional[T], default: T) -> T:
     if value is None:
         return default
@@ -356,9 +363,9 @@ def create_project(self, rendered: RenderComponents) -> "Project":
 
         docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths)
         asset_paths: List[str] = value_or(cfg.asset_paths, [])
-        target_path: str = value_or(cfg.target_path, "target")
+        target_path: str = flag_or(flags.TARGET_PATH, cfg.target_path, "target")
         clean_targets: List[str] = value_or(cfg.clean_targets, [target_path])
-        log_path: str = value_or(cfg.log_path, "logs")
+        log_path: str = flag_or(flags.LOG_PATH, cfg.log_path, "logs")
         packages_install_path: str = value_or(cfg.packages_install_path, "dbt_packages")
         # in the default case we'll populate this once we know the adapter type
         # It would be nice to just pass along a Quoting here, but that would

core/dbt/flags.py

@@ -37,13 +37,17 @@
 QUIET = None
 NO_PRINT = None
 CACHE_SELECTED_ONLY = None
+TARGET_PATH = None
+LOG_PATH = None
 
 _NON_BOOLEAN_FLAGS = [
     "LOG_FORMAT",
     "PRINTER_WIDTH",
     "PROFILES_DIR",
     "INDIRECT_SELECTION",
     "EVENT_BUFFER_SIZE",
+    "TARGET_PATH",
+    "LOG_PATH",
 ]
 
 _NON_DBT_ENV_FLAGS = ["DO_NOT_TRACK"]
@@ -71,6 +75,8 @@
     "QUIET": False,
     "NO_PRINT": False,
     "CACHE_SELECTED_ONLY": False,
+    "TARGET_PATH": None,
+    "LOG_PATH": None,
 }
 
 
@@ -121,6 +127,7 @@ def set_from_args(args, user_config):
     global WRITE_JSON, PARTIAL_PARSE, USE_COLORS, STORE_FAILURES, PROFILES_DIR, DEBUG, LOG_FORMAT
     global INDIRECT_SELECTION, VERSION_CHECK, FAIL_FAST, SEND_ANONYMOUS_USAGE_STATS
     global PRINTER_WIDTH, WHICH, LOG_CACHE_EVENTS, EVENT_BUFFER_SIZE, QUIET, NO_PRINT, CACHE_SELECTED_ONLY
+    global TARGET_PATH, LOG_PATH
 
     STRICT_MODE = False  # backwards compatibility
     # cli args without user_config or env var option
@@ -148,6 +155,8 @@ def set_from_args(args, user_config):
     QUIET = get_flag_value("QUIET", args, user_config)
     NO_PRINT = get_flag_value("NO_PRINT", args, user_config)
     CACHE_SELECTED_ONLY = get_flag_value("CACHE_SELECTED_ONLY", args, user_config)
+    TARGET_PATH = get_flag_value("TARGET_PATH", args, user_config)
+    LOG_PATH = get_flag_value("LOG_PATH", args, user_config)
 
     _set_overrides_from_env()
 

core/dbt/main.py

@@ -652,6 +652,22 @@ def _add_common_arguments(*subparsers):
             settings in profiles.yml.
             """,
         )
+        sub.add_argument(
+            "--target-path",
+            required=False,
+            help="""
+            Configure the 'target-path'. Only applies this setting for the
+            current run. Overrides the 'DBT_TARGET_PATH' if it is set.
+            """,
+        )
+        sub.add_argument(
+            "--log-path",
+            required=False,
+            help="""
+            Configure the 'log-path'. Only applies this setting for the
+            current run. Overrides the 'DBT_LOG_PATH' if it is set.
+            """,
+        )
         _add_version_check(sub)
 
 

test/integration/075_custom_target_path/models/do_nothing_1.sql

@@ -0,0 +1 @@
+select 1 as x

test/integration/075_custom_target_path/models/do_nothing_2.sql

@@ -0,0 +1 @@
+select 1 as y

test/integration/075_custom_target_path/models/do_nothing_3.sql

@@ -0,0 +1 @@
+select 1 as z

test/integration/075_custom_target_path/test_target_path.py

@@ -0,0 +1,44 @@
+import os
+from unittest import mock
+from test.integration.base import DBTIntegrationTest, use_profile
+
+
+class TestTargetPathFromProjectConfig(DBTIntegrationTest):
+    @property
+    def project_config(self):
+        return {"config-version": 2, "target-path": "project_target"}
+
+    @property
+    def schema(self):
+        return "target_path_tests_075"
+
+    @property
+    def models(self):
+        return "models"
+
+    @use_profile("postgres")
+    def test_postgres_overriden_target_path(self):
+        results = self.run_dbt(args=["run"])
+        self.assertFalse(os.path.exists("./target"))
+        self.assertTrue(os.path.exists("./project_target"))
+
+
+class TestTargetPathOverridenEnv(TestTargetPathFromProjectConfig):
+    @use_profile("postgres")
+    def test_postgres_overriden_target_path(self):
+        with mock.patch.dict(os.environ, {"DBT_TARGET_PATH": "env_target"}):
+            results = self.run_dbt(args=["run"])
+        self.assertFalse(os.path.exists("./target"))
+        self.assertFalse(os.path.exists("./project_target"))
+        self.assertTrue(os.path.exists("./env_target"))
+
+
+class TestTargetPathOverridenEnvironment(TestTargetPathFromProjectConfig):
+    @use_profile("postgres")
+    def test_postgres_overriden_target_path(self):
+        with mock.patch.dict(os.environ, {"DBT_TARGET_PATH": "env_target"}):
+            results = self.run_dbt(args=["run", "--target-path", "cli_target"])
+        self.assertFalse(os.path.exists("./target"))
+        self.assertFalse(os.path.exists("./project_target"))
+        self.assertFalse(os.path.exists("./env_target"))
+        self.assertTrue(os.path.exists("./cli_target"))

test/unit/test_flags.py

@@ -250,3 +250,16 @@ def test__flags(self):
         os.environ.pop('DBT_CACHE_SELECTED_ONLY')
         delattr(self.args, 'cache_selected_only')
         self.user_config.cache_selected_only = False
+
+        # target_path/log_path
+        flags.set_from_args(self.args, self.user_config)
+        self.assertIsNone(flags.LOG_PATH)
+        os.environ['DBT_LOG_PATH'] = 'a/b/c'
+        flags.set_from_args(self.args, self.user_config)
+        self.assertEqual(flags.LOG_PATH, 'a/b/c')
+        setattr(self.args, 'log_path', 'd/e/f')
+        flags.set_from_args(self.args, self.user_config)
+        self.assertEqual(flags.LOG_PATH, 'd/e/f')
+        # cleanup
+        os.environ.pop('DBT_LOG_PATH')
+        delattr(self.args, 'log_path')