Do not jinja render packages from dependencies yml (#7910)

* Add some comments to methods constructing Project/RuntimeConfig

* Save flag that packages dict came from dependencies.yml

* Test for not rendering packages_dict

* Changie

* Ensure packages_yml_dict and dependencies_yml_dict are dictionaries

* Ensure "packages" passed to render_packages is a dict

.changes/unreleased/Under the Hood-20230620-180852.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Don't jinja render packages from dependencies.yml
+time: 2023-06-20T18:08:52.848093-04:00
+custom:
+  Author: gshank
+  Issue: "7905"

core/dbt/config/project.py

@@ -101,9 +101,9 @@ def package_and_project_data_from_root(project_root):
     packages_yml_dict = {}
     dependencies_yml_dict = {}
     if path_exists(package_filepath):
-        packages_yml_dict = _load_yaml(package_filepath)
+        packages_yml_dict = _load_yaml(package_filepath) or {}
     if path_exists(dependencies_filepath):
-        dependencies_yml_dict = _load_yaml(dependencies_filepath)
+        dependencies_yml_dict = _load_yaml(dependencies_filepath) or {}
 
     if "packages" in packages_yml_dict and "packages" in dependencies_yml_dict:
         msg = "The 'packages' key cannot be specified in both packages.yml and dependencies.yml"
@@ -116,6 +116,7 @@ def package_and_project_data_from_root(project_root):
     dependent_projects_dict = {}
     if "packages" in dependencies_yml_dict:
         packages_dict["packages"] = dependencies_yml_dict["packages"]
+        packages_dict["packages_from_dependencies"] = True
     else:  # don't check for "packages" here so we capture invalid keys in packages.yml
         packages_dict = packages_yml_dict
     if "projects" in dependencies_yml_dict:
@@ -283,6 +284,7 @@ class RenderComponents:
 
 @dataclass
 class PartialProject(RenderComponents):
+    # This class includes the project_dict, packages_dict, selectors_dict, etc from RenderComponents
     profile_name: Optional[str] = field(
         metadata=dict(description="The unrendered profile name in the project, if set")
     )
@@ -324,7 +326,7 @@ def get_rendered(
             selectors_dict=rendered_selectors,
         )
 
-    # Called by 'collect_parts' in RuntimeConfig
+    # Called by Project.from_project_root (not PartialProject.from_project_root!)
     def render(self, renderer: DbtProjectYamlRenderer) -> "Project":
         try:
             rendered = self.get_rendered(renderer)
@@ -538,6 +540,7 @@ def from_dicts(
         project_name = project_dict.get("name")
         profile_name = project_dict.get("profile")
 
+        # Create a PartialProject
         return cls(
             profile_name=profile_name,
             project_name=project_name,
@@ -555,6 +558,7 @@ def from_project_root(
     ) -> "PartialProject":
         project_root = os.path.normpath(project_root)
         project_dict = load_raw_project(project_root)
+        # Read packages.yml and dependencies.yml and pass dictionaries to "from_dicts" method
         packages_dict, dependent_projects_dict = package_and_project_data_from_root(project_root)
         selectors_dict = selector_data_from_root(project_root)
         return cls.from_dicts(
@@ -715,6 +719,9 @@ def partial_load(cls, project_root: str, *, verify_version: bool = False) -> Par
             verify_version=verify_version,
         )
 
+    # Called by:
+    # RtConfig.load_dependencies => RtConfig.load_projects => RtConfig.new_project => Project.from_project_root
+    # RtConfig.from_args => RtConfig.collect_parts => load_project => Project.from_project_root
     @classmethod
     def from_project_root(
         cls,

core/dbt/config/renderer.py

@@ -134,8 +134,14 @@ def render_project(
 
     def render_packages(self, packages: Dict[str, Any]):
         """Render the given packages dict"""
+        packages = packages or {}  # Sometimes this is none in tests
         package_renderer = self.get_package_renderer()
-        return package_renderer.render_data(packages)
+        if "packages_from_dependencies" in packages:
+            # We don't want to render the "packages" dictionary that came from dependencies.yml
+            packages.pop("packages_from_dependencies")
+            return packages
+        else:
+            return package_renderer.render_data(packages)
 
     def render_dependent_projects(self, dependent_projects: Dict[str, Any]):
         """This is a no-op to maintain regularity in the interfaces. We don't render dependencies.yml."""

core/dbt/config/runtime.py

@@ -38,6 +38,7 @@
 from .renderer import DbtProjectYamlRenderer, ProfileRenderer
 
 
+# Called by RuntimeConfig.collect_parts class method
 def load_project(
     project_root: str,
     version_check: bool,
@@ -237,6 +238,7 @@ def validate(self):
         except ValidationError as e:
             raise ConfigContractBrokenError(e) from e
 
+    # Called by RuntimeConfig.from_args
     @classmethod
     def collect_parts(cls: Type["RuntimeConfig"], args: Any) -> Tuple[Project, Profile]:
         # profile_name from the project
@@ -251,7 +253,7 @@ def collect_parts(cls: Type["RuntimeConfig"], args: Any) -> Tuple[Project, Profi
         project = load_project(project_root, bool(flags.VERSION_CHECK), profile, cli_vars)
         return project, profile
 
-    # Called in main.py, lib.py, task/base.py
+    # Called in task/base.py, in BaseTask.from_args
     @classmethod
     def from_args(cls, args: Any) -> "RuntimeConfig":
         """Given arguments, read in dbt_project.yml from the current directory,

tests/functional/dependencies/test_simple_dependency.py

@@ -127,6 +127,16 @@ def test_dependency_with_dependencies_file(self, run_deps, project):
         assert len(results) == 4
 
 
+class TestSimpleDependencyWithEmptyPackagesFile(SimpleDependencyBase):
+    @pytest.fixture(scope="class")
+    def packages(self):
+        return " "
+
+    def test_dependency_with_empty_packages_file(self, run_deps, project):
+        # Tests that an empty packages file doesn't fail with a Python error
+        run_dbt(["deps"])
+
+
 class TestSimpleDependencyNoProfile(SimpleDependencyBase):
     """dbt deps and clean commands should not require a profile."""
 

tests/unit/test_config.py

@@ -918,6 +918,22 @@ def test_custom_query_comment_append(self):
         self.assertEqual(project.query_comment.comment, "run by user test")
         self.assertEqual(project.query_comment.append, True)
 
+    def test_packages_from_dependencies(self):
+        packages = {
+            "packages_from_dependencies": True,
+            "packages": [
+                {
+                    "git": "{{ env_var('some_package') }}",
+                    "warn-unpinned": True,
+                }
+            ],
+        }
+
+        project = project_from_config_rendered(self.default_project_data, packages)
+        git_package = project.packages.packages[0]
+        # packages did not render because of "packages_from_dependencies" key
+        assert git_package.git == "{{ env_var('some_package') }}"
+
 
 class TestProjectFile(BaseFileTest):
     def setUp(self):