Skip to content

Commit

Permalink
Improve flux-local diff for kustomizations (#48)
Browse files Browse the repository at this point in the history
Improve flux-local diff for kustomizations

Issue #35
  • Loading branch information
allenporter authored Feb 13, 2023
1 parent 2a7b478 commit f4e92f1
Show file tree
Hide file tree
Showing 8 changed files with 232 additions and 53 deletions.
28 changes: 24 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,31 @@ the `--enable-helm` command line flag:
$ flux-local build clusters/prod/ --enable-helm
```

You may also use `flux-local` to verify your local changes to helm charts have the desired
effect on resources within the `HelmRelease`:

You may also use `flux-local` to verify your local changes to cluster resources have the desird
effect. This will run a local `kustomize build` first against the local repo then again
against a prior repo revision, then prints the output:
```bash
$ flux-local diff clusters/prod/ --enable-helm
$ flux-local diff ks apps
---

+++

@@ -2,6 +2,13 @@

kind: Namespace
metadata:
name: podinfo
+- apiVersion: v1
+ data:
+ foo: bar
+ kind: ConfigMap
+ metadata:
+ name: podinfo-config
+ namespace: podinfo
- apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:

```

## Library
Expand Down
97 changes: 86 additions & 11 deletions flux_local/git_repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"""

import contextlib
from dataclasses import dataclass
import logging
import os
import tempfile
Expand All @@ -47,6 +48,7 @@
__all__ = [
"repo_root",
"build_manifest",
"ResourceSelector",
]

_LOGGER = logging.getLogger(__name__)
Expand All @@ -62,7 +64,6 @@ def git_repo(path: Path | None = None) -> git.repo.Repo:
"""Return the local github repo path."""
if path is None:
return git.repo.Repo(os.getcwd(), search_parent_directories=True)
_LOGGER.debug("Creating git repo: %s", path)
return git.repo.Repo(str(path), search_parent_directories=True)


Expand Down Expand Up @@ -90,6 +91,44 @@ def func(doc: dict[str, Any]) -> bool:
KUSTOMIZE_DOMAIN_FILTER = domain_filter(KUSTOMIZE_DOMAIN)


@dataclass
class ResourceSelector:
"""A filter for objects to select from the cluster.
This is invoked when iterating over objects in the cluster to decide which
resources should be inflated and returned, to avoid iterating over
unnecessary resources.
"""

kustomizations: set[str] | None = None
"""Kustomization names to return, or all if empty."""

helm_release_namespace: set[str] | None = None
"""HelmReleases returned will be from this namespace, or all if empty."""

@property
def kustomization_predicate(self) -> Callable[[Kustomization], bool]:
"""A predicate that selects Kustomization objects."""

def predicate(ks: Kustomization) -> bool:
if not self.kustomizations:
return True
return ks.name in self.kustomizations

return predicate

@property
def helm_release_predicate(self) -> Callable[[HelmRelease], bool]:
"""A predicate that selects Kustomization objects."""

def predicate(hr: HelmRelease) -> bool:
if not self.helm_release_namespace:
return True
return hr.namespace in self.helm_release_namespace

return predicate


async def get_clusters(path: Path) -> list[Cluster]:
"""Load Cluster objects from the specified path."""
cmd = kustomize.grep(f"kind={CLUSTER_KUSTOMIZE_KIND}", path).grep(
Expand All @@ -101,17 +140,30 @@ async def get_clusters(path: Path) -> list[Cluster]:
]


async def get_kustomizations(path: Path) -> list[Kustomization]:
async def get_cluster_kustomizations(path: Path) -> list[Kustomization]:
"""Load Kustomization objects from the specified path."""
cmd = kustomize.grep(f"kind={KUSTOMIZE_KIND}", path).grep(
f"metadata.name={CLUSTER_KUSTOMIZE_NAME}",
invert=True,
)
docs = await cmd.objects()
return [Kustomization.from_doc(doc) for doc in docs if KUSTOMIZE_DOMAIN_FILTER(doc)]
return [
Kustomization.from_doc(doc)
for doc in docs
if CLUSTER_KUSTOMIZE_DOMAIN_FILTER(doc)
]


async def build_manifest(path: Path | None = None) -> Manifest:
async def get_kustomizations(path: Path) -> list[dict[str, Any]]:
"""Load Kustomization objects from the specified path."""
cmd = kustomize.grep(f"kind={KUSTOMIZE_KIND}", path)
docs = await cmd.objects()
return list(filter(KUSTOMIZE_DOMAIN_FILTER, docs))


async def build_manifest(
path: Path | None = None, selector: ResourceSelector = ResourceSelector()
) -> Manifest:
"""Build a Manifest object from the local cluster.
This will locate all Kustomizations that represent clusters, then find all
Expand All @@ -120,11 +172,27 @@ async def build_manifest(path: Path | None = None) -> Manifest:
"""
root = repo_root(git_repo(path))

_LOGGER.debug("Processing as a cluster: %s", path or root)
clusters = await get_clusters(path or root)
if len(clusters) > 0:
for cluster in clusters:
_LOGGER.debug("Processing cluster: %s", cluster.path)
cluster.kustomizations = await get_cluster_kustomizations(
root / cluster.path.lstrip("./")
)
elif path:
_LOGGER.debug("No clusters found; Processing as a Kustomization: %s", path)
# The argument path may be a Kustomization inside a cluster. Create a synthetic
# cluster with any found Kustomizations
cluster = Cluster(name="", path="")
objects = await get_kustomizations(path)
if objects:
cluster.kustomizations = [Kustomization(name="", path=str(path))]
clusters.append(cluster)

for cluster in clusters:
_LOGGER.debug("Processing cluster: %s", cluster.path)
cluster.kustomizations = await get_kustomizations(
root / cluster.path.lstrip("./")
cluster.kustomizations = list(
filter(selector.kustomization_predicate, cluster.kustomizations)
)
for kustomization in cluster.kustomizations:
_LOGGER.debug("Processing kustomization: %s", kustomization.path)
Expand All @@ -133,10 +201,17 @@ async def build_manifest(path: Path | None = None) -> Manifest:
HelmRepository.from_doc(doc)
for doc in await cmd.grep(f"kind=^{HELM_REPO_KIND}$").objects()
]
kustomization.helm_releases = [
HelmRelease.from_doc(doc)
for doc in await cmd.grep(f"kind=^{HELM_RELEASE_KIND}$").objects()
]
kustomization.helm_releases = list(
filter(
selector.helm_release_predicate,
[
HelmRelease.from_doc(doc)
for doc in await cmd.grep(
f"kind=^{HELM_RELEASE_KIND}$"
).objects()
],
)
)
return Manifest(clusters=clusters)


Expand Down
4 changes: 2 additions & 2 deletions flux_local/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
# Match a prefix of apiVersion to ensure we have the right type of object.
# We don't check specific versions for forward compatibility on upgrade.
CLUSTER_KUSTOMIZE_DOMAIN = "kustomize.toolkit.fluxcd.io"
KUSTOMIZE_DOMAIN = "kustomize.toolkit.fluxcd.io"
KUSTOMIZE_DOMAIN = "kustomize.config.k8s.io"
HELM_REPO_DOMAIN = "source.toolkit.fluxcd.io"
HELM_RELEASE_DOMAIN = "helm.toolkit.fluxcd.io"

Expand Down Expand Up @@ -182,7 +182,7 @@ class Kustomization(BaseModel):
@classmethod
def from_doc(cls, doc: dict[str, Any]) -> "Kustomization":
"""Parse a partial Kustomization from a kubernetes resource."""
_check_version(doc, KUSTOMIZE_DOMAIN)
_check_version(doc, CLUSTER_KUSTOMIZE_DOMAIN)
if not (metadata := doc.get("metadata")):
raise ValueError(f"Invalid {cls} missing metadata: {doc}")
if not (name := metadata.get("name")):
Expand Down
135 changes: 102 additions & 33 deletions flux_local/tool/diff.py
Original file line number Diff line number Diff line change
@@ -1,71 +1,140 @@
"""Flux-local diff action."""

from argparse import ArgumentParser
from argparse import _SubParsersAction as SubParsersAction
import difflib
import logging
import pathlib
from argparse import ArgumentParser, BooleanOptionalAction
from argparse import _SubParsersAction as SubParsersAction
from typing import cast

import git
from flux_local import git_repo

from . import build

_LOGGER = logging.getLogger(__name__)


class DiffAction:
"""Flux-local diff action."""
def changed_files(repo: git.repo.Repo) -> set[str]:
"""Return the set of changed files in the repo."""
index = repo.index
return set(
{diff.a_path for diff in index.diff("HEAD")}
| {diff.b_path for diff in index.diff("HEAD")}
| {diff.a_path for diff in index.diff(None)}
| {diff.b_path for diff in index.diff(None)}
| {*repo.untracked_files}
)


async def build_kustomization(
name: str, root: pathlib.Path, path: pathlib.Path
) -> list[str] | None:
"""Return the contents of a kustomization object with the specified name."""
selector = git_repo.ResourceSelector(kustomizations={name})
manifest = await git_repo.build_manifest(path, selector)
# XXX: Skip crds is a flag
content_list = []
for cluster in manifest.clusters:
for kustomization in cluster.kustomizations:
_LOGGER.debug(
"Building Kustomization for diff: %s", root / kustomization.path
)
async for content in build.build(
root / kustomization.path,
enable_helm=False,
skip_crds=False,
):
content_list.extend(content.split("\n"))
return content_list


class DiffKustomizationAction:
"""Flux-local diff for Kustomizations."""

@classmethod
def register(
cls, subparsers: SubParsersAction # type: ignore[type-arg]
) -> ArgumentParser:
"""Register the subparser commands."""
args: ArgumentParser = subparsers.add_parser(
"diff",
help="Diff a local flux resource",
args: ArgumentParser = cast(
ArgumentParser,
subparsers.add_parser(
"kustomizations",
aliases=["ks", "kustomization"],
help="Diff Kustomization objects",
description=(
"The diff command does a local kustomize build compared "
"with the repo and prints the diff."
),
),
)
args.add_argument(
"path", type=pathlib.Path, help="Path to the kustomization or charts"
"kustomization",
help="The name of the flux Kustomization",
type=str,
)
args.add_argument(
"--enable-helm",
type=bool,
action=BooleanOptionalAction,
help="Enable use of HelmRelease inflation",
)
args.add_argument(
"--skip-crds",
type=bool,
default=False,
action=BooleanOptionalAction,
help="Allows disabling of outputting CRDs to reduce output size",
"path",
help="Optional path with flux Kustomization resources (multi-cluster ok)",
type=pathlib.Path,
default=".",
nargs="?",
)
args.set_defaults(cls=cls)
return args

async def run( # type: ignore[no-untyped-def]
self,
kustomization: str,
path: pathlib.Path,
enable_helm: bool,
skip_crds: bool,
**kwargs, # pylint: disable=unused-argument
) -> None:
"""Async Action implementation."""
repo = git_repo.git_repo(path)
content = await build_kustomization(
kustomization, git_repo.repo_root(repo), path
)
with git_repo.create_worktree(repo) as worktree:
orig_content = await build_kustomization(kustomization, worktree, path)
if not content and not orig_content:
print(f"Kustomization '{kustomization}' not found in cluster")
return

content1 = []
with git_repo.create_worktree(repo) as worktree_dir:
async for content in build.build(worktree_dir, enable_helm, skip_crds):
content1.append(content)
diff_text = difflib.unified_diff(
orig_content or [],
content or [],
)
for line in diff_text:
print(line)

content2 = []
async for content in build.build(
git_repo.repo_root(repo), enable_helm, skip_crds
):
content2.append(content)

diff_text = difflib.unified_diff(
"".join(content1).split("\n"), "".join(content2).split("\n")
class DiffAction:
"""Flux-local diff action."""

@classmethod
def register(
cls, subparsers: SubParsersAction # type: ignore[type-arg]
) -> ArgumentParser:
"""Register the subparser commands."""
args: ArgumentParser = subparsers.add_parser(
"diff",
help="Diff a local flux resource",
)
print("\n".join(diff_text))
subcmds = args.add_subparsers(
title="Available commands",
required=True,
)
DiffKustomizationAction.register(subcmds)
args.set_defaults(cls=cls)
return args

async def run( # type: ignore[no-untyped-def]
self,
path: pathlib.Path,
enable_helm: bool,
skip_crds: bool,
**kwargs, # pylint: disable=unused-argument
) -> None:
"""Async Action implementation."""
# No-op given subcommands are dispatched
Loading

0 comments on commit f4e92f1

Please sign in to comment.