feat: add images manifest command (#133)

go.mod

@@ -15,12 +15,12 @@ require (
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/pterm/pterm v0.12.79
 	golang.org/x/mod v0.17.0
-	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.14.2
 	k8s.io/api v0.29.2
 	k8s.io/apimachinery v0.29.2
 	k8s.io/client-go v0.29.2
+	k8s.io/kubectl v0.29.0
 	sigs.k8s.io/kind v0.24.0
 )
 
@@ -158,13 +158,13 @@ require (
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/evanphx/json-patch.v5 v5.7.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/apiextensions-apiserver v0.29.0 // indirect
 	k8s.io/apiserver v0.29.0 // indirect
 	k8s.io/cli-runtime v0.29.0 // indirect
 	k8s.io/component-base v0.29.0 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240103195357-a9f8850cb432 // indirect
-	k8s.io/kubectl v0.29.0 // indirect
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect
 	oras.land/oras-go v1.2.5 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

internal/cmd/cmd.go

@@ -2,8 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 
+	"github.com/airbytehq/abctl/internal/cmd/images"
 	"github.com/airbytehq/abctl/internal/cmd/local"
 	"github.com/airbytehq/abctl/internal/cmd/local/k8s"
 	"github.com/airbytehq/abctl/internal/cmd/local/localerr"
@@ -24,14 +24,12 @@ func (v verbose) BeforeApply() error {
 
 type Cmd struct {
 	Local   local.Cmd   `cmd:"" help:"Manage the local Airbyte installation."`
+	Images  images.Cmd  `cmd:"" help:"Manage images used by Airbyte and abctl."`
 	Version version.Cmd `cmd:"" help:"Display version information."`
 	Verbose verbose     `short:"v" help:"Enable verbose output."`
 }
 
 func (c *Cmd) BeforeApply(ctx *kong.Context) error {
-	if _, envVarDNT := os.LookupEnv("DO_NOT_TRACK"); envVarDNT {
-		pterm.Info.Println("Telemetry collection disabled (DO_NOT_TRACK)")
-	}
 	ctx.BindTo(k8s.DefaultProvider, (*k8s.Provider)(nil))
 	ctx.BindTo(telemetry.Get(), (*telemetry.Client)(nil))
 	if err := ctx.BindToProvider(bindK8sClient(&k8s.DefaultProvider)); err != nil {

internal/cmd/images/images_cmd.go

@@ -0,0 +1,5 @@
+package images
+
+type Cmd struct {
+	Manifest ManifestCmd `cmd:"" help:"Display a manifest of images used by Airbyte and abctl."`
+}

internal/cmd/images/manifest_cmd.go

@@ -0,0 +1,166 @@
+package images
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/airbytehq/abctl/internal/cmd/local/helm"
+	"github.com/airbytehq/abctl/internal/cmd/local/k8s"
+	helmlib "github.com/mittwald/go-helm-client"
+	"helm.sh/helm/v3/pkg/repo"
+
+	"github.com/airbytehq/abctl/internal/common"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubectl/pkg/scheme"
+)
+
+type ManifestCmd struct {
+	Chart        string `help:"Path to chart." xor:"chartver"`
+	ChartVersion string `help:"Version of the chart." xor:"chartver"`
+	Values       string `type:"existingfile" help:"An Airbyte helm chart values file to configure helm."`
+}
+
+func (c *ManifestCmd) Run(provider k8s.Provider) error {
+
+	client, err := helm.New(provider.Kubeconfig, provider.Context, common.AirbyteNamespace)
+	if err != nil {
+		return err
+	}
+
+	images, err := c.findAirbyteImages(client)
+	if err != nil {
+		return err
+	}
+
+	for _, img := range images {
+		fmt.Println(img)
+	}
+
+	return nil
+}
+
+func (c *ManifestCmd) findAirbyteImages(client helm.Client) ([]string, error) {
+	valuesYaml, err := helm.BuildAirbyteValues(helm.ValuesOpts{
+		ValuesFile: c.Values,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	airbyteChartLoc := helm.LocateLatestAirbyteChart(c.ChartVersion, c.Chart)
+	return findImagesFromChart(client, valuesYaml, airbyteChartLoc, c.ChartVersion)
+}
+
+func findImagesFromChart(client helm.Client, valuesYaml, chartName, chartVersion string) ([]string, error) {
+	err := client.AddOrUpdateChartRepo(repo.Entry{
+		Name: common.AirbyteRepoName,
+		URL:  common.AirbyteRepoURL,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	bytes, err := client.TemplateChart(&helmlib.ChartSpec{
+		ChartName:    chartName,
+		GenerateName: true,
+		ValuesYaml:   valuesYaml,
+		Version:      chartVersion,
+	}, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	images := findAllImages(string(bytes))
+	return images, nil
+}
+
+// findAllImages walks through the Helm chart, looking for container images in k8s PodSpecs.
+// It also looks for env vars in the airbyte-env config map that end with "_IMAGE".
+// It returns a unique, sorted list of images found.
+func findAllImages(chartYaml string) []string {
+	objs := decodeK8sResources(chartYaml)
+	imageSet := set[string]{}
+
+	for _, obj := range objs {
+
+		var podSpec *corev1.PodSpec
+		switch z := obj.(type) {
+		case *corev1.ConfigMap:
+			if strings.HasSuffix(z.Name, "airbyte-env") {
+				for k, v := range z.Data {
+					if strings.HasSuffix(k, "_IMAGE") {
+						imageSet.add(v)
+					}
+				}
+			}
+			continue
+		case *corev1.Pod:
+			podSpec = &z.Spec
+		case *batchv1.Job:
+			podSpec = &z.Spec.Template.Spec
+		case *appsv1.Deployment:
+			podSpec = &z.Spec.Template.Spec
+		case *appsv1.StatefulSet:
+			podSpec = &z.Spec.Template.Spec
+		default:
+			continue
+		}
+
+		for _, c := range podSpec.InitContainers {
+			imageSet.add(c.Image)
+		}
+		for _, c := range podSpec.Containers {
+			imageSet.add(c.Image)
+		}
+	}
+
+	var out []string
+	for _, k := range imageSet.items() {
+		if k != "" {
+			out = append(out, k)
+		}
+	}
+	slices.Sort(out)
+
+	return out
+}
+
+func decodeK8sResources(renderedYaml string) []runtime.Object {
+	out := []runtime.Object{}
+	chunks := strings.Split(renderedYaml, "---")
+	for _, chunk := range chunks {
+		if len(chunk) == 0 {
+			continue
+		}
+		obj, _, err := scheme.Codecs.UniversalDeserializer().Decode([]byte(chunk), nil, nil)
+		if err != nil {
+			continue
+		}
+		out = append(out, obj)
+	}
+	return out
+}
+
+type set[T comparable] struct {
+	vals map[T]struct{}
+}
+
+func (s *set[T]) add(v T) {
+	if s.vals == nil {
+		s.vals = map[T]struct{}{}
+	}
+	s.vals[v] = struct{}{}
+}
+
+func (s *set[T]) items() []T {
+	out := make([]T, len(s.vals))
+	for k := range s.vals {
+		out = append(out, k)
+	}
+	return out
+}

internal/cmd/images/manifest_cmd_test.go

@@ -0,0 +1,127 @@
+package images
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	helmlib "github.com/mittwald/go-helm-client"
+
+	"github.com/airbytehq/abctl/internal/cmd/local/helm"
+)
+
+func getHelmTestClient(t *testing.T) helm.Client {
+	client, err := helmlib.New(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return client
+}
+
+func TestManifestCmd(t *testing.T) {
+	client := getHelmTestClient(t)
+	cmd := ManifestCmd{
+		ChartVersion: "1.1.0",
+	}
+	actual, err := cmd.findAirbyteImages(client)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expect := []string{
+		"airbyte/bootloader:1.1.0",
+		"airbyte/connector-builder-server:1.1.0",
+		"airbyte/cron:1.1.0",
+		"airbyte/db:1.1.0",
+		"airbyte/mc",
+		"airbyte/server:1.1.0",
+		"airbyte/webapp:1.1.0",
+		"airbyte/worker:1.1.0",
+		"airbyte/workload-api-server:1.1.0",
+		"airbyte/workload-launcher:1.1.0",
+		"bitnami/kubectl:1.28.9",
+		"busybox",
+		"minio/minio:RELEASE.2023-11-20T22-40-07Z",
+		"temporalio/auto-setup:1.23.0",
+	}
+	compareList(t, expect, actual)
+}
+
+func TestManifestCmd_Enterprise(t *testing.T) {
+	client := getHelmTestClient(t)
+	cmd := ManifestCmd{
+		ChartVersion: "1.1.0",
+		Values:       "testdata/enterprise.values.yaml",
+	}
+	actual, err := cmd.findAirbyteImages(client)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expect := []string{
+		"airbyte/bootloader:1.1.0",
+		"airbyte/connector-builder-server:1.1.0",
+		"airbyte/cron:1.1.0",
+		"airbyte/db:1.1.0",
+		"airbyte/keycloak-setup:1.1.0",
+		"airbyte/keycloak:1.1.0",
+		"airbyte/mc",
+		"airbyte/server:1.1.0",
+		"airbyte/webapp:1.1.0",
+		"airbyte/worker:1.1.0",
+		"airbyte/workload-api-server:1.1.0",
+		"airbyte/workload-launcher:1.1.0",
+		"bitnami/kubectl:1.28.9",
+		"busybox",
+		"curlimages/curl:8.1.1",
+		"minio/minio:RELEASE.2023-11-20T22-40-07Z",
+		"postgres:13-alpine",
+		"temporalio/auto-setup:1.23.0",
+	}
+	compareList(t, expect, actual)
+}
+
+func TestManifestCmd_Nightly(t *testing.T) {
+	client := getHelmTestClient(t)
+	cmd := ManifestCmd{
+		// This version includes chart fixes that expose images more consistently and completely.
+		ChartVersion: "1.1.0-nightly-1728428783-9025e1a46e",
+		Values:       "testdata/enterprise.values.yaml",
+	}
+	actual, err := cmd.findAirbyteImages(client)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expect := []string{
+		"airbyte/bootloader:nightly-1728428783-9025e1a46e",
+		"airbyte/connector-builder-server:nightly-1728428783-9025e1a46e",
+		"airbyte/connector-sidecar:nightly-1728428783-9025e1a46e",
+		"airbyte/container-orchestrator:nightly-1728428783-9025e1a46e",
+		"airbyte/cron:nightly-1728428783-9025e1a46e",
+		"airbyte/db:nightly-1728428783-9025e1a46e",
+		"airbyte/keycloak-setup:nightly-1728428783-9025e1a46e",
+		"airbyte/keycloak:nightly-1728428783-9025e1a46e",
+		"airbyte/mc:latest",
+		"airbyte/server:nightly-1728428783-9025e1a46e",
+		"airbyte/webapp:nightly-1728428783-9025e1a46e",
+		"airbyte/worker:nightly-1728428783-9025e1a46e",
+		"airbyte/workload-api-server:nightly-1728428783-9025e1a46e",
+		"airbyte/workload-init-container:nightly-1728428783-9025e1a46e",
+		"airbyte/workload-launcher:nightly-1728428783-9025e1a46e",
+		"bitnami/kubectl:1.28.9",
+		"busybox:1.35",
+		"busybox:latest",
+		"curlimages/curl:8.1.1",
+		"minio/minio:RELEASE.2023-11-20T22-40-07Z",
+		"postgres:13-alpine",
+		"temporalio/auto-setup:1.23.0",
+	}
+	compareList(t, expect, actual)
+}
+
+func compareList(t *testing.T, expect, actual []string) {
+	t.Helper()
+	sort.Strings(expect)
+	sort.Strings(actual)
+	if d := cmp.Diff(expect, actual); d != "" {
+		t.Error(d)
+	}
+}

internal/cmd/images/testdata/enterprise.values.yaml

@@ -0,0 +1,14 @@
+global:
+  airbyteUrl: "http://localhost:8000"
+  edition: "enterprise"
+
+  auth:
+    enabled: false
+    instanceAdmin:
+      firstName: "test"
+      lastName:  "user"
+
+keycloak:
+  auth:
+    adminUsername: airbyteAdmin
+    adminPassword: keycloak123