feat: add images manifest command #133
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| package images | ||
|
|
||
| type Cmd struct { | ||
| Manifest ManifestCmd `cmd:"" help:"Display a manifest of images used by Airbyte and abctl."` | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| package images | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "slices" | ||
| "strings" | ||
|
|
||
| "github.com/airbytehq/abctl/internal/cmd/local/helm" | ||
| "github.com/airbytehq/abctl/internal/cmd/local/k8s" | ||
| helmlib "github.com/mittwald/go-helm-client" | ||
| "helm.sh/helm/v3/pkg/repo" | ||
|
|
||
| "github.com/airbytehq/abctl/internal/common" | ||
| appsv1 "k8s.io/api/apps/v1" | ||
| batchv1 "k8s.io/api/batch/v1" | ||
| corev1 "k8s.io/api/core/v1" | ||
|
|
||
| "k8s.io/apimachinery/pkg/runtime" | ||
| "k8s.io/kubectl/pkg/scheme" | ||
| ) | ||
|
|
||
| type ManifestCmd struct { | ||
| Chart string `help:"Path to chart." xor:"chartver"` | ||
| ChartVersion string `help:"Version to install." xor:"chartver"` | ||
|
There was a problem hiding this comment. The |
||
| Values string `type:"existingfile" help:"An Airbyte helm chart values file to configure helm."` | ||
| } | ||
|
|
||
| func (c *ManifestCmd) Run(provider k8s.Provider) error { | ||
|
|
||
| client, err := helm.New(provider.Kubeconfig, provider.Context, common.AirbyteNamespace) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| images, err := c.findAirbyteImages(client) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for _, img := range images { | ||
| fmt.Println(img) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (c *ManifestCmd) findAirbyteImages(client helm.Client) ([]string, error) { | ||
| valuesYaml, err := helm.BuildAirbyteValues(helm.ValuesOpts{ | ||
| ValuesFile: c.Values, | ||
| }) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| airbyteChartLoc := helm.LocateLatestAirbyteChart(c.ChartVersion, c.Chart) | ||
| return findImagesFromChart(client, valuesYaml, airbyteChartLoc, c.ChartVersion) | ||
| } | ||
|
|
||
| func findImagesFromChart(client helm.Client, valuesYaml, chartName, chartVersion string) ([]string, error) { | ||
| err := client.AddOrUpdateChartRepo(repo.Entry{ | ||
| Name: common.AirbyteRepoName, | ||
| URL: common.AirbyteRepoURL, | ||
| }) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| bytes, err := client.TemplateChart(&helmlib.ChartSpec{ | ||
| ChartName: chartName, | ||
| GenerateName: true, | ||
| ValuesYaml: valuesYaml, | ||
| Version: chartVersion, | ||
| }, nil) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| images := findAllImages(string(bytes)) | ||
| return images, nil | ||
| } | ||
|
|
||
| // findAllImages walks through the Helm chart, looking for container images in k8s PodSpecs. | ||
| // It also looks for env vars in the airbyte-env config map that end with "_IMAGE". | ||
| // It returns a unique, sorted list of images found. | ||
| func findAllImages(chartYaml string) []string { | ||
| objs := decodeK8sResources(chartYaml) | ||
| imageSet := map[string]bool{} | ||
|
There was a problem hiding this comment. Could use There was a problem hiding this comment. I prefer bool because it's shorter There was a problem hiding this comment. I get the shorter argument. In this specific case I'm ok with the There was a problem hiding this comment. Good point. I do wish Go would add a container/set package. I wrote a small one in the code to replace the |
||
|
|
||
| for _, obj := range objs { | ||
|
|
||
| var podSpec *corev1.PodSpec | ||
| switch z := obj.(type) { | ||
| case *corev1.ConfigMap: | ||
| if strings.HasSuffix(z.Name, "airbyte-env") { | ||
| for k, v := range z.Data { | ||
| if strings.HasSuffix(k, "_IMAGE") { | ||
| imageSet[v] = true | ||
| } | ||
| } | ||
| } | ||
| continue | ||
| case *corev1.Pod: | ||
| podSpec = &z.Spec | ||
| case *batchv1.Job: | ||
| podSpec = &z.Spec.Template.Spec | ||
| case *appsv1.Deployment: | ||
| podSpec = &z.Spec.Template.Spec | ||
| case *appsv1.StatefulSet: | ||
| podSpec = &z.Spec.Template.Spec | ||
| default: | ||
| continue | ||
| } | ||
|
|
||
| for _, c := range podSpec.InitContainers { | ||
| imageSet[c.Image] = true | ||
| } | ||
| for _, c := range podSpec.Containers { | ||
| imageSet[c.Image] = true | ||
| } | ||
| } | ||
|
|
||
| var out []string | ||
| for k := range imageSet { | ||
| if k != "" { | ||
| out = append(out, k) | ||
| } | ||
| } | ||
| slices.Sort(out) | ||
|
|
||
| return out | ||
| } | ||
|
|
||
| func decodeK8sResources(renderedYaml string) []runtime.Object { | ||
| out := []runtime.Object{} | ||
| chunks := strings.Split(renderedYaml, "---") | ||
| for _, chunk := range chunks { | ||
| if len(chunk) == 0 { | ||
| continue | ||
| } | ||
| obj, _, err := scheme.Codecs.UniversalDeserializer().Decode([]byte(chunk), nil, nil) | ||
| if err != nil { | ||
| continue | ||
| } | ||
| out = append(out, obj) | ||
| } | ||
| return out | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| package images | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| helmlib "github.com/mittwald/go-helm-client" | ||
|
|
||
| "github.com/airbytehq/abctl/internal/cmd/local/helm" | ||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func getHelmTestClient(t *testing.T) helm.Client { | ||
| client, err := helmlib.New(nil) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| return client | ||
| } | ||
|
|
||
| func TestManifestCmd(t *testing.T) { | ||
| client := getHelmTestClient(t) | ||
| cmd := ManifestCmd{ | ||
| ChartVersion: "1.1.0", | ||
| } | ||
| actual, err := cmd.findAirbyteImages(client) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| expect := []string{ | ||
| "airbyte/bootloader:1.1.0", | ||
| "airbyte/connector-builder-server:1.1.0", | ||
| "airbyte/cron:1.1.0", | ||
| "airbyte/db:1.1.0", | ||
| "airbyte/mc", | ||
| "airbyte/server:1.1.0", | ||
| "airbyte/webapp:1.1.0", | ||
| "airbyte/worker:1.1.0", | ||
| "airbyte/workload-api-server:1.1.0", | ||
| "airbyte/workload-launcher:1.1.0", | ||
| "bitnami/kubectl:1.28.9", | ||
| "busybox", | ||
| "minio/minio:RELEASE.2023-11-20T22-40-07Z", | ||
| "temporalio/auto-setup:1.23.0", | ||
| } | ||
| assert.Equal(t, expect, actual) | ||
| } | ||
|
|
||
| func TestManifestCmd_Enterprise(t *testing.T) { | ||
| client := getHelmTestClient(t) | ||
| cmd := ManifestCmd{ | ||
| ChartVersion: "1.1.0", | ||
| Values: "testdata/enterprise.values.yaml", | ||
| } | ||
| actual, err := cmd.findAirbyteImages(client) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| expect := []string{ | ||
| "airbyte/bootloader:1.1.0", | ||
| "airbyte/connector-builder-server:1.1.0", | ||
| "airbyte/cron:1.1.0", | ||
| "airbyte/db:1.1.0", | ||
| "airbyte/keycloak-setup:1.1.0", | ||
| "airbyte/keycloak:1.1.0", | ||
| "airbyte/mc", | ||
| "airbyte/server:1.1.0", | ||
| "airbyte/webapp:1.1.0", | ||
| "airbyte/worker:1.1.0", | ||
| "airbyte/workload-api-server:1.1.0", | ||
| "airbyte/workload-launcher:1.1.0", | ||
| "bitnami/kubectl:1.28.9", | ||
| "busybox", | ||
| "curlimages/curl:8.1.1", | ||
| "minio/minio:RELEASE.2023-11-20T22-40-07Z", | ||
| "postgres:13-alpine", | ||
| "temporalio/auto-setup:1.23.0", | ||
| } | ||
| assert.Equal(t, expect, actual) | ||
| } | ||
|
|
||
| func TestManifestCmd_Nightly(t *testing.T) { | ||
| client := getHelmTestClient(t) | ||
| cmd := ManifestCmd{ | ||
| // This version includes chart fixes that expose images more consistently and completely. | ||
| ChartVersion: "1.1.0-nightly-1728428783-9025e1a46e", | ||
| Values: "testdata/enterprise.values.yaml", | ||
| } | ||
| actual, err := cmd.findAirbyteImages(client) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| expect := []string{ | ||
| "airbyte/bootloader:nightly-1728428783-9025e1a46e", | ||
| "airbyte/connector-builder-server:nightly-1728428783-9025e1a46e", | ||
| "airbyte/connector-sidecar:nightly-1728428783-9025e1a46e", | ||
| "airbyte/container-orchestrator:nightly-1728428783-9025e1a46e", | ||
| "airbyte/cron:nightly-1728428783-9025e1a46e", | ||
| "airbyte/db:nightly-1728428783-9025e1a46e", | ||
| "airbyte/keycloak-setup:nightly-1728428783-9025e1a46e", | ||
| "airbyte/keycloak:nightly-1728428783-9025e1a46e", | ||
| "airbyte/mc:latest", | ||
| "airbyte/server:nightly-1728428783-9025e1a46e", | ||
| "airbyte/webapp:nightly-1728428783-9025e1a46e", | ||
| "airbyte/worker:nightly-1728428783-9025e1a46e", | ||
| "airbyte/workload-api-server:nightly-1728428783-9025e1a46e", | ||
| "airbyte/workload-init-container:nightly-1728428783-9025e1a46e", | ||
| "airbyte/workload-launcher:nightly-1728428783-9025e1a46e", | ||
| "bitnami/kubectl:1.28.9", | ||
| "busybox:1.35", | ||
| "busybox:latest", | ||
| "curlimages/curl:8.1.1", | ||
| "minio/minio:RELEASE.2023-11-20T22-40-07Z", | ||
| "postgres:13-alpine", | ||
| "temporalio/auto-setup:1.23.0", | ||
| } | ||
| assert.Equal(t, expect, actual) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| global: | ||
| airbyteUrl: "http://localhost:8000" | ||
| edition: "enterprise" | ||
|
|
||
| auth: | ||
| enabled: false | ||
| instanceAdmin: | ||
| firstName: "test" | ||
| lastName: "user" | ||
|
|
||
| keycloak: | ||
| auth: | ||
| adminUsername: airbyteAdmin | ||
| adminPassword: keycloak123 |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we communicate this to the rest of the team.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@colesnodgrass @perangel I used testify/assert to assert a list of values.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I've been avoiding using the testify package as I don't think it's necessary for most situations, and it feels a little overkill to bring it in for one function.
Most of the code in
abctluses thecmppackage and we can easily use that package for verifying the two lists matchThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is it really causing any trouble? It's handy and popular.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
While I'm personally not a huge fan of testify (I also haven't looked at in in years, so maybe it is worth taking another look with older eyes). If everyone else wants to start using it, I'll be fine with that. I care mostly about simplicity and consistency.