[RFC-0003] Add commands for managing OCI artifacts

.github/workflows/e2e.yaml

@@ -4,11 +4,16 @@ on:
   push:
     branches: [ main ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, oci ]
 
 jobs:
   kind:
     runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -168,6 +173,36 @@ jobs:
       - name: flux delete source git
         run: |
           /tmp/flux delete source git podinfo --silent
+      - name: flux oci artifacts
+        run: |
+          /tmp/flux push artifact oci://localhost:5000/fluxcd/flux:${{ github.sha }} \
+            --path="./manifests" \
+            --source="${{ github.repositoryUrl }}" \
+            --revision="${{ github.ref }}/${{ github.sha }}"
+          /tmp/flux tag artifact oci://localhost:5000/fluxcd/flux:${{ github.sha }} \
+            --tag latest
+          /tmp/flux list artifacts oci://localhost:5000/fluxcd/flux
+      - name: flux oci repositories
+        run: |
+          /tmp/flux create source oci podinfo-oci \
+            --url oci://ghcr.io/stefanprodan/manifests/podinfo \
+            --tag-semver 6.1.x \
+            --interval 10m
+          /tmp/flux create kustomization podinfo-oci \
+            --source=OCIRepository/podinfo-oci \
+            --path="./kustomize" \
+            --prune=true \
+            --interval=5m \
+            --target-namespace=default \
+            --wait=true \
+            --health-check-timeout=3m
+          /tmp/flux reconcile source oci podinfo-oci
+          /tmp/flux suspend source oci podinfo-oci
+          /tmp/flux get sources oci
+          /tmp/flux resume source oci podinfo-oci
+          /tmp/flux export source oci podinfo-oci
+          /tmp/flux delete ks podinfo-oci --silent
+          /tmp/flux delete source oci podinfo-oci --silent
       - name: flux create tenant
         run: |
           /tmp/flux create tenant dev-team --with-namespace=apps

Makefile

@@ -1,4 +1,5 @@
 VERSION?=$(shell grep 'VERSION' cmd/flux/main.go | awk '{ print $$4 }' | head -n 1 | tr -d '"')
+DEV_VERSION?=0.0.0-$(shell git rev-parse --abbrev-ref HEAD)-$(shell git rev-parse --short HEAD)-$(shell date +%s)
 EMBEDDED_MANIFESTS_TARGET=cmd/flux/.manifests.done
 TEST_KUBECONFIG?=/tmp/flux-e2e-test-kubeconfig
 # Architecture to use envtest with
@@ -55,6 +56,9 @@ $(EMBEDDED_MANIFESTS_TARGET): $(call rwildcard,manifests/,*.yaml *.json)
 build: $(EMBEDDED_MANIFESTS_TARGET)
 	CGO_ENABLED=0 go build -ldflags="-s -w -X main.VERSION=$(VERSION)" -o ./bin/flux ./cmd/flux
 
+build-dev: $(EMBEDDED_MANIFESTS_TARGET)
+	CGO_ENABLED=0 go build -ldflags="-s -w -X main.VERSION=$(DEV_VERSION)" -o ./bin/flux ./cmd/flux
+
 .PHONY: install
 install:
 	CGO_ENABLED=0 go install ./cmd/flux

action/README.md

@@ -32,7 +32,7 @@ You can download a specific version with:
       - name: Setup Flux CLI
         uses: fluxcd/flux2/action@main
         with:
-          version: 0.8.0
+          version: 0.32.0
 ```
 
 ### Automate Flux updates
@@ -74,6 +74,86 @@ jobs:
               ${{ steps.update.outputs.flux_version }}
 ```
 
+### Push Kubernetes manifests to container registries
+
+Example workflow for publishing Kubernetes manifests bundled as OCI artifacts to GitHub Container Registry:
+
+```yaml
+name: push-artifact-staging
+
+on:
+  push:
+    branches:
+      - 'main'
+
+permissions:
+  packages: write # needed for ghcr.io access
+
+env:
+  OCI_REPO: "oci://ghcr.io/my-org/manifests/${{ github.event.repository.name }}"
+
+jobs:
+  kubernetes:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Flux CLI
+        uses: fluxcd/flux2/action@main
+      - name: Generate manifests
+        run: |
+          kustomize build ./manifests/staging > ./deploy/app.yaml
+      - name: Push manifests
+        run: |
+          flux push artifact $OCI_REPO:$(git rev-parse --short HEAD) \
+            --path="./deploy" \
+            --source="$(git config --get remote.origin.url)" \
+            --revision="$(git branch --show-current)/$(git rev-parse HEAD)"
+      - name: Deploy manifests to staging
+        run: |
+          flux tag artifact $OCI_REPO:$(git rev-parse --short HEAD) --tag staging
+```
+
+Example workflow for publishing Kubernetes manifests bundled as OCI artifacts to Docker Hub:
+
+```yaml
+name: push-artifact-production
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  OCI_REPO: "oci://docker.io/my-org/app-config"
+
+jobs:
+  kubernetes:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Flux CLI
+        uses: fluxcd/flux2/action@main
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Generate manifests
+        run: |
+          kustomize build ./manifests/production > ./deploy/app.yaml
+      - name: Push manifests
+        run: |
+          flux push artifact $OCI_REPO:$(git tag --points-at HEAD) \
+            --path="./deploy" \
+            --source="$(git config --get remote.origin.url)" \
+            --revision="$(git tag --points-at HEAD)/$(git rev-parse HEAD)"
+      - name: Deploy manifests to production
+        run: |
+          flux tag artifact $OCI_REPO:$(git tag --points-at HEAD) --tag production
+```
+
 ### End-to-end testing
 
 Example workflow for running Flux in Kubernetes Kind:

cmd/flux/build_artifact.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	oci "github.com/fluxcd/pkg/oci/client"
+)
+
+var buildArtifactCmd = &cobra.Command{
+	Use:   "artifact",
+	Short: "Build artifact",
+	Long:  `The build artifact command creates an tgz file with the manifests from the given directory.`,
+	Example: `  # Build the given manifests directory into an artifact
+  flux build artifact --path ./path/to/local/manifests --output ./path/to/artifact.tgz
+
+  # List the files bundles in the artifact
+  tar -ztvf ./path/to/artifact.tgz
+`,
+	RunE: buildArtifactCmdRun,
+}
+
+type buildArtifactFlags struct {
+	output string
+	path   string
+}
+
+var buildArtifactArgs buildArtifactFlags
+
+func init() {
+	buildArtifactCmd.Flags().StringVar(&buildArtifactArgs.path, "path", "", "Path to the directory where the Kubernetes manifests are located.")
+	buildArtifactCmd.Flags().StringVarP(&buildArtifactArgs.output, "output", "0", "artifact.tgz", "Path to where the artifact tgz file should be written.")
+	buildCmd.AddCommand(buildArtifactCmd)
+}
+
+func buildArtifactCmdRun(cmd *cobra.Command, args []string) error {
+	if buildArtifactArgs.path == "" {
+		return fmt.Errorf("invalid path %q", buildArtifactArgs.path)
+	}
+
+	if fs, err := os.Stat(buildArtifactArgs.path); err != nil || !fs.IsDir() {
+		return fmt.Errorf("invalid path %q", buildArtifactArgs.path)
+	}
+
+	logger.Actionf("building artifact from %s", buildArtifactArgs.path)
+
+	ociClient := oci.NewLocalClient()
+	if err := ociClient.Build(buildArtifactArgs.output, buildArtifactArgs.path); err != nil {
+		return fmt.Errorf("bulding artifact failed, error: %w", err)
+	}
+
+	logger.Successf("artifact created at %s", buildArtifactArgs.output)
+
+	return nil
+}

cmd/flux/create_kustomization.go

@@ -68,6 +68,13 @@ var createKsCmd = &cobra.Command{
     --prune=true \
     --interval=5m
 
+  # Create a Kustomization resource that references an OCIRepository
+  flux create kustomization podinfo \
+    --source=OCIRepository/podinfo \
+    --target-namespace=default \
+    --prune=true \
+    --interval=5m
+
   # Create a Kustomization resource that references a Bucket
   flux create kustomization secrets \
     --source=Bucket/secrets \

cmd/flux/create_secret_helm_test.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package main
 
 import (

cmd/flux/create_secret_oci.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/fluxcd/flux2/internal/utils"
+	"github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/spf13/cobra"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/yaml"
+)
+
+var createSecretOCICmd = &cobra.Command{
+	Use:   "oci [name]",
+	Short: "Create or update a Kubernetes secret for OCI Registry authentication",
+	Long:  `The create secret oci command generates a Kubernetes secret with `,
+	Example: `  # Create an OCI authentication secret on disk and encrypt it with Mozilla SOPS
+  flux create secret oci podinfo-auth \
+    --url=ghcr.io \
+    --username=username \
+    --password=password \
+    --export > repo-auth.yaml
+
+    sops --encrypt --encrypted-regex '^(data|stringData)$' \
+    --in-place repo-auth.yaml
+	`,
+	RunE: createSecretOCICmdRun,
+}
+
+type secretOCIFlags struct {
+	url      string
+	password string
+	username string
+}
+
+var secretOCIArgs = secretOCIFlags{}
+
+func init() {
+	createSecretOCICmd.Flags().StringVar(&secretOCIArgs.url, "url", "", "oci repository address e.g ghcr.io/stefanprodan/charts")
+	createSecretOCICmd.Flags().StringVarP(&secretOCIArgs.username, "username", "u", "", "basic authentication username")
+	createSecretOCICmd.Flags().StringVarP(&secretOCIArgs.password, "password", "p", "", "basic authentication password")
+
+	createSecretCmd.AddCommand(createSecretOCICmd)
+}
+
+func createSecretOCICmdRun(cmd *cobra.Command, args []string) error {
+	if len(args) < 1 {
+		return fmt.Errorf("name is required")
+	}
+
+	secretName := args[0]
+
+	if secretOCIArgs.url == "" {
+		return fmt.Errorf("--url is required")
+	}
+
+	if secretOCIArgs.username == "" {
+		return fmt.Errorf("--username is required")
+	}
+
+	if secretOCIArgs.password == "" {
+		return fmt.Errorf("--password is required")
+	}
+
+	if _, err := name.ParseReference(secretOCIArgs.url); err != nil {
+		return fmt.Errorf("error parsing url: '%s'", err)
+	}
+
+	opts := sourcesecret.Options{
+		Name:      secretName,
+		Namespace: *kubeconfigArgs.Namespace,
+		Registry:  secretOCIArgs.url,
+		Password:  secretOCIArgs.password,
+		Username:  secretOCIArgs.username,
+	}
+
+	secret, err := sourcesecret.Generate(opts)
+	if err != nil {
+		return err
+	}
+
+	if createArgs.export {
+		rootCmd.Println(secret.Content)
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
+	defer cancel()
+	kubeClient, err := utils.KubeClient(kubeconfigArgs, kubeclientOptions)
+	if err != nil {
+		return err
+	}
+	var s corev1.Secret
+	if err := yaml.Unmarshal([]byte(secret.Content), &s); err != nil {
+		return err
+	}
+	if err := upsertSecret(ctx, kubeClient, s); err != nil {
+		return err
+	}
+
+	logger.Actionf("oci secret '%s' created in '%s' namespace", secretName, *kubeconfigArgs.Namespace)
+	return nil
+}