Merge pull request #508 from joshrwolf/k8s-runner-bundle-fix

ensure k8s runner bundles are rooted correctly

.github/workflows/e2e.yaml

@@ -72,7 +72,8 @@ jobs:
 
     - name: Build
       run: |
-        ./melange build --signing-key local-melange.rsa --pipeline-dir=pipelines examples/gnu-hello.yaml --arch=x86_64 --empty-workspace --runner kubernetes
+        # Pick an example that requires mounting to flex kontext.Bundle()
+        ./melange build --signing-key local-melange.rsa examples/simple-hello/melange.yaml --source-dir="examples/simple-hello" --workspace-dir="examples/simple-hello" --arch=x86_64 --runner kubernetes
 
   bootstrap:
     name: bootstrap package

pkg/build/build.go

@@ -1889,6 +1889,7 @@ func (ctx *Context) WorkspaceConfig() *container.Config {
 // to the workspace directory. The workspace retrieved from the runner is in a
 // tar stream containing the workspace contents rooted at ./melange-out
 func (ctx *Context) RetrieveWorkspace(sigh context.Context, cfg *container.Config) error {
+	ctx.Logger.Infof("retrieving workspace from builder: %s", cfg.PodID)
 	r, err := ctx.Runner.WorkspaceTar(sigh, ctx.containerConfig)
 	if err != nil {
 		return err

pkg/container/kubernetes_runner.go

@@ -49,8 +49,9 @@ import (
 )
 
 const (
-	KubernetesName           = "kubernetes"
-	KubernetesConfigFileName = ".melange.k8s.yaml"
+	KubernetesName                             = "kubernetes"
+	KubernetesConfigFileName                   = ".melange.k8s.yaml"
+	kubernetesBuilderPodWorkspaceContainerName = "workspace"
 )
 
 // k8s is a Runner implementation that uses kubernetes pods.
@@ -103,6 +104,8 @@ func (k *k8s) StartPod(ctx context.Context, cfg *Config) error {
 
 	pod, err := podclient.Create(ctx, builderPod, metav1.CreateOptions{})
 	if err != nil {
+		data, _ := yaml.Marshal(builderPod)
+		k.logger.Warnf("failed creating builder pod\n%v", string(data))
 		return fmt.Errorf("creating builder pod: %v", err)
 	}
 	k.logger.Infof("created builder pod '%s' with UID '%s'", pod.Name, pod.UID)
@@ -251,9 +254,10 @@ func (k *k8s) Exec(ctx context.Context, podName string, cmd []string, streamOpts
 		Namespace(k.Config.Namespace).
 		SubResource("exec").
 		VersionedParams(&corev1.PodExecOptions{
-			Command: cmd,
-			Stdout:  true,
-			Stderr:  true,
+			Container: kubernetesBuilderPodWorkspaceContainerName,
+			Command:   cmd,
+			Stdout:    true,
+			Stderr:    true,
 		}, scheme.ParameterCodec)
 
 	k.logger.Infof("executing command %v", cmd)
@@ -284,7 +288,7 @@ func (k *k8s) NewBuildPod(ctx context.Context, cfg *Config) (*corev1.Pod, error)
 
 		mountName := fmt.Sprintf("mount-%d", i)
 		k.logger.Infof("creating mount '%s' from %s at %s", mountName, mount.Source, mount.Destination)
-		bundle, err := kontext.Bundle(ctx, mount.Source, repo.Tag(fmt.Sprintf("%s-%s", cfg.PackageName, mountName)))
+		bundle, err := k.bundle(ctx, mount.Source, repo.Tag(fmt.Sprintf("%s-%s", cfg.PackageName, mountName)))
 		if err != nil {
 			return nil, err
 		}
@@ -314,6 +318,26 @@ func (k *k8s) NewBuildPod(ctx context.Context, cfg *Config) (*corev1.Pod, error)
 	return pod, nil
 }
 
+// bundle is a thin wrapper around kontext.Bundle that ensures the bundle is rooted in the given path
+// TODO: This should be upstreamed in kontext.Bundle() when we change that to use go-apk.FullFS
+func (k *k8s) bundle(ctx context.Context, path string, tag name.Tag) (name.Digest, error) {
+	var ref name.Digest
+	cwd, err := os.Getwd()
+	if err != nil {
+		return ref, err
+	}
+	defer func() {
+		if err := os.Chdir(cwd); err != nil {
+			k.logger.Warnf("error changing directory back to %s: %v", cwd, err)
+		}
+	}()
+
+	if err := os.Chdir(path); err != nil {
+		return ref, err
+	}
+	return kontext.Bundle(ctx, ".", tag)
+}
+
 // filterMounts filters mounts that are not supported by the k8s runner
 func (k *k8s) filterMounts(mount BindMount) bool {
 	// the kubelet handles this
@@ -356,11 +380,13 @@ type KubernetesRunnerConfig struct {
 }
 
 type KubernetesRunnerConfigPodTemplate struct {
-	NodeSelector     map[string]string `json:"nodeSelector,omitempty" yaml:"nodeSelector,omitempty"`
-	Env              []corev1.EnvVar   `json:"env,omitempty" yaml:"env,omitempty"`
-	Affinity         *corev1.Affinity  `json:"affinity,omitempty" yaml:"affinity,omitempty"`
-	RuntimeClassName *string           `json:"runtimeClassName,omitempty" yaml:"runtimeClassName,omitempty"`
-	Volumes          []corev1.Volume   `json:"volumes,omitempty" yaml:"volumes,omitempty"`
+	ServiceAccountName string               `json:"serviceAccountName,omitempty" yaml:"serviceAccountName,omitempty"`
+	NodeSelector       map[string]string    `json:"nodeSelector,omitempty" yaml:"nodeSelector,omitempty"`
+	Env                []corev1.EnvVar      `json:"env,omitempty" yaml:"env,omitempty"`
+	Affinity           *corev1.Affinity     `json:"affinity,omitempty" yaml:"affinity,omitempty"`
+	RuntimeClassName   *string              `json:"runtimeClassName,omitempty" yaml:"runtimeClassName,omitempty"`
+	Volumes            []corev1.Volume      `json:"volumes,omitempty" yaml:"volumes,omitempty"`
+	VolumeMounts       []corev1.VolumeMount `json:"volumeMounts,omitempty" yaml:"volumeMounts,omitempty"`
 }
 
 // NewKubernetesConfig returns a default Kubernetes runner config setup
@@ -407,7 +433,7 @@ func (c KubernetesRunnerConfig) defaultBuilderPod(cfg *Config) *corev1.Pod {
 		},
 		Spec: corev1.PodSpec{
 			Containers: []corev1.Container{{
-				Name:  "workspace",
+				Name:  kubernetesBuilderPodWorkspaceContainerName,
 				Image: cfg.ImgRef,
 				// ldconfig is run to prime ld.so.cache for glibc packages which require it.
 				Command: []string{"/bin/sh", "-c", "[ -x /sbin/ldconfig ] && /sbin/ldconfig /lib || true\nwhile true; do sleep 5; done"},
@@ -444,6 +470,11 @@ func (c KubernetesRunnerConfig) defaultBuilderPod(cfg *Config) *corev1.Pod {
 			pod.Spec.Volumes = append(pod.Spec.Volumes, pt.Volumes...)
 		}
 
+		if pt.VolumeMounts != nil {
+			// Only mount to the workspace container
+			pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, pt.VolumeMounts...)
+		}
+
 		for k, v := range pt.NodeSelector {
 			pod.Spec.NodeSelector[k] = v
 		}
@@ -459,6 +490,10 @@ func (c KubernetesRunnerConfig) defaultBuilderPod(cfg *Config) *corev1.Pod {
 		if pt.Env != nil {
 			pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, pt.Env...)
 		}
+
+		if pt.ServiceAccountName != "" {
+			pod.Spec.ServiceAccountName = pt.ServiceAccountName
+		}
 	}
 
 	switch c.Provider {

pkg/container/kubernetes_runner_test.go

@@ -14,6 +14,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes/fake"
 	ktesting "k8s.io/client-go/testing"
+	"knative.dev/pkg/ptr"
 	"sigs.k8s.io/yaml"
 )
 
@@ -94,18 +95,22 @@ func Test_k8s_StartPod(t *testing.T) {
 			},
 		},
 		{
-			name:   "should support custom volume mounts",
+			name:   "should support custom volumes",
 			pkgCfg: &Config{PackageName: "donkey", Arch: types.Architecture("arm64")},
 			k8sCfg: &KubernetesRunnerConfig{
 				PodTemplate: &KubernetesRunnerConfigPodTemplate{
 					Volumes: []corev1.Volume{{
 						Name:         "foo",
 						VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
 					}},
+					VolumeMounts: []corev1.VolumeMount{{
+						Name:      "foo",
+						MountPath: "/foo",
+					}},
 				},
 			},
 			wanter: func(got corev1.Pod) bool {
-				return got.Spec.Volumes[0].Name == "foo"
+				return got.Spec.Volumes[0].Name == "foo" && got.Spec.Containers[0].VolumeMounts[0].Name == "foo"
 			},
 		},
 		{
@@ -133,6 +138,30 @@ func Test_k8s_StartPod(t *testing.T) {
 				return got.Spec.NodeSelector["cloud.google.com/compute-class"] != ""
 			},
 		},
+		{
+			name:   "should support custom service account names",
+			pkgCfg: &Config{PackageName: "donkey", Arch: types.Architecture("arm64")},
+			k8sCfg: &KubernetesRunnerConfig{
+				PodTemplate: &KubernetesRunnerConfigPodTemplate{
+					ServiceAccountName: "foo",
+				},
+			},
+			wanter: func(got corev1.Pod) bool {
+				return got.Spec.ServiceAccountName == "foo"
+			},
+		},
+		{
+			name:   "should support custom runtime classes",
+			pkgCfg: &Config{PackageName: "donkey", Arch: types.Architecture("arm64")},
+			k8sCfg: &KubernetesRunnerConfig{
+				PodTemplate: &KubernetesRunnerConfigPodTemplate{
+					RuntimeClassName: ptr.String("foo"),
+				},
+			},
+			wanter: func(got corev1.Pod) bool {
+				return *got.Spec.RuntimeClassName == "foo"
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {