kata: support confidential container's api for pulling image in the g…

…uest

For confidential container, we want to have the ability to pull the image
on the guest side, and not rely on the host's repository.

To achieve this, an extension to the kata shim's options for CreateContainer
was added. Provided the right information through these options,
the kata agent in the remote VM will proceed with the PullImage before
handling the actual CreateContainer.

This commit adds support for this method.

Signed-off-by: Julien Ropé <jrope@redhat.com>

go.mod

@@ -45,6 +45,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/intel/goresctrl v0.5.0
 	github.com/json-iterator/go v1.1.12
+	github.com/kata-containers/kata-containers/src/runtime v0.0.0-20231214145909-c7c763220340
 	github.com/onsi/ginkgo/v2 v2.13.2
 	github.com/onsi/gomega v1.30.0
 	github.com/opencontainers/go-digest v1.0.0

go.sum

@@ -726,6 +726,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
+github.com/kata-containers/kata-containers/src/runtime v0.0.0-20231214145909-c7c763220340 h1:jcPu+nNmDGbXAE5tL7BCt2gNZyYcgnN+aatdEOYtTlY=
+github.com/kata-containers/kata-containers/src/runtime v0.0.0-20231214145909-c7c763220340/go.mod h1:eKVh2ScPyeprBK6BdE75P6EsyWNazV/g3GXge+QZUIE=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=

internal/oci/oci.go

@@ -199,7 +199,7 @@ func (r *Runtime) newRuntimeImpl(c *Container) (RuntimeImpl, error) {
 	}
 
 	if rh.RuntimeType == config.RuntimeTypeVM {
-		return newRuntimeVM(rh.RuntimePath, rh.RuntimeRoot, rh.RuntimeConfigPath, r.config.RuntimeConfig.ContainerExitsDir), nil
+		return newRuntimeVM(rh, r.config.RuntimeConfig.ContainerExitsDir), nil
 	}
 
 	if rh.RuntimeType == config.RuntimeTypePod {

internal/oci/runtime_vm.go

@@ -2,6 +2,8 @@ package oci
 
 import (
 	"bytes"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -16,6 +18,7 @@ import (
 	cgroupsV1 "github.com/containerd/cgroups/stats/v1"
 	cgroupsV2 "github.com/containerd/cgroups/v2/stats"
 	"github.com/containerd/containerd/api/runtime/task/v2"
+	containerdTypes "github.com/containerd/containerd/api/types"
 	tasktypes "github.com/containerd/containerd/api/types/task"
 	ctrio "github.com/containerd/containerd/cio"
 	"github.com/containerd/containerd/namespaces"
@@ -28,11 +31,14 @@ import (
 	"github.com/containerd/ttrpc"
 	"github.com/containerd/typeurl"
 	conmonconfig "github.com/containers/conmon/runner/config"
+	"github.com/containers/podman/v4/pkg/annotations"
 	"github.com/cri-o/cri-o/internal/config/cgmgr"
 	"github.com/cri-o/cri-o/internal/log"
+	"github.com/cri-o/cri-o/pkg/config"
 	"github.com/cri-o/cri-o/server/metrics"
 	"github.com/cri-o/cri-o/utils"
 	"github.com/cri-o/cri-o/utils/errdefs"
+	katavolume "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
@@ -50,6 +56,7 @@ type runtimeVM struct {
 	fifoDir    string
 	configPath string
 	exitsPath  string
+	pullImage  bool
 	ctx        context.Context
 	client     *ttrpc.Client
 	task       task.TaskService
@@ -62,13 +69,17 @@ type containerInfo struct {
 	cio *cio.ContainerIO
 }
 
+const (
+	KataVirtualVolumeOptionName = "io.katacontainers.volume"
+)
+
 const (
 	execError   = -1
 	execTimeout = -2
 )
 
 // newRuntimeVM creates a new runtimeVM instance
-func newRuntimeVM(path, root, configPath, exitsPath string) RuntimeImpl {
+func newRuntimeVM(handler *config.RuntimeHandler, exitsPath string) RuntimeImpl {
 	logrus.Debug("oci.newRuntimeVM() start")
 	defer logrus.Debug("oci.newRuntimeVM() end")
 
@@ -85,15 +96,51 @@ func newRuntimeVM(path, root, configPath, exitsPath string) RuntimeImpl {
 	typeurl.Register(&rspec.WindowsResources{}, prefix, "opencontainers/runtime-spec", major, "WindowsResources")
 
 	return &runtimeVM{
-		path:       path,
-		configPath: configPath,
+		path:       handler.RuntimePath,
+		configPath: handler.RuntimeConfigPath,
 		exitsPath:  exitsPath,
-		fifoDir:    filepath.Join(root, "crio", "fifo"),
+		pullImage:  handler.RuntimePullImage,
+		fifoDir:    filepath.Join(handler.RuntimeRoot, "crio", "fifo"),
 		ctx:        context.Background(),
 		ctrs:       make(map[string]containerInfo),
 	}
 }
 
+func addVolumeMountsToCreateRequest(ctx context.Context, request *task.CreateTaskRequest, c *Container) error {
+	// To make the kata agent pull the image{"volume_type":"image_guest_pull","source":"quay.io/kata-containers/confidential-containers:unsigned","fs_type":"overlayfs","image_pull":{"metadata":{}}}
+
+	imageSource := c.Spec().Annotations[annotations.ImageName]
+	log.Infof(ctx, "Adding mount info to pull image %s", imageSource)
+
+	volume := &katavolume.KataVirtualVolume{
+		VolumeType: katavolume.KataVirtualVolumeImageGuestPullType,
+		Source:     imageSource,
+		FSType:     "overlay_fs",
+		ImagePull:  &katavolume.ImagePullVolume{Metadata: c.Spec().Annotations},
+	}
+	if !volume.IsValid() {
+		return fmt.Errorf("got invalid kata volume, %v", volume)
+	}
+
+	info, err := EncodeKataVirtualVolumeToBase64(ctx, volume)
+	if err != nil {
+		return fmt.Errorf("failed to encode Kata Volume info %v: %w", volume, err)
+	}
+
+	log.Infof(ctx, "CoCo : Mount volume information: %v (encoded: %v)", volume, info)
+
+	opt := fmt.Sprintf("%s=%s", KataVirtualVolumeOptionName, info)
+
+	request.Rootfs = append(request.Rootfs, &containerdTypes.Mount{
+		// we don't actually use nydus, but this string is what makes the kata
+		// shim check for the extended options and ignore the mount, passing the
+		// information to the kata agent instead.
+		Type:    "fuse.nydus-overlayfs",
+		Options: []string{opt},
+	})
+	return nil
+}
+
 // CreateContainer creates a container.
 func (r *runtimeVM) CreateContainer(ctx context.Context, c *Container, cgroupParent string, restore bool) (retErr error) {
 	log.Debugf(ctx, "RuntimeVM.CreateContainer() start")
@@ -154,6 +201,13 @@ func (r *runtimeVM) CreateContainer(ctx context.Context, c *Container, cgroupPar
 		Options:  opts,
 	}
 
+	if r.pullImage {
+		err := addVolumeMountsToCreateRequest(ctx, request, c)
+		if err != nil {
+			log.Warnf(ctx, "Failed to add KataVirtualVolume information to CreateContainer: %v", err)
+		}
+	}
+
 	createdCh := make(chan error)
 	go func() {
 		// Create the container
@@ -1176,3 +1230,13 @@ func (r *runtimeVM) RestoreContainer(ctx context.Context, c *Container, cgroupPa
 
 	return errors.New("restoring not implemented for runtimeVM")
 }
+
+func EncodeKataVirtualVolumeToBase64(ctx context.Context, volume *katavolume.KataVirtualVolume) (string, error) {
+	validKataVirtualVolumeJSON, err := json.Marshal(volume)
+	if err != nil {
+		return "", fmt.Errorf("marshal KataVirtualVolume object; error=%w", err)
+	}
+	log.Infof(ctx, "Encode kata volume %v", validKataVirtualVolumeJSON)
+	option := base64.StdEncoding.EncodeToString(validKataVirtualVolumeJSON)
+	return option, nil
+}

pkg/config/config.go

@@ -224,6 +224,10 @@ type RuntimeHandler struct {
 	// the runtime paths for different platforms.
 	PlatformRuntimePaths map[string]string `toml:"platform_runtime_paths,omitempty"`
 
+	// Marks the runtime as performing image pulling on its own, and doesn't
+	// require crio to do it.
+	RuntimePullImage bool `toml:"runtime_pull_image,omitempty"`
+
 	// Output of the "features" subcommand.
 	// This is populated dynamically and not read from config.
 	features features.Features