cloud-config: add support for CDH config

fixes #1720

This change will add a write_files entry to the cloud-config file that
is produced by CAA. aa-kbc-params are converted into a config file with
kbc name and kbc url.

process-user-data has been made more flexible to also support this
entry.

guest-components in versions.yaml has been updated to a new revision
that requires a cdh config file.

the kata-agent service unit has been extended to have the env
CDH_CONFIG_FILE=/run/confidential-containers/cdh.toml set, which is the
path that we add as a cloud-config directive.

Signed-off-by: Magnus Kulke <magnuskulke@microsoft.com>

cmd/process-user-data/main.go

@@ -8,6 +8,7 @@ import (
 
 	cmdUtil "github.com/confidential-containers/cloud-api-adaptor/cmd"
 	"github.com/confidential-containers/cloud-api-adaptor/pkg/agent"
+	"github.com/confidential-containers/cloud-api-adaptor/pkg/cdh"
 	daemon "github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder"
 	"github.com/confidential-containers/cloud-api-adaptor/pkg/userdata"
 	"github.com/spf13/cobra"
@@ -36,17 +37,18 @@ var rootCmd = &cobra.Command{
 }
 
 func init() {
-	var agentConfigPath, daemonConfigPath string
+	var agentConfigPath, cdhConfigPath, daemonConfigPath string
 	var fetchTimeout int
 
 	rootCmd.PersistentFlags().BoolVarP(&versionFlag, "version", "v", false, "Print the version")
 	rootCmd.PersistentFlags().StringVarP(&daemonConfigPath, "daemon-config-path", "d", daemon.DefaultConfigPath, "Path to a daemon config file")
+	rootCmd.PersistentFlags().StringVarP(&cdhConfigPath, "cdh-config-path", "d", cdh.ConfigFilePath, "Path to a CDH config file")
 
 	var provisionFilesCmd = &cobra.Command{
 		Use:   "provision-files",
 		Short: "Provision required files based on user data",
 		RunE: func(_ *cobra.Command, _ []string) error {
-			cfg := userdata.NewConfig(defaultAuthJsonPath, daemonConfigPath, fetchTimeout)
+			cfg := userdata.NewConfig(defaultAuthJsonPath, daemonConfigPath, cdhConfigPath, fetchTimeout)
 			return userdata.ProvisionFiles(cfg)
 		},
 		SilenceUsage: true, // Silence usage on error

pkg/adaptor/cloud/cloud.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/k8sops"
 	"github.com/confidential-containers/cloud-api-adaptor/pkg/adaptor/proxy"
+	"github.com/confidential-containers/cloud-api-adaptor/pkg/cdh"
 	"github.com/confidential-containers/cloud-api-adaptor/pkg/forwarder"
 	"github.com/confidential-containers/cloud-api-adaptor/pkg/podnetwork"
 	"github.com/confidential-containers/cloud-api-adaptor/pkg/util"
@@ -281,6 +282,17 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 		},
 	}
 
+	if s.aaKBCParams != "" {
+		toml, err := cdh.CreateConfigFile(s.aaKBCParams)
+		if err != nil {
+			return nil, fmt.Errorf("creating CDH config: %w", err)
+		}
+		cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{
+			Path:    cdh.ConfigFilePath,
+			Content: toml,
+		})
+	}
+
 	sandbox := &sandbox{
 		id:           sid,
 		podName:      pod,

pkg/cdh/config.go

@@ -0,0 +1,45 @@
+package cdh
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pelletier/go-toml/v2"
+)
+
+const (
+	ConfigFilePath = "/run/confidential-containers/cdh.toml"
+	Socket         = "unix://run/confidential-containers/cdh.sock"
+)
+
+type Config struct {
+	Socket string    `toml:"socket"`
+	KBC    KBCConfig `toml:"kbc"`
+}
+
+type KBCConfig struct {
+	Name string `toml:"name"`
+	URL  string `toml:"url"`
+}
+
+func parseAAKBCParams(aaKBCParams string) (*Config, error) {
+	parts := strings.SplitN(aaKBCParams, "::", 2)
+	if len(parts) != 2 {
+		return nil, fmt.Errorf("Invalid aa-kbs-params input: %s", aaKBCParams)
+	}
+	name, url := parts[0], parts[1]
+	kbcConfig := KBCConfig{name, url}
+	return &Config{Socket, kbcConfig}, nil
+}
+
+func CreateConfigFile(aaKBCParams string) (string, error) {
+	config, err := parseAAKBCParams(aaKBCParams)
+	if err != nil {
+		return "", err
+	}
+	bytes, err := toml.Marshal(config)
+	if err != nil {
+		return "", err
+	}
+	return string(bytes), nil
+}

pkg/cdh/config_test.go

@@ -0,0 +1,38 @@
+package cdh
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/pelletier/go-toml/v2"
+)
+
+func TestCDHConfigFileFromAAKBCParams(t *testing.T) {
+	refdoc := `
+socket = "%s"
+[kbc]
+name = "cc_kbc"
+url = "http://1.2.3.4:8080"
+`
+	refdoc = fmt.Sprintf(refdoc, Socket)
+	var refcfg Config
+	err := toml.Unmarshal([]byte(refdoc), &refcfg)
+	if err != nil {
+		panic(err)
+	}
+
+	config, err := parseAAKBCParams("cc_kbc::http://1.2.3.4:8080")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if config.KBC.Name != refcfg.KBC.Name {
+		t.Errorf("Expected %s, got %s", refcfg.KBC.Name, config.KBC.Name)
+	}
+	if config.KBC.URL != refcfg.KBC.URL {
+		t.Errorf("Expected %s, got %s", refcfg.KBC.URL, config.KBC.URL)
+	}
+	if config.Socket != refcfg.Socket {
+		t.Errorf("Expected %s, got %s", refcfg.Socket, config.Socket)
+	}
+}

pkg/userdata/provision.go

@@ -17,14 +17,20 @@ import (
 
 var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix)
 
+type paths struct {
+	authJson     string
+	cdhConfig    string
+	daemonConfig string
+}
+
 type Config struct {
-	authJsonPath     string
-	daemonConfigPath string
-	fetchTimeout     int
+	fetchTimeout int
+	paths        paths
 }
 
-func NewConfig(authJsonPath, daemonConfigPath string, fetchTimeout int) *Config {
-	return &Config{authJsonPath, daemonConfigPath, fetchTimeout}
+func NewConfig(authJsonPath, daemonConfigPath, cdhConfig string, fetchTimeout int) *Config {
+	paths := paths{authJsonPath, cdhConfig, daemonConfigPath}
+	return &Config{fetchTimeout, paths}
 }
 
 type WriteFile struct {
@@ -127,19 +133,14 @@ func parseDaemonConfig(content []byte) (*daemon.Config, error) {
 	return &dc, nil
 }
 
-func findDaemonConfigEntry(path string, cc *CloudConfig) (*daemon.Config, []byte, error) {
+func findConfigEntry(path string, cc *CloudConfig) []byte {
 	for _, wf := range cc.WriteFiles {
 		if wf.Path != path {
 			continue
 		}
-		bytes := []byte(wf.Content)
-		daemonConfig, err := parseDaemonConfig(bytes)
-		if err != nil {
-			return nil, nil, err
-		}
-		return daemonConfig, bytes, nil
+		return []byte(wf.Content)
 	}
-	return nil, nil, fmt.Errorf("failed to find entry for %s in cloud config", path)
+	return nil
 }
 
 func writeFile(path string, bytes []byte) error {
@@ -152,18 +153,27 @@ func writeFile(path string, bytes []byte) error {
 }
 
 func processCloudConfig(cfg *Config, cc *CloudConfig) error {
-	daemonConfig, bytes, err := findDaemonConfigEntry(cfg.daemonConfigPath, cc)
+	bytes := findConfigEntry(cfg.paths.daemonConfig, cc)
+	if bytes == nil {
+		return fmt.Errorf("failed to find daemon config entry in cloud config")
+	}
+	daemonConfig, err := parseDaemonConfig(bytes)
 	if err != nil {
-		return fmt.Errorf("failed to process daemon config: %w", err)
+		return fmt.Errorf("failed to parse daemon config: %w", err)
 	}
-
-	if err = writeFile(cfg.daemonConfigPath, bytes); err != nil {
+	if err = writeFile(cfg.paths.daemonConfig, bytes); err != nil {
 		return fmt.Errorf("failed to write daemon config file: %w", err)
 	}
 
+	if bytes := findConfigEntry(cfg.paths.cdhConfig, cc); bytes != nil {
+		if err = writeFile(cfg.paths.cdhConfig, bytes); err != nil {
+			return fmt.Errorf("failed to write cdh config file: %w", err)
+		}
+	}
+
 	if daemonConfig.AuthJson != "" {
 		bytes := []byte(daemonConfig.AuthJson)
-		if err = writeFile(cfg.authJsonPath, bytes); err != nil {
+		if err = writeFile(cfg.paths.authJson, bytes); err != nil {
 			return fmt.Errorf("failed to write auth json file: %w", err)
 		}
 	}

pkg/userdata/provision_test.go

@@ -42,7 +42,15 @@ var testDaemonConfig string = `{
 	"tls-client-ca": "-----BEGIN CERTIFICATE-----\n....\n-----END CERTIFICATE-----\n",
 	"aa-kbc-params": "cc_kbc::http://192.168.100.2:8080",
 	"auth-json": "{\"auths\":{}}"
-}`
+}
+`
+
+var testCDHConfig string = `socket = 'unix://run/confidential-containers/cdh.sock'
+
+[kbc]
+name = 'cc_kbc'
+url = 'http://1.2.3.4:8080'
+`
 
 // Test server to simulate the metadata service
 func startTestServer() *httptest.Server {
@@ -75,7 +83,6 @@ func startTestServer() *httptest.Server {
 	fmt.Printf("Started metadata server at srv.URL: %s\n", srv.URL)
 
 	return srv
-
 }
 
 // test server, serving plain text userData
@@ -207,22 +214,18 @@ write_files:
     test`}
 	_, err = retrieveCloudConfig(context.TODO(), &provider)
 	if err != nil {
-		t.Fatalf("couldn't retrieve and parse valid cloud config: %v", err)
+		t.Fatalf("couldn't retrieve valid cloud config: %v", err)
 	}
 }
 
-// TestProcessCloudConfig fail tests
-func TestFailProcessCloudConfig(t *testing.T) {
-	content := "#cloud-config\nwrite_files:\n- path: /wrong\n  content: bla"
-	provider := TestProvider{content: content}
-	cc, err := retrieveCloudConfig(context.TODO(), &provider)
-	if err != nil {
-		t.Fatalf("couldn't retrieve and parse cloud config: %v", err)
-	}
-	_, _, err = findDaemonConfigEntry("/other", cc)
-	if err == nil {
-		t.Fatalf("it should fail as there is no file w/ $daemonConfigPath")
+func indentTextBlock(text string, by int) string {
+	whiteSpace := strings.Repeat(" ", by)
+	split := strings.Split(text, "\n")
+	indented := ""
+	for _, line := range split {
+		indented += whiteSpace + line + "\n"
 	}
+	return indented
 }
 
 // TestProcessCloudConfig tests parsing and provisioning of a daemon config
@@ -241,43 +244,100 @@ func TestProcessCloudConfig(t *testing.T) {
 	}
 	defer os.Remove(tmpAuthJsonFile.Name())
 
-	// embed daemon config fixture in cloud config
-	indented := strings.ReplaceAll(testDaemonConfig, "\n", "\n    ")
-	content := fmt.Sprintf("#cloud-config\nwrite_files:\n- path: %s\n  content: |\n    %s", tmpDaemonConfigFile.Name(), indented)
+	// create temporary cdh config file
+	tmpCDHConfigFile, err := os.CreateTemp("", "test")
+	if err != nil {
+		t.Fatalf("failed to create temp file: %v", err)
+	}
+	defer os.Remove(tmpCDHConfigFile.Name())
+
+	content := fmt.Sprintf(`#cloud-config
+write_files:
+- path: %s
+  content: |
+%s
+- path: %s
+  content: |
+%s
+`,
+		tmpDaemonConfigFile.Name(),
+		indentTextBlock(testDaemonConfig, 4),
+		tmpCDHConfigFile.Name(),
+		indentTextBlock(testCDHConfig, 4))
+
 	provider := TestProvider{content: content}
 
 	cc, err := retrieveCloudConfig(context.TODO(), &provider)
 	if err != nil {
-		t.Fatalf("couldn't retrieve and parse cloud config: %v", err)
+		t.Fatalf("couldn't retrieve cloud config: %v", err)
 	}
 
 	cfg := Config{
-		daemonConfigPath: tmpDaemonConfigFile.Name(),
-		authJsonPath:     tmpAuthJsonFile.Name(),
+		paths: paths{
+			daemonConfig: tmpDaemonConfigFile.Name(),
+			cdhConfig:    tmpCDHConfigFile.Name(),
+			authJson:     tmpAuthJsonFile.Name(),
+		},
 	}
 	if err := processCloudConfig(&cfg, cc); err != nil {
 		t.Fatalf("failed to process cloud config file: %v", err)
 	}
 
 	// check if files have been written correctly
-	data, err := os.ReadFile(tmpDaemonConfigFile.Name())
-	if err != nil {
-		t.Fatalf("failed to read daemon config file: %v", err)
-	}
+	data, _ := os.ReadFile(tmpDaemonConfigFile.Name())
 	fileContent := string(data)
-
 	if fileContent != testDaemonConfig {
 		t.Fatalf("file content does not match daemon config fixture: got %q", fileContent)
 	}
 
-	data, _ = os.ReadFile(tmpAuthJsonFile.Name())
+	data, _ = os.ReadFile(tmpCDHConfigFile.Name())
 	fileContent = string(data)
+	if fileContent != testCDHConfig {
+		t.Fatalf("file content does not match cdh config fixture: got %q", fileContent)
+	}
 
+	data, _ = os.ReadFile(tmpAuthJsonFile.Name())
+	fileContent = string(data)
 	if fileContent != `{"auths":{}}` {
 		t.Fatalf("file content does not match auth json fixture: got %q", fileContent)
 	}
 }
 
+func TestProcessWithoutCDHConfig(t *testing.T) {
+	tmpDaemonConfigFile, _ := os.CreateTemp("", "test")
+	defer os.Remove(tmpDaemonConfigFile.Name())
+	tmpAuthJsonFile, _ := os.CreateTemp("", "test")
+	defer os.Remove(tmpAuthJsonFile.Name())
+	tmpCDHConfigFile, _ := os.CreateTemp("", "test")
+	defer os.Remove(tmpCDHConfigFile.Name())
+
+	content := fmt.Sprintf(`#cloud-config
+write_files:
+- path: %s
+  content: |
+%s
+`,
+		tmpDaemonConfigFile.Name(),
+		indentTextBlock(testDaemonConfig, 4))
+	provider := TestProvider{content: content}
+
+	cc, err := retrieveCloudConfig(context.TODO(), &provider)
+	if err != nil {
+		t.Fatalf("couldn't retrieve cloud config: %v", err)
+	}
+
+	cfg := Config{
+		paths: paths{
+			daemonConfig: tmpDaemonConfigFile.Name(),
+			cdhConfig:    tmpCDHConfigFile.Name(),
+			authJson:     tmpAuthJsonFile.Name(),
+		},
+	}
+	if err := processCloudConfig(&cfg, cc); err != nil {
+		t.Fatalf("failed to process cloud config file: %v", err)
+	}
+}
+
 // TestFailPlainTextUserData tests with plain text userData
 func TestFailPlainTextUserData(t *testing.T) {
 	// startTestServerPlainText

podvm/files/etc/systemd/system/kata-agent.service

@@ -5,6 +5,7 @@ Wants=process-user-data.service
 After=netns@podns.service process-user-data.service
 
 [Service]
+Environment=CDH_CONFIG_PATH=/run/confidential-containers/cdh.toml
 ExecStartPre=mkdir -p /run/kata-containers
 ExecStart=/usr/local/bin/kata-agent --config /etc/agent-config.toml
 ExecStartPre=-umount /sys/fs/cgroup/misc