From 73a063bec330f745c1d762e3f521953e8b1aa04d Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 13 Aug 2024 16:03:38 +0200 Subject: [PATCH 01/12] Move registry mirrors into certs.d directory. When importing this from the conf.d directory, the plugin merge results in a weird state. --- .../generator/ignition/generator.go | 63 +++++++++++-------- .../generator/ignition/generator_test.go | 50 +++++++++------ 2 files changed, 67 insertions(+), 46 deletions(-) diff --git a/pkg/controller/operatingsystemconfig/generator/ignition/generator.go b/pkg/controller/operatingsystemconfig/generator/ignition/generator.go index f5e62c1..68a9238 100644 --- a/pkg/controller/operatingsystemconfig/generator/ignition/generator.go +++ b/pkg/controller/operatingsystemconfig/generator/ignition/generator.go @@ -56,7 +56,21 @@ func (t *IgnitionGenerator) Generate(logr logr.Logger, config *generator.Operati cmd = &c } - data, err := ignitionFromOperatingSystemConfig(config) + cfg, err := ignitionFromOperatingSystemConfig(config) + if err != nil { + return nil, nil, err + } + + outCfg, report := types.Convert(cfg, "", nil) + if report.IsFatal() { + return nil, nil, fmt.Errorf("could not transpile ignition config: %s", report.String()) + } + + data, err := json.Marshal(outCfg) + if err != nil { + return nil, nil, err + } + return data, cmd, err } @@ -67,7 +81,7 @@ func (t *IgnitionGenerator) Generate(logr logr.Logger, config *generator.Operati // Starting with ignition 2.0, ignition itself contains the required parsing logic, so we can use ignition directly. // see https://github.com/coreos/ignition/blob/master/config/config.go#L38 // Therefore we must update ignition to 2.0.0 in the images and transform the gardener config to the ignition config types instead. -func ignitionFromOperatingSystemConfig(config *generator.OperatingSystemConfig) ([]byte, error) { +func ignitionFromOperatingSystemConfig(config *generator.OperatingSystemConfig) (types.Config, error) { cfg := types.Config{} imageProviderConfig := &metalextensionv1alpha1.ImageProviderConfig{} @@ -75,7 +89,7 @@ func ignitionFromOperatingSystemConfig(config *generator.OperatingSystemConfig) if config.Object != nil && config.Object.Spec.ProviderConfig != nil { err := decodeProviderConfig(config.Object.Spec.ProviderConfig, imageProviderConfig) if err != nil { - return nil, fmt.Errorf("unable to decode providerConfig") + return types.Config{}, fmt.Errorf("unable to decode providerConfig") } } if imageProviderConfig.NetworkIsolation != nil { @@ -134,6 +148,7 @@ func ignitionFromOperatingSystemConfig(config *generator.OperatingSystemConfig) cfg.Storage.Files = append(cfg.Storage.Files, types.File{ Path: "/etc/containerd/config.toml", Filesystem: "root", + Mode: &types.DefaultFileMode, Contents: types.FileContents{ Inline: containerdConfig, }, @@ -141,16 +156,11 @@ func ignitionFromOperatingSystemConfig(config *generator.OperatingSystemConfig) }) if len(networkIsolation.RegistryMirrors) > 0 { - cfg.Storage.Files = append(cfg.Storage.Files, additionalContainerdMirrors(networkIsolation.RegistryMirrors)) + cfg.Storage.Files = append(cfg.Storage.Files, additionalContainerdMirrors(networkIsolation.RegistryMirrors)...) } } - outCfg, report := types.Convert(cfg, "", nil) - if report.IsFatal() { - return nil, fmt.Errorf("could not transpile ignition config: %s", report.String()) - } - - return json.Marshal(outCfg) + return cfg, nil } // decodeProviderConfig decodes the provider config into the given struct @@ -180,30 +190,29 @@ func getGardenerDecoder() runtime.Decoder { return decoder } -func additionalContainerdMirrors(mirrors []metalextensionv1alpha1.RegistryMirror) types.File { - content := `# Generated by os-extension-metal -version = 2 +func additionalContainerdMirrors(mirrors []metalextensionv1alpha1.RegistryMirror) []types.File { + var files []types.File -[plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] -` for _, m := range mirrors { for _, of := range m.MirrorOf { - content += fmt.Sprintf(` [plugins."io.containerd.grpc.v1.cri".registry.mirrors.%q] - endpoint = [%q] + content := fmt.Sprintf(`server = "https://%s" + +[host.%q] + capabilities = ["pull", "resolve"] `, of, m.Endpoint) - } - } - return types.File{ - Path: "/etc/containerd/conf.d/isolated-cluster.toml", - Filesystem: "root", - Mode: &types.DefaultFileMode, - Contents: types.FileContents{ - Inline: content, - }, + files = append(files, types.File{ + Path: fmt.Sprintf("/etc/containerd/certs.d/%s/hosts.toml", of), + Filesystem: "root", + Mode: &types.DefaultFileMode, + Contents: types.FileContents{ + Inline: content, + }, + }) + } } + return files } func additionalDNSConfFiles(dnsServers []string) []types.File { diff --git a/pkg/controller/operatingsystemconfig/generator/ignition/generator_test.go b/pkg/controller/operatingsystemconfig/generator/ignition/generator_test.go index 4c17fed..89ca59f 100644 --- a/pkg/controller/operatingsystemconfig/generator/ignition/generator_test.go +++ b/pkg/controller/operatingsystemconfig/generator/ignition/generator_test.go @@ -2,12 +2,12 @@ package ignition import ( "encoding/json" - "reflect" "testing" "github.com/flatcar/container-linux-config-transpiler/config/types" "github.com/gardener/gardener/extensions/pkg/controller/operatingsystemconfig/oscommon/generator" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/google/go-cmp/cmp" metalextensionv1alpha1 "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal/v1alpha1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" @@ -164,20 +164,37 @@ NTP=134.60.1.27 134.60.111.110 }, { Filesystem: "root", - Path: "/etc/containerd/conf.d/isolated-cluster.toml", + Path: "/etc/containerd/certs.d/ghcr.io/hosts.toml", Mode: &types.DefaultFileMode, Contents: types.FileContents{ - Inline: `# Generated by os-extension-metal -version = 2 + Inline: `server = "https://ghcr.io" + +[host."https://r.metal-stack.dev"] + capabilities = ["pull", "resolve"] +`, + }, + }, + { + Filesystem: "root", + Path: "/etc/containerd/certs.d/quay.io/hosts.toml", + Mode: &types.DefaultFileMode, + Contents: types.FileContents{ + Inline: `server = "https://quay.io" -[plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."ghcr.io"] - endpoint = ["https://r.metal-stack.dev"] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."quay.io"] - endpoint = ["https://r.metal-stack.dev"] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["http://localhost:8080"] +[host."https://r.metal-stack.dev"] + capabilities = ["pull", "resolve"] +`, + }, + }, + { + Filesystem: "root", + Path: "/etc/containerd/certs.d/docker.io/hosts.toml", + Mode: &types.DefaultFileMode, + Contents: types.FileContents{ + Inline: `server = "https://docker.io" + +[host."http://localhost:8080"] + capabilities = ["pull", "resolve"] `, }, }, @@ -195,13 +212,8 @@ version = 2 return } - cfg, _ := types.Convert(tt.want, "", nil) - want, err := json.Marshal(cfg) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(got, want) { - t.Errorf("ignitionFromOperatingSystemConfig()\ngot:\n%v\nwant:\n%v", string(got), string(want)) + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf(diff) } }) } From a63debdc57ae48383046a5b5a0d9345a5a7f5f75 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 20 Aug 2024 11:31:58 +0200 Subject: [PATCH 02/12] Move modifications to actuator in order to pick them up with GNA later. --- go.mod | 2 +- .../operatingsystemconfig/actuator.go | 176 ++++++++++++++- .../generator/ignition/generator.go | 172 +------------- .../generator/ignition/generator_test.go | 211 +++++------------- 4 files changed, 236 insertions(+), 325 deletions(-) diff --git a/go.mod b/go.mod index 6864dca..bbc0c40 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/gardener/gardener v1.89.4 github.com/go-logr/logr v1.4.2 github.com/golang/mock v1.6.0 + github.com/google/go-cmp v0.6.0 github.com/metal-stack/gardener-extension-provider-metal v0.24.4 github.com/onsi/ginkgo/v2 v2.20.0 github.com/onsi/gomega v1.34.1 @@ -58,7 +59,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect github.com/google/uuid v1.6.0 // indirect diff --git a/pkg/controller/operatingsystemconfig/actuator.go b/pkg/controller/operatingsystemconfig/actuator.go index 511afbd..2de9ec8 100644 --- a/pkg/controller/operatingsystemconfig/actuator.go +++ b/pkg/controller/operatingsystemconfig/actuator.go @@ -18,29 +18,66 @@ import ( "context" _ "embed" "fmt" + "strings" "github.com/gardener/gardener/extensions/pkg/controller/operatingsystemconfig" oscommonactuator "github.com/gardener/gardener/extensions/pkg/controller/operatingsystemconfig/oscommon/actuator" + gardenv1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/go-logr/logr" + metalextensionv1alpha1 "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal/v1alpha1" + "github.com/metal-stack/os-metal-extension/pkg/controller/operatingsystemconfig/generator" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" +) - "github.com/metal-stack/os-metal-extension/pkg/controller/operatingsystemconfig/generator" +const ( + containerdConfig = `# Generated by os-extension-metal +imports = ["/etc/containerd/conf.d/*.toml"] +disabled_plugins = [] + +[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +` ) type actuator struct { - client client.Client + client client.Client + decoder runtime.Decoder } // NewActuator creates a new Actuator that updates the status of the handled OperatingSystemConfig resources. func NewActuator(mgr manager.Manager) operatingsystemconfig.Actuator { + scheme := runtime.NewScheme() + utilruntime.Must(gardenv1beta1.AddToScheme(scheme)) + decoder := serializer.NewCodecFactory(scheme).UniversalDecoder() + return &actuator{ - client: mgr.GetClient(), + client: mgr.GetClient(), + decoder: decoder, } } func (a *actuator) Reconcile(ctx context.Context, log logr.Logger, osc *extensionsv1alpha1.OperatingSystemConfig) ([]byte, *string, []string, []string, []extensionsv1alpha1.Unit, []extensionsv1alpha1.File, error) { + imageProviderConfig := &metalextensionv1alpha1.ImageProviderConfig{} + + networkIsolation := &metalextensionv1alpha1.NetworkIsolation{} + if osc.Spec.ProviderConfig != nil { + err := decodeProviderConfig(a.decoder, osc.Spec.ProviderConfig, imageProviderConfig) + if err != nil { + return nil, nil, nil, nil, nil, nil, fmt.Errorf("unable to decode providerConfig") + } + } + if imageProviderConfig.NetworkIsolation != nil { + networkIsolation = imageProviderConfig.NetworkIsolation + } + + osc = addOsSpecifics(osc, networkIsolation) + cloudConfig, command, err := oscommonactuator.CloudConfigFromOperatingSystemConfig(ctx, log, a.client, osc, generator.IgnitionGenerator()) if err != nil { return nil, nil, nil, nil, nil, nil, fmt.Errorf("could not generate cloud config: %w", err) @@ -64,3 +101,136 @@ func (a *actuator) ForceDelete(ctx context.Context, log logr.Logger, osc *extens func (a *actuator) Restore(ctx context.Context, log logr.Logger, osc *extensionsv1alpha1.OperatingSystemConfig) ([]byte, *string, []string, []string, []extensionsv1alpha1.Unit, []extensionsv1alpha1.File, error) { return a.Reconcile(ctx, log, osc) } + +func addOsSpecifics(osc *extensionsv1alpha1.OperatingSystemConfig, networkIsolation *metalextensionv1alpha1.NetworkIsolation) *extensionsv1alpha1.OperatingSystemConfig { + osc = osc.DeepCopy() + + dnsFiles := additionalDNSConfFiles(networkIsolation.DNSServers) + osc.Spec.Files = append(osc.Spec.Files, dnsFiles...) + + ntpFiles := additionalNTPConfFiles(networkIsolation.NTPServers) + osc.Spec.Files = append(osc.Spec.Files, ntpFiles...) + + if osc.Spec.CRIConfig != nil && osc.Spec.CRIConfig.Name == extensionsv1alpha1.CRINameContainerD { + // the debian:12 containerd ships with "cri" plugin disabled, so we need override the config that ships with the os + // + // with g/g v1.100 it would be best to just remove the config.toml and let the GNA generate the default config. + // unfortunately, ignition does not allow to remove files easily. + // along with the default import paths. see https://github.com/gardener/gardener/pull/10050) + osc.Spec.Files = append(osc.Spec.Files, extensionsv1alpha1.File{ + Path: "/etc/containerd/config.toml", + Permissions: ptr.To(int32(0644)), + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: containerdConfig, + }, + }, + }) + + if len(networkIsolation.RegistryMirrors) > 0 { + osc.Spec.Files = append(osc.Spec.Files, additionalContainerdMirrors(networkIsolation.RegistryMirrors)...) + } + } + + return osc +} + +// decodeProviderConfig decodes the provider config into the given struct +func decodeProviderConfig(decoder runtime.Decoder, providerConfig *runtime.RawExtension, into runtime.Object) error { + if providerConfig == nil { + return nil + } + + if _, _, err := decoder.Decode(providerConfig.Raw, nil, into); err != nil { + return fmt.Errorf("could not decode provider config: %w", err) + } + + return nil +} + +func additionalContainerdMirrors(mirrors []metalextensionv1alpha1.RegistryMirror) []extensionsv1alpha1.File { + var files []extensionsv1alpha1.File + + for _, m := range mirrors { + for _, of := range m.MirrorOf { + content := fmt.Sprintf(`server = "https://%s" + +[host.%q] + capabilities = ["pull", "resolve"] +`, of, m.Endpoint) + + files = append(files, extensionsv1alpha1.File{ + Path: fmt.Sprintf("/etc/containerd/certs.d/%s/hosts.toml", of), + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: content, + }, + }, + }) + } + } + + return files +} + +func additionalDNSConfFiles(dnsServers []string) []extensionsv1alpha1.File { + if len(dnsServers) == 0 { + return nil + } + resolveDNS := strings.Join(dnsServers, " ") + systemdResolvedConfd := fmt.Sprintf(`# Generated by os-extension-metal +[Resolve] +DNS=%s +Domain=~. +`, resolveDNS) + resolvConf := "# Generated by os-extension-metal\n" + for _, ip := range dnsServers { + resolvConf += fmt.Sprintf("nameserver %s\n", ip) + } + + return []extensionsv1alpha1.File{ + { + Path: "/etc/systemd/resolved.conf.d/dns.conf", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: systemdResolvedConfd, + }, + }, + }, + { + Path: "/etc/resolv.conf", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: resolvConf, + }, + }, + }, + } +} + +func additionalNTPConfFiles(ntpServers []string) []extensionsv1alpha1.File { + if len(ntpServers) == 0 { + return nil + } + ntps := strings.Join(ntpServers, " ") + renderedContent := fmt.Sprintf(`# Generated by os-extension-metal +[Time] +NTP=%s +`, ntps) + + return []extensionsv1alpha1.File{ + { + Path: "/etc/systemd/timesyncd.conf", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: renderedContent, + }, + }, + }, + } +} diff --git a/pkg/controller/operatingsystemconfig/generator/ignition/generator.go b/pkg/controller/operatingsystemconfig/generator/ignition/generator.go index 68a9238..f2ce066 100644 --- a/pkg/controller/operatingsystemconfig/generator/ignition/generator.go +++ b/pkg/controller/operatingsystemconfig/generator/ignition/generator.go @@ -3,33 +3,16 @@ package ignition import ( "encoding/json" "fmt" - "strings" "text/template" - gardenv1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" - metalextensionv1alpha1 "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal/v1alpha1" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "github.com/flatcar/container-linux-config-transpiler/config/types" "github.com/gardener/gardener/extensions/pkg/controller/operatingsystemconfig/oscommon/generator" ostemplate "github.com/gardener/gardener/extensions/pkg/controller/operatingsystemconfig/oscommon/template" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/utils/ptr" ) -const ( - containerdConfig = `# Generated by os-extension-metal -imports = ["/etc/containerd/conf.d/*.toml"] -disabled_plugins = [] - -[plugins."io.containerd.grpc.v1.cri".registry] - config_path = "/etc/containerd/certs.d" -` -) - // IgnitionGenerator generates cloud-init scripts. type IgnitionGenerator struct { cloudInitGenerator generator.Generator @@ -56,10 +39,7 @@ func (t *IgnitionGenerator) Generate(logr logr.Logger, config *generator.Operati cmd = &c } - cfg, err := ignitionFromOperatingSystemConfig(config) - if err != nil { - return nil, nil, err - } + cfg := ignitionFromOperatingSystemConfig(config) outCfg, report := types.Convert(cfg, "", nil) if report.IsFatal() { @@ -81,21 +61,9 @@ func (t *IgnitionGenerator) Generate(logr logr.Logger, config *generator.Operati // Starting with ignition 2.0, ignition itself contains the required parsing logic, so we can use ignition directly. // see https://github.com/coreos/ignition/blob/master/config/config.go#L38 // Therefore we must update ignition to 2.0.0 in the images and transform the gardener config to the ignition config types instead. -func ignitionFromOperatingSystemConfig(config *generator.OperatingSystemConfig) (types.Config, error) { +func ignitionFromOperatingSystemConfig(config *generator.OperatingSystemConfig) types.Config { cfg := types.Config{} - imageProviderConfig := &metalextensionv1alpha1.ImageProviderConfig{} - networkIsolation := &metalextensionv1alpha1.NetworkIsolation{} - if config.Object != nil && config.Object.Spec.ProviderConfig != nil { - err := decodeProviderConfig(config.Object.Spec.ProviderConfig, imageProviderConfig) - if err != nil { - return types.Config{}, fmt.Errorf("unable to decode providerConfig") - } - } - if imageProviderConfig.NetworkIsolation != nil { - networkIsolation = imageProviderConfig.NetworkIsolation - } - cfg.Systemd = types.Systemd{} for _, u := range config.Units { contents := string(u.Content) @@ -129,142 +97,10 @@ func ignitionFromOperatingSystemConfig(config *generator.OperatingSystemConfig) Contents: types.FileContents{ Inline: string(f.Content), }, - } - cfg.Storage.Files = append(cfg.Storage.Files, ignitionFile) - } - - dnsFiles := additionalDNSConfFiles(networkIsolation.DNSServers) - cfg.Storage.Files = append(cfg.Storage.Files, dnsFiles...) - - ntpFiles := additionalNTPConfFiles(networkIsolation.NTPServers) - cfg.Storage.Files = append(cfg.Storage.Files, ntpFiles...) - - if config.CRI != nil && config.CRI.Name == extensionsv1alpha1.CRINameContainerD { - // the debian:12 containerd ships with "cri" plugin disabled, so we need override the config that ships with the os - // - // with g/g v1.100 it would be best to just remove the config.toml and let the GNA generate the default config. - // unfortunately, ignition does not allow to remove files easily. - // along with the default import paths. see https://github.com/gardener/gardener/pull/10050) - cfg.Storage.Files = append(cfg.Storage.Files, types.File{ - Path: "/etc/containerd/config.toml", - Filesystem: "root", - Mode: &types.DefaultFileMode, - Contents: types.FileContents{ - Inline: containerdConfig, - }, Overwrite: ptr.To(true), - }) - - if len(networkIsolation.RegistryMirrors) > 0 { - cfg.Storage.Files = append(cfg.Storage.Files, additionalContainerdMirrors(networkIsolation.RegistryMirrors)...) - } - } - - return cfg, nil -} - -// decodeProviderConfig decodes the provider config into the given struct -func decodeProviderConfig(providerConfig *runtime.RawExtension, into runtime.Object) error { - if providerConfig == nil { - return nil - } - - if _, _, err := getGardenerDecoder().Decode(providerConfig.Raw, nil, into); err != nil { - return fmt.Errorf("could not decode provider config: %w", err) - } - - return nil -} - -var ( - decoder runtime.Decoder -) - -// getGardenerDecoder returns a decoder to decode Gardener objects -func getGardenerDecoder() runtime.Decoder { - if decoder == nil { - scheme := runtime.NewScheme() - utilruntime.Must(gardenv1beta1.AddToScheme(scheme)) - decoder = serializer.NewCodecFactory(scheme).UniversalDecoder() - } - return decoder -} - -func additionalContainerdMirrors(mirrors []metalextensionv1alpha1.RegistryMirror) []types.File { - var files []types.File - - for _, m := range mirrors { - for _, of := range m.MirrorOf { - content := fmt.Sprintf(`server = "https://%s" - -[host.%q] - capabilities = ["pull", "resolve"] -`, of, m.Endpoint) - - files = append(files, types.File{ - Path: fmt.Sprintf("/etc/containerd/certs.d/%s/hosts.toml", of), - Filesystem: "root", - Mode: &types.DefaultFileMode, - Contents: types.FileContents{ - Inline: content, - }, - }) } + cfg.Storage.Files = append(cfg.Storage.Files, ignitionFile) } - return files -} - -func additionalDNSConfFiles(dnsServers []string) []types.File { - if len(dnsServers) == 0 { - return nil - } - resolveDNS := strings.Join(dnsServers, " ") - systemdResolvedConfd := fmt.Sprintf(`# Generated by os-extension-metal - -[Resolve] -DNS=%s -Domain=~. - -`, resolveDNS) - resolvConf := "# Generated by os-extension-metal\n" - for _, ip := range dnsServers { - resolvConf += fmt.Sprintf("nameserver %s\n", ip) - } - - return []types.File{ - { - Path: "/etc/systemd/resolved.conf.d/dns.conf", - Contents: types.FileContents{ - Inline: systemdResolvedConfd, - }, - }, - { - Path: "/etc/resolv.conf", - Contents: types.FileContents{ - Inline: resolvConf, - }, - }, - } -} - -func additionalNTPConfFiles(ntpServers []string) []types.File { - if len(ntpServers) == 0 { - return nil - } - ntps := strings.Join(ntpServers, " ") - renderedContent := fmt.Sprintf(`# Generated by os-extension-metal - -[Time] -NTP=%s -`, ntps) - - return []types.File{ - { - Path: "/etc/systemd/timesyncd.conf", - Contents: types.FileContents{ - Inline: renderedContent, - }, - }, - } + return cfg } diff --git a/pkg/controller/operatingsystemconfig/generator/ignition/generator_test.go b/pkg/controller/operatingsystemconfig/generator/ignition/generator_test.go index 89ca59f..a1263f1 100644 --- a/pkg/controller/operatingsystemconfig/generator/ignition/generator_test.go +++ b/pkg/controller/operatingsystemconfig/generator/ignition/generator_test.go @@ -1,24 +1,21 @@ package ignition import ( - "encoding/json" "testing" "github.com/flatcar/container-linux-config-transpiler/config/types" "github.com/gardener/gardener/extensions/pkg/controller/operatingsystemconfig/oscommon/generator" - extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/go-logr/logr" "github.com/google/go-cmp/cmp" - metalextensionv1alpha1 "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal/v1alpha1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" ) func TestIgnitionFromOperatingSystemConfig(t *testing.T) { tests := []struct { - name string - config *generator.OperatingSystemConfig - want types.Config - wantErr bool + name string + config *generator.OperatingSystemConfig + want types.Config }{ { name: "simple service", @@ -26,11 +23,10 @@ func TestIgnitionFromOperatingSystemConfig(t *testing.T) { Units: []*generator.Unit{ { Name: "kubelet.service", - Content: []byte(("[Unit]\nDescription=kubelet\n[Install]\nWantedBy=multi-user.target\n[Service]\nExecStart=/bin/kubelet")), + Content: []byte("[Unit]\nDescription=kubelet\n[Install]\nWantedBy=multi-user.target\n[Service]\nExecStart=/bin/kubelet"), }, }, }, - wantErr: false, want: types.Config{ Systemd: types.Systemd{ Units: []types.SystemdUnit{ @@ -43,20 +39,22 @@ func TestIgnitionFromOperatingSystemConfig(t *testing.T) { }, }, }, - { name: "simple files", config: &generator.OperatingSystemConfig{ Files: []*generator.File{ { - Path: "/etc/hostname", - TransmitUnencoded: ptr.To(true), - Content: []byte("testhost"), - Permissions: ptr.To(int32(0644)), + Path: "/etc/hostname", + Content: []byte("testhost"), + Permissions: ptr.To(int32(0644)), + }, + { + Path: "/etc/foo", + Content: []byte("foo"), + Permissions: ptr.To(int32(0744)), }, }, }, - wantErr: false, want: types.Config{ Storage: types.Storage{ Files: []types.File{ @@ -64,139 +62,19 @@ func TestIgnitionFromOperatingSystemConfig(t *testing.T) { Filesystem: "root", Path: "/etc/hostname", Contents: types.FileContents{ - // FIXME here should be testhosts ??? Inline: "testhost", }, - Mode: ptr.To(0644), - }, - }, - }, - }, - }, - { - name: "containerd with network isolation", - config: &generator.OperatingSystemConfig{ - CRI: &extensionsv1alpha1.CRIConfig{ - Name: extensionsv1alpha1.CRINameContainerD, - }, - Object: &extensionsv1alpha1.OperatingSystemConfig{ - Spec: extensionsv1alpha1.OperatingSystemConfigSpec{ - DefaultSpec: extensionsv1alpha1.DefaultSpec{ - ProviderConfig: &runtime.RawExtension{ - Raw: mustMarshal(t, &metalextensionv1alpha1.ImageProviderConfig{ - NetworkIsolation: &metalextensionv1alpha1.NetworkIsolation{ - AllowedNetworks: metalextensionv1alpha1.AllowedNetworks{ - Ingress: []string{"10.0.0.1/24"}, - Egress: []string{"100.0.0.1/24"}, - }, - DNSServers: []string{"1.1.1.1", "1.0.0.1"}, - NTPServers: []string{"134.60.1.27", "134.60.111.110"}, - RegistryMirrors: []metalextensionv1alpha1.RegistryMirror{ - { - Name: "metal-stack registry", - Endpoint: "https://r.metal-stack.dev", - IP: "1.2.3.4", - Port: 443, - MirrorOf: []string{ - "ghcr.io", - "quay.io", - }, - }, - { - Name: "local registry", - Endpoint: "http://localhost:8080", - IP: "127.0.0.1", - Port: 8080, - MirrorOf: []string{ - "docker.io", - }, - }, - }, - }}), - }, - }, - }, - }, - }, - wantErr: false, - want: types.Config{ - Storage: types.Storage{ - Files: []types.File{ - { - Path: "/etc/systemd/resolved.conf.d/dns.conf", - Contents: types.FileContents{ - Inline: `# Generated by os-extension-metal - -[Resolve] -DNS=1.1.1.1 1.0.0.1 -Domain=~. - -`, - }, - }, - { - Path: "/etc/resolv.conf", - Contents: types.FileContents{ - Inline: `# Generated by os-extension-metal -nameserver 1.1.1.1 -nameserver 1.0.0.1 -`, - }, - }, - { - Path: "/etc/systemd/timesyncd.conf", - Contents: types.FileContents{ - Inline: `# Generated by os-extension-metal - -[Time] -NTP=134.60.1.27 134.60.111.110 -`, - }, - }, - { - Filesystem: "root", - Path: "/etc/containerd/config.toml", - Mode: &types.DefaultFileMode, - Contents: types.FileContents{ - Inline: containerdConfig, - }, + Mode: ptr.To(0644), Overwrite: ptr.To(true), }, { Filesystem: "root", - Path: "/etc/containerd/certs.d/ghcr.io/hosts.toml", - Mode: &types.DefaultFileMode, + Path: "/etc/foo", Contents: types.FileContents{ - Inline: `server = "https://ghcr.io" - -[host."https://r.metal-stack.dev"] - capabilities = ["pull", "resolve"] -`, - }, - }, - { - Filesystem: "root", - Path: "/etc/containerd/certs.d/quay.io/hosts.toml", - Mode: &types.DefaultFileMode, - Contents: types.FileContents{ - Inline: `server = "https://quay.io" - -[host."https://r.metal-stack.dev"] - capabilities = ["pull", "resolve"] -`, - }, - }, - { - Filesystem: "root", - Path: "/etc/containerd/certs.d/docker.io/hosts.toml", - Mode: &types.DefaultFileMode, - Contents: types.FileContents{ - Inline: `server = "https://docker.io" - -[host."http://localhost:8080"] - capabilities = ["pull", "resolve"] -`, + Inline: "foo", }, + Mode: ptr.To(0744), + Overwrite: ptr.To(true), }, }, }, @@ -206,23 +84,50 @@ NTP=134.60.1.27 134.60.111.110 for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, err := ignitionFromOperatingSystemConfig(tt.config) - if (err != nil) != tt.wantErr { - t.Errorf("ignitionFromOperatingSystemConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - + got := ignitionFromOperatingSystemConfig(tt.config) if diff := cmp.Diff(got, tt.want); diff != "" { - t.Errorf(diff) + t.Errorf("diff = %s", diff) } }) } } -func mustMarshal(t *testing.T, obj runtime.Object) []byte { - data, err := json.Marshal(obj) - if err != nil { - t.Errorf("failed to marshal object %s", err) +func Test_ignition_Transpile(t *testing.T) { + tests := []struct { + name string + osc *generator.OperatingSystemConfig + want string + wantErr bool + }{ + { + name: "transpiles to ignition format 2.3.0", + osc: &generator.OperatingSystemConfig{ + Object: &v1alpha1.OperatingSystemConfig{ + Spec: v1alpha1.OperatingSystemConfigSpec{ + Purpose: v1alpha1.OperatingSystemConfigPurposeProvision, + }, + }, + Files: []*generator.File{ + { + Path: "/etc/a", + }, + }, + }, + want: `{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{},"passwd":{},"storage":{"files":[{"filesystem":"root","overwrite":true,"path":"/etc/a","contents":{"source":"data:,","verification":{}},"mode":420}]},"systemd":{}}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr := &IgnitionGenerator{} + + got, _, err := tr.Generate(logr.Discard(), tt.osc) + if (err != nil) != tt.wantErr { + t.Errorf("error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(string(got), tt.want); diff != "" { + t.Errorf("diff = %s", diff) + } + }) } - return data } From b9c7338e69f6323cf321b3e3099ea391c0c3acc7 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 20 Aug 2024 11:42:21 +0200 Subject: [PATCH 03/12] Fix test. --- pkg/controller/operatingsystemconfig/actuator_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/operatingsystemconfig/actuator_test.go b/pkg/controller/operatingsystemconfig/actuator_test.go index f4c52b4..d324b75 100644 --- a/pkg/controller/operatingsystemconfig/actuator_test.go +++ b/pkg/controller/operatingsystemconfig/actuator_test.go @@ -74,7 +74,7 @@ var _ = Describe("Actuator", func() { Expect(string(userData)).To(HaveSuffix("}")) // check we have ignition format Expect(command).To(BeNil()) Expect(unitNames).To(ConsistOf("some-unit.service")) - Expect(fileNames).To(ConsistOf("/some/file")) + Expect(fileNames).To(ConsistOf("/some/file", "/etc/containerd/config.toml")) Expect(extensionUnits).To(BeEmpty()) Expect(extensionFiles).To(BeEmpty()) }) @@ -94,7 +94,7 @@ var _ = Describe("Actuator", func() { Expect(userData).NotTo(BeEmpty()) // legacy logic is tested in ./generator/generator_test.go Expect(command).To(BeNil()) Expect(unitNames).To(ConsistOf("some-unit.service")) - Expect(fileNames).To(ConsistOf("/some/file")) + Expect(fileNames).To(ConsistOf("/some/file", "/etc/containerd/config.toml")) }) }) }) From b88aca5448f4f2309b0f387eed2fce6e21ab7265 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 20 Aug 2024 12:08:28 +0200 Subject: [PATCH 04/12] Ensure files. --- .gitignore | 3 +- .../operatingsystemconfig/actuator.go | 29 ++++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index e8fabe8..84a0280 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ .vscode/ os-metal vendor -hack/tools/bin/* \ No newline at end of file +hack/tools/bin/* +ginkgo.report diff --git a/pkg/controller/operatingsystemconfig/actuator.go b/pkg/controller/operatingsystemconfig/actuator.go index 2de9ec8..fcf969d 100644 --- a/pkg/controller/operatingsystemconfig/actuator.go +++ b/pkg/controller/operatingsystemconfig/actuator.go @@ -18,6 +18,7 @@ import ( "context" _ "embed" "fmt" + "slices" "strings" "github.com/gardener/gardener/extensions/pkg/controller/operatingsystemconfig" @@ -106,10 +107,10 @@ func addOsSpecifics(osc *extensionsv1alpha1.OperatingSystemConfig, networkIsolat osc = osc.DeepCopy() dnsFiles := additionalDNSConfFiles(networkIsolation.DNSServers) - osc.Spec.Files = append(osc.Spec.Files, dnsFiles...) + osc.Spec.Files = ensureFile(osc.Spec.Files, dnsFiles...) ntpFiles := additionalNTPConfFiles(networkIsolation.NTPServers) - osc.Spec.Files = append(osc.Spec.Files, ntpFiles...) + osc.Spec.Files = ensureFile(osc.Spec.Files, ntpFiles...) if osc.Spec.CRIConfig != nil && osc.Spec.CRIConfig.Name == extensionsv1alpha1.CRINameContainerD { // the debian:12 containerd ships with "cri" plugin disabled, so we need override the config that ships with the os @@ -117,7 +118,7 @@ func addOsSpecifics(osc *extensionsv1alpha1.OperatingSystemConfig, networkIsolat // with g/g v1.100 it would be best to just remove the config.toml and let the GNA generate the default config. // unfortunately, ignition does not allow to remove files easily. // along with the default import paths. see https://github.com/gardener/gardener/pull/10050) - osc.Spec.Files = append(osc.Spec.Files, extensionsv1alpha1.File{ + osc.Spec.Files = ensureFile(osc.Spec.Files, extensionsv1alpha1.File{ Path: "/etc/containerd/config.toml", Permissions: ptr.To(int32(0644)), Content: extensionsv1alpha1.FileContent{ @@ -129,7 +130,7 @@ func addOsSpecifics(osc *extensionsv1alpha1.OperatingSystemConfig, networkIsolat }) if len(networkIsolation.RegistryMirrors) > 0 { - osc.Spec.Files = append(osc.Spec.Files, additionalContainerdMirrors(networkIsolation.RegistryMirrors)...) + osc.Spec.Files = ensureFile(osc.Spec.Files, additionalContainerdMirrors(networkIsolation.RegistryMirrors)...) } } @@ -234,3 +235,23 @@ NTP=%s }, } } + +func ensureFile(files []extensionsv1alpha1.File, file ...extensionsv1alpha1.File) []extensionsv1alpha1.File { + var res []extensionsv1alpha1.File + + res = append(res, files...) + + for _, f := range file { + index := slices.IndexFunc(files, func(elem extensionsv1alpha1.File) bool { + return elem.Path == f.Path + }) + + if index < 0 { + res = append(res, f) + } else { + res[index] = f + } + } + + return res +} From ebc93f59c430012c20fc0fad9940644229c60233 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 20 Aug 2024 12:31:21 +0200 Subject: [PATCH 05/12] Version. --- pkg/controller/operatingsystemconfig/actuator.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/operatingsystemconfig/actuator.go b/pkg/controller/operatingsystemconfig/actuator.go index fcf969d..a1be97b 100644 --- a/pkg/controller/operatingsystemconfig/actuator.go +++ b/pkg/controller/operatingsystemconfig/actuator.go @@ -38,6 +38,7 @@ import ( const ( containerdConfig = `# Generated by os-extension-metal +version = 2 imports = ["/etc/containerd/conf.d/*.toml"] disabled_plugins = [] From 39a76d9f8a53abb89c39ee4306d30cf2aff683e9 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 20 Aug 2024 13:15:34 +0200 Subject: [PATCH 06/12] Return units and files, too. --- pkg/controller/operatingsystemconfig/actuator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/operatingsystemconfig/actuator.go b/pkg/controller/operatingsystemconfig/actuator.go index a1be97b..77006fd 100644 --- a/pkg/controller/operatingsystemconfig/actuator.go +++ b/pkg/controller/operatingsystemconfig/actuator.go @@ -85,7 +85,7 @@ func (a *actuator) Reconcile(ctx context.Context, log logr.Logger, osc *extensio return nil, nil, nil, nil, nil, nil, fmt.Errorf("could not generate cloud config: %w", err) } - return cloudConfig, command, oscommonactuator.OperatingSystemConfigUnitNames(osc), oscommonactuator.OperatingSystemConfigFilePaths(osc), nil, nil, nil + return cloudConfig, command, oscommonactuator.OperatingSystemConfigUnitNames(osc), oscommonactuator.OperatingSystemConfigFilePaths(osc), osc.Spec.Units, osc.Spec.Files, nil } func (a *actuator) Delete(_ context.Context, _ logr.Logger, _ *extensionsv1alpha1.OperatingSystemConfig) error { From 5147924e247ccf5c90ced93128d6f89efe38ea4c Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 20 Aug 2024 14:26:30 +0200 Subject: [PATCH 07/12] Tests. --- pkg/controller/operatingsystemconfig/actuator_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/operatingsystemconfig/actuator_test.go b/pkg/controller/operatingsystemconfig/actuator_test.go index d324b75..b122deb 100644 --- a/pkg/controller/operatingsystemconfig/actuator_test.go +++ b/pkg/controller/operatingsystemconfig/actuator_test.go @@ -75,8 +75,8 @@ var _ = Describe("Actuator", func() { Expect(command).To(BeNil()) Expect(unitNames).To(ConsistOf("some-unit.service")) Expect(fileNames).To(ConsistOf("/some/file", "/etc/containerd/config.toml")) - Expect(extensionUnits).To(BeEmpty()) - Expect(extensionFiles).To(BeEmpty()) + Expect(extensionUnits).To(ContainElements(osc.Spec.Units)) + Expect(extensionFiles).To(ContainElements(osc.Spec.Files)) }) }) }) From 6bb669f3bdd68e998dc1cb9c8645420d228f298e Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 20 Aug 2024 15:05:43 +0200 Subject: [PATCH 08/12] Only contain extension files. --- .../operatingsystemconfig/actuator.go | 25 +++++++++++-------- .../operatingsystemconfig/actuator_test.go | 4 +-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/pkg/controller/operatingsystemconfig/actuator.go b/pkg/controller/operatingsystemconfig/actuator.go index 77006fd..2126bbb 100644 --- a/pkg/controller/operatingsystemconfig/actuator.go +++ b/pkg/controller/operatingsystemconfig/actuator.go @@ -78,14 +78,15 @@ func (a *actuator) Reconcile(ctx context.Context, log logr.Logger, osc *extensio networkIsolation = imageProviderConfig.NetworkIsolation } - osc = addOsSpecifics(osc, networkIsolation) + extensionFiles := addOsSpecifics(osc, networkIsolation) + osc.Spec.Files = ensureFile(osc.Spec.Files, extensionFiles...) cloudConfig, command, err := oscommonactuator.CloudConfigFromOperatingSystemConfig(ctx, log, a.client, osc, generator.IgnitionGenerator()) if err != nil { return nil, nil, nil, nil, nil, nil, fmt.Errorf("could not generate cloud config: %w", err) } - return cloudConfig, command, oscommonactuator.OperatingSystemConfigUnitNames(osc), oscommonactuator.OperatingSystemConfigFilePaths(osc), osc.Spec.Units, osc.Spec.Files, nil + return cloudConfig, command, oscommonactuator.OperatingSystemConfigUnitNames(osc), oscommonactuator.OperatingSystemConfigFilePaths(osc), nil, extensionFiles, nil } func (a *actuator) Delete(_ context.Context, _ logr.Logger, _ *extensionsv1alpha1.OperatingSystemConfig) error { @@ -104,14 +105,16 @@ func (a *actuator) Restore(ctx context.Context, log logr.Logger, osc *extensions return a.Reconcile(ctx, log, osc) } -func addOsSpecifics(osc *extensionsv1alpha1.OperatingSystemConfig, networkIsolation *metalextensionv1alpha1.NetworkIsolation) *extensionsv1alpha1.OperatingSystemConfig { - osc = osc.DeepCopy() +func addOsSpecifics(osc *extensionsv1alpha1.OperatingSystemConfig, networkIsolation *metalextensionv1alpha1.NetworkIsolation) []extensionsv1alpha1.File { + var extensionFiles []extensionsv1alpha1.File - dnsFiles := additionalDNSConfFiles(networkIsolation.DNSServers) - osc.Spec.Files = ensureFile(osc.Spec.Files, dnsFiles...) + if len(networkIsolation.RegistryMirrors) > 0 { + dnsFiles := additionalDNSConfFiles(networkIsolation.DNSServers) + extensionFiles = append(extensionFiles, dnsFiles...) - ntpFiles := additionalNTPConfFiles(networkIsolation.NTPServers) - osc.Spec.Files = ensureFile(osc.Spec.Files, ntpFiles...) + ntpFiles := additionalNTPConfFiles(networkIsolation.NTPServers) + extensionFiles = append(extensionFiles, ntpFiles...) + } if osc.Spec.CRIConfig != nil && osc.Spec.CRIConfig.Name == extensionsv1alpha1.CRINameContainerD { // the debian:12 containerd ships with "cri" plugin disabled, so we need override the config that ships with the os @@ -119,7 +122,7 @@ func addOsSpecifics(osc *extensionsv1alpha1.OperatingSystemConfig, networkIsolat // with g/g v1.100 it would be best to just remove the config.toml and let the GNA generate the default config. // unfortunately, ignition does not allow to remove files easily. // along with the default import paths. see https://github.com/gardener/gardener/pull/10050) - osc.Spec.Files = ensureFile(osc.Spec.Files, extensionsv1alpha1.File{ + extensionFiles = ensureFile(extensionFiles, extensionsv1alpha1.File{ Path: "/etc/containerd/config.toml", Permissions: ptr.To(int32(0644)), Content: extensionsv1alpha1.FileContent{ @@ -131,11 +134,11 @@ func addOsSpecifics(osc *extensionsv1alpha1.OperatingSystemConfig, networkIsolat }) if len(networkIsolation.RegistryMirrors) > 0 { - osc.Spec.Files = ensureFile(osc.Spec.Files, additionalContainerdMirrors(networkIsolation.RegistryMirrors)...) + extensionFiles = ensureFile(extensionFiles, additionalContainerdMirrors(networkIsolation.RegistryMirrors)...) } } - return osc + return extensionFiles } // decodeProviderConfig decodes the provider config into the given struct diff --git a/pkg/controller/operatingsystemconfig/actuator_test.go b/pkg/controller/operatingsystemconfig/actuator_test.go index b122deb..3ef3861 100644 --- a/pkg/controller/operatingsystemconfig/actuator_test.go +++ b/pkg/controller/operatingsystemconfig/actuator_test.go @@ -75,8 +75,8 @@ var _ = Describe("Actuator", func() { Expect(command).To(BeNil()) Expect(unitNames).To(ConsistOf("some-unit.service")) Expect(fileNames).To(ConsistOf("/some/file", "/etc/containerd/config.toml")) - Expect(extensionUnits).To(ContainElements(osc.Spec.Units)) - Expect(extensionFiles).To(ContainElements(osc.Spec.Files)) + Expect(extensionUnits).To(BeEmpty()) + Expect(extensionFiles).To(HaveLen(1)) }) }) }) From deb9437a5cbdd0973efcdc0c92074a46844841fc Mon Sep 17 00:00:00 2001 From: Gerrit Date: Thu, 22 Aug 2024 13:03:58 +0200 Subject: [PATCH 09/12] Naming. --- pkg/controller/operatingsystemconfig/actuator.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/controller/operatingsystemconfig/actuator.go b/pkg/controller/operatingsystemconfig/actuator.go index 2126bbb..83a72a0 100644 --- a/pkg/controller/operatingsystemconfig/actuator.go +++ b/pkg/controller/operatingsystemconfig/actuator.go @@ -78,7 +78,8 @@ func (a *actuator) Reconcile(ctx context.Context, log logr.Logger, osc *extensio networkIsolation = imageProviderConfig.NetworkIsolation } - extensionFiles := addOsSpecifics(osc, networkIsolation) + extensionFiles := getExtensionFiles(osc, networkIsolation) + osc.Spec.Files = ensureFile(osc.Spec.Files, extensionFiles...) cloudConfig, command, err := oscommonactuator.CloudConfigFromOperatingSystemConfig(ctx, log, a.client, osc, generator.IgnitionGenerator()) @@ -105,7 +106,7 @@ func (a *actuator) Restore(ctx context.Context, log logr.Logger, osc *extensions return a.Reconcile(ctx, log, osc) } -func addOsSpecifics(osc *extensionsv1alpha1.OperatingSystemConfig, networkIsolation *metalextensionv1alpha1.NetworkIsolation) []extensionsv1alpha1.File { +func getExtensionFiles(osc *extensionsv1alpha1.OperatingSystemConfig, networkIsolation *metalextensionv1alpha1.NetworkIsolation) []extensionsv1alpha1.File { var extensionFiles []extensionsv1alpha1.File if len(networkIsolation.RegistryMirrors) > 0 { From 73dd14992584d8b71ba568ed293aaa06c6481558 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Fri, 30 Aug 2024 11:01:35 +0200 Subject: [PATCH 10/12] Review. --- .../operatingsystemconfig/actuator.go | 23 ++++--- .../operatingsystemconfig/actuator_test.go | 61 ++++++++++++++++++- 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/pkg/controller/operatingsystemconfig/actuator.go b/pkg/controller/operatingsystemconfig/actuator.go index 83a72a0..f7990bc 100644 --- a/pkg/controller/operatingsystemconfig/actuator.go +++ b/pkg/controller/operatingsystemconfig/actuator.go @@ -80,7 +80,7 @@ func (a *actuator) Reconcile(ctx context.Context, log logr.Logger, osc *extensio extensionFiles := getExtensionFiles(osc, networkIsolation) - osc.Spec.Files = ensureFile(osc.Spec.Files, extensionFiles...) + osc.Spec.Files = EnsureFiles(osc.Spec.Files, extensionFiles...) cloudConfig, command, err := oscommonactuator.CloudConfigFromOperatingSystemConfig(ctx, log, a.client, osc, generator.IgnitionGenerator()) if err != nil { @@ -123,7 +123,7 @@ func getExtensionFiles(osc *extensionsv1alpha1.OperatingSystemConfig, networkIso // with g/g v1.100 it would be best to just remove the config.toml and let the GNA generate the default config. // unfortunately, ignition does not allow to remove files easily. // along with the default import paths. see https://github.com/gardener/gardener/pull/10050) - extensionFiles = ensureFile(extensionFiles, extensionsv1alpha1.File{ + extensionFiles = append(extensionFiles, extensionsv1alpha1.File{ Path: "/etc/containerd/config.toml", Permissions: ptr.To(int32(0644)), Content: extensionsv1alpha1.FileContent{ @@ -135,7 +135,7 @@ func getExtensionFiles(osc *extensionsv1alpha1.OperatingSystemConfig, networkIso }) if len(networkIsolation.RegistryMirrors) > 0 { - extensionFiles = ensureFile(extensionFiles, additionalContainerdMirrors(networkIsolation.RegistryMirrors)...) + extensionFiles = append(extensionFiles, additionalContainerdMirrors(networkIsolation.RegistryMirrors)...) } } @@ -196,6 +196,9 @@ Domain=~. resolvConf += fmt.Sprintf("nameserver %s\n", ip) } + // TODO: in osc.Spec.Type we can get the distro "ubuntu", "debian", "nvidia", ... + // from this information we should be able to deduce if systemd-resolved is used or not + return []extensionsv1alpha1.File{ { Path: "/etc/systemd/resolved.conf.d/dns.conf", @@ -241,20 +244,20 @@ NTP=%s } } -func ensureFile(files []extensionsv1alpha1.File, file ...extensionsv1alpha1.File) []extensionsv1alpha1.File { +func EnsureFiles(base []extensionsv1alpha1.File, files ...extensionsv1alpha1.File) []extensionsv1alpha1.File { var res []extensionsv1alpha1.File - res = append(res, files...) + res = append(res, base...) - for _, f := range file { - index := slices.IndexFunc(files, func(elem extensionsv1alpha1.File) bool { - return elem.Path == f.Path + for _, file := range files { + index := slices.IndexFunc(base, func(elem extensionsv1alpha1.File) bool { + return elem.Path == file.Path }) if index < 0 { - res = append(res, f) + res = append(res, file) } else { - res[index] = f + res[index] = file } } diff --git a/pkg/controller/operatingsystemconfig/actuator_test.go b/pkg/controller/operatingsystemconfig/actuator_test.go index 3ef3861..b19429d 100644 --- a/pkg/controller/operatingsystemconfig/actuator_test.go +++ b/pkg/controller/operatingsystemconfig/actuator_test.go @@ -16,19 +16,19 @@ package operatingsystemconfig_test import ( "context" + _ "embed" "github.com/gardener/gardener/extensions/pkg/controller/operatingsystemconfig" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/gardener/gardener/pkg/utils/test" "github.com/go-logr/logr" + . "github.com/metal-stack/os-metal-extension/pkg/controller/operatingsystemconfig" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/manager" - - . "github.com/metal-stack/os-metal-extension/pkg/controller/operatingsystemconfig" ) var _ = Describe("Actuator", func() { @@ -99,4 +99,61 @@ var _ = Describe("Actuator", func() { }) }) }) + + When("EnsureFiles", func() { + Describe("Ensures files", func() { + var ( + testFile1 = extensionsv1alpha1.File{ + Path: "/etc/foo", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Data: "foo", + }, + }, + } + testFile2 = extensionsv1alpha1.File{ + Path: "/etc/bar", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Data: "bar", + }, + }, + } + testFile3 = extensionsv1alpha1.File{ + Path: "/etc/bar", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Data: "bar different", + }, + }, + } + ) + + It("Ensures a single file into empty base", func() { + result := EnsureFiles([]extensionsv1alpha1.File{}, testFile1) + Expect(result).To(ConsistOf(testFile1)) + }) + + It("Ensures no file into non-empty base", func() { + result := EnsureFiles([]extensionsv1alpha1.File{ + testFile2, + }) + Expect(result).To(ConsistOf(testFile2)) + }) + + It("Ensures a single file into non-empty base", func() { + result := EnsureFiles([]extensionsv1alpha1.File{ + testFile2, + }, testFile1) + Expect(result).To(ConsistOf(testFile2, testFile1)) + }) + + It("Ensures only single file is added", func() { + result := EnsureFiles([]extensionsv1alpha1.File{ + testFile2, + }, testFile3) + Expect(result).To(ConsistOf(testFile3)) + }) + }) + }) }) From 33f0968d12f7f7508ed04ca75939c10b32c590b5 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Fri, 30 Aug 2024 11:21:35 +0200 Subject: [PATCH 11/12] Tests. --- .../operatingsystemconfig/actuator_test.go | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/pkg/controller/operatingsystemconfig/actuator_test.go b/pkg/controller/operatingsystemconfig/actuator_test.go index b19429d..581eaa5 100644 --- a/pkg/controller/operatingsystemconfig/actuator_test.go +++ b/pkg/controller/operatingsystemconfig/actuator_test.go @@ -17,14 +17,17 @@ package operatingsystemconfig_test import ( "context" _ "embed" + "encoding/json" "github.com/gardener/gardener/extensions/pkg/controller/operatingsystemconfig" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/gardener/gardener/pkg/utils/test" "github.com/go-logr/logr" + metalextensionv1alpha1 "github.com/metal-stack/gardener-extension-provider-metal/pkg/apis/metal/v1alpha1" . "github.com/metal-stack/os-metal-extension/pkg/controller/operatingsystemconfig" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -78,6 +81,158 @@ var _ = Describe("Actuator", func() { Expect(extensionUnits).To(BeEmpty()) Expect(extensionFiles).To(HaveLen(1)) }) + + It("network isolation files are added", func() { + osc = osc.DeepCopy() + osc.Spec.ProviderConfig = &runtime.RawExtension{ + Raw: mustMarshal(&metalextensionv1alpha1.ImageProviderConfig{ + NetworkIsolation: &metalextensionv1alpha1.NetworkIsolation{ + AllowedNetworks: metalextensionv1alpha1.AllowedNetworks{ + Ingress: []string{"10.0.0.1/24"}, + Egress: []string{"100.0.0.1/24"}, + }, + DNSServers: []string{"1.1.1.1", "1.0.0.1"}, + NTPServers: []string{"134.60.1.27", "134.60.111.110"}, + RegistryMirrors: []metalextensionv1alpha1.RegistryMirror{ + { + Name: "metal-stack registry", + Endpoint: "https://r.metal-stack.dev", + IP: "1.2.3.4", + Port: 443, + MirrorOf: []string{ + "ghcr.io", + "quay.io", + }, + }, + { + Name: "local registry", + Endpoint: "http://localhost:8080", + IP: "127.0.0.1", + Port: 8080, + MirrorOf: []string{ + "docker.io", + }, + }, + }, + }, + }), + } + + userData, command, unitNames, fileNames, extensionUnits, extensionFiles, err := actuator.Reconcile(ctx, log, osc) + Expect(err).NotTo(HaveOccurred()) + + Expect(string(userData)).To(ContainSubstring("/etc/containerd/config.toml")) + Expect(string(userData)).To(HavePrefix("{")) // check we have ignition format + Expect(string(userData)).To(HaveSuffix("}")) // check we have ignition format + Expect(command).To(BeNil()) + Expect(unitNames).To(ConsistOf("some-unit.service")) + Expect(fileNames).To(ConsistOf( + "/some/file", + "/etc/containerd/config.toml", + "/etc/systemd/resolved.conf.d/dns.conf", + "/etc/resolv.conf", + "/etc/systemd/timesyncd.conf", + "/etc/containerd/certs.d/ghcr.io/hosts.toml", + "/etc/containerd/certs.d/quay.io/hosts.toml", + "/etc/containerd/certs.d/docker.io/hosts.toml", + )) + Expect(extensionUnits).To(BeEmpty()) + Expect(extensionFiles).To(ConsistOf( + extensionsv1alpha1.File{ + Path: "/etc/systemd/resolved.conf.d/dns.conf", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: `# Generated by os-extension-metal +[Resolve] +DNS=1.1.1.1 1.0.0.1 +Domain=~. +`, + }, + }, + }, + extensionsv1alpha1.File{ + Path: "/etc/resolv.conf", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: `# Generated by os-extension-metal +nameserver 1.1.1.1 +nameserver 1.0.0.1 +`, + }, + }, + }, + extensionsv1alpha1.File{ + Path: "/etc/systemd/timesyncd.conf", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: `# Generated by os-extension-metal +[Time] +NTP=134.60.1.27 134.60.111.110 +`, + }, + }, + }, + extensionsv1alpha1.File{ + Path: "/etc/containerd/config.toml", + Permissions: ptr.To(int32(420)), + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: `# Generated by os-extension-metal +version = 2 +imports = ["/etc/containerd/conf.d/*.toml"] +disabled_plugins = [] + +[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +`, + }, + }, + }, + extensionsv1alpha1.File{ + Path: "/etc/containerd/certs.d/ghcr.io/hosts.toml", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: `server = "https://ghcr.io" + +[host."https://r.metal-stack.dev"] + capabilities = ["pull", "resolve"] +`, + }, + }, + }, + extensionsv1alpha1.File{ + Path: "/etc/containerd/certs.d/quay.io/hosts.toml", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: `server = "https://quay.io" + +[host."https://r.metal-stack.dev"] + capabilities = ["pull", "resolve"] +`, + }, + }, + }, + extensionsv1alpha1.File{ + Path: "/etc/containerd/certs.d/docker.io/hosts.toml", + Content: extensionsv1alpha1.FileContent{ + Inline: &extensionsv1alpha1.FileContentInline{ + Encoding: string(extensionsv1alpha1.PlainFileCodecID), + Data: `server = "https://docker.io" + +[host."http://localhost:8080"] + capabilities = ["pull", "resolve"] +`, + }, + }, + }, + )) + }) }) }) @@ -157,3 +312,8 @@ var _ = Describe("Actuator", func() { }) }) }) + +func mustMarshal(data any) []byte { + raw, _ := json.Marshal(data) + return raw +} From 86e0995b2c9a82d877fef9e59a13612f7d07a714 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Fri, 30 Aug 2024 11:23:36 +0200 Subject: [PATCH 12/12] nolint. --- pkg/controller/operatingsystemconfig/actuator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/operatingsystemconfig/actuator_test.go b/pkg/controller/operatingsystemconfig/actuator_test.go index 581eaa5..9d9825c 100644 --- a/pkg/controller/operatingsystemconfig/actuator_test.go +++ b/pkg/controller/operatingsystemconfig/actuator_test.go @@ -314,6 +314,6 @@ disabled_plugins = [] }) func mustMarshal(data any) []byte { - raw, _ := json.Marshal(data) + raw, _ := json.Marshal(data) //nolint return raw }