Skip to content

Commit

Permalink
fix: bind HostDNS to 169.254.x link-local address
Browse files Browse the repository at this point in the history 
This is an attempt to fix many issues related with trying to use Service
IP for host DNS.

Fixes #9196

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
  • Loading branch information
@smira
smira committed Aug 19, 2024
1 parent c312a46 commit ee4290f
Show file tree
Hide file tree
Showing 14 changed files with 349 additions and 476 deletions.
Binary file modified api/api.descriptors
View file
Binary file not shown.
1 change: 0 additions & 1 deletion api/resource/definitions/k8s/k8s.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ message BootstrapManifestsConfigSpec {
repeated string flannel_extra_args = 16;
string flannel_kube_service_host = 17;
string flannel_kube_service_port = 18;
string service_host_dns_address = 19;
}

// ConfigStatusSpec describes status of rendered secrets.
Expand Down
4 changes: 4 additions & 0 deletions hack/release.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,10 @@ machine:
```
Please note that on running cluster you will have to kill CoreDNS pods for this change to apply.
The IP address used to forward DNS queries has changed to the fixed `169.254.116.108` address.
For those upgrading from Talos 1.7 with `forwardKubeDNSToHost` enabled, the old Kubernetes service
can be cleaned up with `kubectl delete -n kube-system service host-dns`.
"""

[notes.lspci]
Expand Down
18 changes: 0 additions & 18 deletions internal/app/machined/pkg/controllers/k8s/control_plane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@ import (
"github.com/cosi-project/runtime/pkg/controller"
"github.com/cosi-project/runtime/pkg/controller/generic"
"github.com/cosi-project/runtime/pkg/controller/generic/transform"
"github.com/cosi-project/runtime/pkg/safe"
"github.com/cosi-project/runtime/pkg/state"
"github.com/siderolabs/gen/optional"
"github.com/siderolabs/gen/value"
"github.com/siderolabs/gen/xslices"
"go.uber.org/zap"
v1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -215,8 +212,6 @@ func NewControlPlaneSchedulerController() *ControlPlaneSchedulerController {
type ControlPlaneBootstrapManifestsController = transform.Controller[*config.MachineConfig, *k8s.BootstrapManifestsConfig]

// NewControlPlaneBootstrapManifestsController instanciates the controller.
//
//nolint:gocyclo
func NewControlPlaneBootstrapManifestsController() *ControlPlaneBootstrapManifestsController {
return transform.NewController(
transform.Settings[*config.MachineConfig, *k8s.BootstrapManifestsConfig]{
Expand Down Expand Up @@ -263,17 +258,6 @@ func NewControlPlaneBootstrapManifestsController() *ControlPlaneBootstrapManifes
server = cfgProvider.Cluster().Endpoint().String()
}

hostDNSCfg, err := safe.ReaderGetByID[*network.HostDNSConfig](ctx, r, network.HostDNSConfigID)
if err != nil && !state.IsNotFoundError(err) {
return fmt.Errorf("error getting host DNS config: %w", err)
}

var serviceHostDNSAddress string

if hostDNSCfg != nil && !value.IsZero(hostDNSCfg.TypedSpec().ServiceHostDNSAddress) {
serviceHostDNSAddress = hostDNSCfg.TypedSpec().ServiceHostDNSAddress.String()
}

*res.TypedSpec() = k8s.BootstrapManifestsConfigSpec{
Server: server,
ClusterDomain: cfgProvider.Cluster().Network().DNSDomain(),
Expand All @@ -299,8 +283,6 @@ func NewControlPlaneBootstrapManifestsController() *ControlPlaneBootstrapManifes
PodSecurityPolicyEnabled: !cfgProvider.Cluster().APIServer().DisablePodSecurityPolicy(),

TalosAPIServiceEnabled: cfgProvider.Machine().Features().KubernetesTalosAPIAccess().Enabled(),

ServiceHostDNSAddress: serviceHostDNSAddress,
}

return nil
Expand Down
8 changes: 0 additions & 8 deletions internal/app/machined/pkg/controllers/k8s/manifest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,8 +169,6 @@ func (ctrl *ManifestController) render(cfg k8s.BootstrapManifestsConfigSpec, scr
ApidPort int

TalosServiceAccount TalosServiceAccount

HostDNSAddr string
}{
BootstrapManifestsConfigSpec: cfg,
Secrets: scrt,
Expand Down Expand Up @@ -237,12 +235,6 @@ func (ctrl *ManifestController) render(cfg k8s.BootstrapManifestsConfigSpec, scr
)
}

if cfg.ServiceHostDNSAddress != "" {
defaultManifests = append(defaultManifests,
manifestDesc{"15-host-dns-service", talosHostDNSSvcTemplate},
)
}

manifests := make([]renderedManifest, len(defaultManifests))

for i := range defaultManifests {
Expand Down
3 changes: 0 additions & 3 deletions internal/app/machined/pkg/controllers/k8s/templates.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,3 @@ var flannelTemplate = string(flannel.Template)
//
//go:embed templates/talos-service-account-crd-template.yaml
var talosServiceAccountCRDTemplate string

//go:embed templates/talos-host-dns-svc-template.yaml
var talosHostDNSSvcTemplate string
33 changes: 0 additions & 33 deletions internal/app/machined/pkg/controllers/k8s/templates/talos-host-dns-svc-template.yaml
View file

This file was deleted.

27 changes: 9 additions & 18 deletions internal/app/machined/pkg/controllers/network/hostdns_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"go.uber.org/zap"

talosconfig "github.com/siderolabs/talos/pkg/machinery/config"
"github.com/siderolabs/talos/pkg/machinery/constants"
"github.com/siderolabs/talos/pkg/machinery/nethelpers"
"github.com/siderolabs/talos/pkg/machinery/resources/config"
"github.com/siderolabs/talos/pkg/machinery/resources/network"
Expand Down Expand Up @@ -101,17 +102,7 @@ func (ctrl *HostDNSConfigController) Run(ctx context.Context, r controller.Runti
res.TypedSpec().ResolveMemberNames = cfgProvider.Machine().Features().HostDNS().ResolveMemberNames()

if cfgProvider.Machine().Features().HostDNS().ForwardKubeDNSToHost() {
serviceCIDRStr := cfgProvider.Cluster().Network().ServiceCIDRs()[0]

serviceCIDR, err := netip.ParsePrefix(serviceCIDRStr)
if err != nil {
return fmt.Errorf("error parsing service CIDR: %w", err)
}

newServiceAddr = serviceCIDR.Addr()
for range 9 {
newServiceAddr = newServiceAddr.Next()
}
newServiceAddr = netip.MustParseAddr(constants.HostDNSAddress)

res.TypedSpec().ListenAddresses = append(res.TypedSpec().ListenAddresses, netip.AddrPortFrom(newServiceAddr, 53))
res.TypedSpec().ServiceHostDNSAddress = newServiceAddr
Expand All @@ -131,7 +122,7 @@ func (ctrl *HostDNSConfigController) Run(ctx context.Context, r controller.Runti
}
}

if err = ctrl.cleanupLinkSpecs(
if err = ctrl.cleanupAddressSpecs(
ctx,
r,
func(id resource.ID) bool {
Expand All @@ -150,28 +141,28 @@ func (ctrl *HostDNSConfigController) Run(ctx context.Context, r controller.Runti
}
}

func (ctrl *HostDNSConfigController) cleanupLinkSpecs(ctx context.Context, r controller.Runtime, checkResource func(id resource.ID) bool, logger *zap.Logger) error {
func (ctrl *HostDNSConfigController) cleanupAddressSpecs(ctx context.Context, r controller.Runtime, checkResource func(id resource.ID) bool, logger *zap.Logger) error {
list, err := safe.ReaderList[*network.AddressSpec](ctx, r, network.NewAddressSpec(network.ConfigNamespaceName, "").Metadata())
if err != nil {
return err
}

for iter := list.Iterator(); iter.Next(); {
link := iter.Value()
address := iter.Value()

if link.Metadata().Owner() != ctrl.Name() {
if address.Metadata().Owner() != ctrl.Name() {
continue
}

if checkResource(link.Metadata().ID()) {
if checkResource(address.Metadata().ID()) {
continue
}

if err = r.Destroy(ctx, link.Metadata()); err != nil && !state.IsNotFoundError(err) {
if err = r.Destroy(ctx, address.Metadata()); err != nil && !state.IsNotFoundError(err) {
return err
}

logger.Info("destroyed link spec", zap.String("link_id", link.Metadata().ID()))
logger.Info("destroyed address spec", zap.String("address_id", address.Metadata().ID()))
}

return nil
Expand Down
Loading

0 comments on commit ee4290f

Please sign in to comment.