diff --git a/Dockerfile b/Dockerfile index 11c992cdd9..5c517ad027 100644 --- a/Dockerfile +++ b/Dockerfile @@ -155,6 +155,8 @@ COPY ./api/cluster/cluster.proto /api/cluster/cluster.proto RUN protoc -I/api -I/api/vendor/ --go_out=paths=source_relative:/api --go-grpc_out=paths=source_relative:/api cluster/cluster.proto COPY ./api/resource/resource.proto /api/resource/resource.proto RUN protoc -I/api -I/api/vendor/ --go_out=paths=source_relative:/api --go-grpc_out=paths=source_relative:/api resource/resource.proto +COPY ./api/resource/secrets/secrets.proto /api/resource/secrets/secrets.proto +RUN protoc -I/api -I/api/vendor/ --go_out=paths=source_relative:/api --go-grpc_out=paths=source_relative:/api resource/secrets/secrets.proto COPY ./api/inspect/inspect.proto /api/inspect/inspect.proto RUN protoc -I/api -I/api/vendor/ --go_out=paths=source_relative:/api --go-grpc_out=paths=source_relative:/api inspect/inspect.proto # Gofumports generated files to adjust import order @@ -176,6 +178,7 @@ COPY --from=generate-build /api/network/*.pb.go /pkg/machinery/api/network/ COPY --from=generate-build /api/cluster/*.pb.go /pkg/machinery/api/cluster/ COPY --from=generate-build /api/storage/*.pb.go /pkg/machinery/api/storage/ COPY --from=generate-build /api/resource/*.pb.go /pkg/machinery/api/resource/ +COPY --from=generate-build /api/resource/secrets/*.pb.go /pkg/machinery/api/resource/secrets/ COPY --from=generate-build /api/inspect/*.pb.go /pkg/machinery/api/inspect/ COPY --from=go-generate /src/pkg/resources/network/ /pkg/resources/network/ COPY --from=go-generate /src/pkg/machinery/config/types/v1alpha1/ /pkg/machinery/config/types/v1alpha1/ diff --git a/api/resource/secrets/secrets.proto b/api/resource/secrets/secrets.proto new file mode 100644 index 0000000000..b2c84a60e8 --- /dev/null +++ b/api/resource/secrets/secrets.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package resource.secrets; + +option go_package = "github.com/talos-systems/talos/pkg/machinery/api/resource/secrets"; + +message CertAndKeyPEM { + bytes cert = 1; + bytes key = 2; +} + +// APISpec describes secrets.API. +message APISpec { + bytes ca_pem = 1; + CertAndKeyPEM server = 2; + CertAndKeyPEM client = 3; +} diff --git a/go.mod b/go.mod index 7821cd1f51..24588e05bc 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/containernetworking/plugins v0.9.1 github.com/coreos/go-iptables v0.6.0 github.com/coreos/go-semver v0.3.0 - github.com/cosi-project/runtime v0.0.0-20210621171302-3698c5142954 + github.com/cosi-project/runtime v0.0.0-20210623125951-f1649aff7641 github.com/docker/distribution v2.7.1+incompatible github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 diff --git a/go.sum b/go.sum index d172b985df..9bc05a9ec9 100644 --- a/go.sum +++ b/go.sum @@ -305,8 +305,8 @@ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzA github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cosi-project/runtime v0.0.0-20210621171302-3698c5142954 h1:IvvTxWEugWa0kbkSELltW7idPl35CSZ7Q+M/yJ2tIFs= -github.com/cosi-project/runtime v0.0.0-20210621171302-3698c5142954/go.mod h1:v/3MIWNuuOSdXXMl3QgCSwZrAk1fTOmQHEnTAfvDqP4= +github.com/cosi-project/runtime v0.0.0-20210623125951-f1649aff7641 h1:InlDrG3Vg+wAwA/V9Uts5h+/upC9cNwmoB6kSYehBPg= +github.com/cosi-project/runtime v0.0.0-20210623125951-f1649aff7641/go.mod h1:v/3MIWNuuOSdXXMl3QgCSwZrAk1fTOmQHEnTAfvDqP4= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= diff --git a/internal/app/apid/main.go b/internal/app/apid/main.go index e0137ab9a0..ce9a9ef7a7 100644 --- a/internal/app/apid/main.go +++ b/internal/app/apid/main.go @@ -9,8 +9,10 @@ import ( "flag" "log" "regexp" - "strings" + "github.com/cosi-project/runtime/api/v1alpha1" + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/protobuf/client" debug "github.com/talos-systems/go-debug" "github.com/talos-systems/grpc-proxy/proxy" "golang.org/x/sync/errgroup" @@ -23,15 +25,11 @@ import ( "github.com/talos-systems/talos/pkg/grpc/factory" "github.com/talos-systems/talos/pkg/grpc/middleware/authz" "github.com/talos-systems/talos/pkg/grpc/proxy/backend" - "github.com/talos-systems/talos/pkg/machinery/config/configloader" "github.com/talos-systems/talos/pkg/machinery/constants" "github.com/talos-systems/talos/pkg/startup" ) -var ( - endpoints *string - useK8sEndpoints *bool -) +var rbacEnabled *bool func runDebugServer(ctx context.Context) { const debugAddr = ":9981" @@ -49,8 +47,7 @@ func runDebugServer(ctx context.Context) { func Main() { log.SetFlags(log.Lshortfile | log.Ldate | log.Lmicroseconds | log.Ltime) - endpoints = flag.String("endpoints", "", "the static list of IPs of the control plane nodes") - useK8sEndpoints = flag.Bool("use-kubernetes-endpoints", false, "use Kubernetes master node endpoints as control plane endpoints") + rbacEnabled = flag.Bool("enable-rbac", false, "enable RBAC for Talos API") flag.Parse() @@ -60,22 +57,15 @@ func Main() { log.Fatalf("failed to seed RNG: %v", err) } - config, err := configloader.NewFromStdin() + runtimeConn, err := grpc.Dial("unix://"+constants.APIRuntimeSocketPath, grpc.WithInsecure()) if err != nil { - log.Fatalf("open config: %v", err) + log.Fatalf("failed to dial runtime connection: %v", err) } - var endpointsProvider provider.Endpoints - - if *useK8sEndpoints { - endpointsProvider = &provider.KubernetesEndpoints{} - } else { - endpointsProvider = &provider.StaticEndpoints{ - Endpoints: strings.Split(*endpoints, ","), - } - } + stateClient := v1alpha1.NewStateClient(runtimeConn) + resources := state.WrapCore(client.NewAdapter(stateClient)) - tlsConfig, err := provider.NewTLSConfig(config, endpointsProvider) + tlsConfig, err := provider.NewTLSConfig(resources) if err != nil { log.Fatalf("failed to create remote certificate provider: %+v", err) } @@ -121,7 +111,7 @@ func Main() { errGroup.Go(func() error { mode := authz.Disabled - if config.Machine().Features().RBACEnabled() { + if *rbacEnabled { mode = authz.Enabled } diff --git a/internal/app/apid/pkg/provider/endpoints.go b/internal/app/apid/pkg/provider/endpoints.go deleted file mode 100644 index 66dbc20a81..0000000000 --- a/internal/app/apid/pkg/provider/endpoints.go +++ /dev/null @@ -1,57 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package provider - -import ( - "context" - "fmt" - "time" - - "github.com/talos-systems/go-retry/retry" - - "github.com/talos-systems/talos/pkg/kubernetes" -) - -// Endpoints interfaces describes a control plane endpoints provider. -type Endpoints interface { - GetEndpoints() (endpoints []string, err error) -} - -// StaticEndpoints provides static list of endpoints. -type StaticEndpoints struct { - Endpoints []string -} - -// GetEndpoints implements Endpoints inteface. -func (e *StaticEndpoints) GetEndpoints() (endpoints []string, err error) { - return e.Endpoints, nil -} - -// KubernetesEndpoints provides dynamic list of control plane endpoints via Kubernetes Endpoints resource. -type KubernetesEndpoints struct{} - -// GetEndpoints implements Endpoints inteface. -func (e *KubernetesEndpoints) GetEndpoints() (endpoints []string, err error) { - err = retry.Constant(8*time.Minute, retry.WithUnits(3*time.Second), retry.WithJitter(time.Second), retry.WithErrorLogging(true)).Retry(func() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 30*time.Second) - defer ctxCancel() - - var client *kubernetes.Client - - client, err = kubernetes.NewClientFromKubeletKubeconfig() - if err != nil { - return retry.ExpectedError(fmt.Errorf("failed to create client: %w", err)) - } - - endpoints, err = client.MasterIPs(ctx) - if err != nil { - return retry.ExpectedError(err) - } - - return nil - }) - - return endpoints, err -} diff --git a/internal/app/apid/pkg/provider/provider.go b/internal/app/apid/pkg/provider/provider.go new file mode 100644 index 0000000000..89ba237114 --- /dev/null +++ b/internal/app/apid/pkg/provider/provider.go @@ -0,0 +1,150 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package provider provides TLS config for client & server. +package provider + +import ( + "context" + stdlibtls "crypto/tls" + "fmt" + "log" + "sync" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/talos-systems/crypto/tls" + + "github.com/talos-systems/talos/pkg/resources/secrets" +) + +// TLSConfig provides client & server TLS configs for apid. +type TLSConfig struct { + certificateProvider *certificateProvider +} + +// NewTLSConfig builds provider from configuration and endpoints. +func NewTLSConfig(resources state.State) (*TLSConfig, error) { + watchCh := make(chan state.Event) + + if err := resources.Watch(context.TODO(), resource.NewMetadata(secrets.NamespaceName, secrets.APIType, secrets.APIID, resource.VersionUndefined), watchCh); err != nil { + return nil, fmt.Errorf("error setting up watch: %w", err) + } + + // wait for the first event to set up certificate provider + provider := &certificateProvider{} + + for { + event := <-watchCh + if event.Type == state.Destroyed { + continue + } + + apiCerts := event.Resource.(*secrets.API) //nolint:errcheck,forcetypeassert + + if err := provider.Update(apiCerts); err != nil { + return nil, err + } + + break + } + + go func() { + for { + event := <-watchCh + if event.Type == state.Destroyed { + continue + } + + apiCerts := event.Resource.(*secrets.API) //nolint:errcheck,forcetypeassert + + if err := provider.Update(apiCerts); err != nil { + log.Printf("failed updating cert: %v", err) + } + } + }() + + return &TLSConfig{ + certificateProvider: provider, + }, nil +} + +// ServerConfig generates server-side tls.Config. +func (tlsConfig *TLSConfig) ServerConfig() (*stdlibtls.Config, error) { + ca, err := tlsConfig.certificateProvider.GetCA() + if err != nil { + return nil, fmt.Errorf("failed to get root CA: %w", err) + } + + return tls.New( + tls.WithClientAuthType(tls.Mutual), + tls.WithCACertPEM(ca), + tls.WithServerCertificateProvider(tlsConfig.certificateProvider), + ) +} + +// ClientConfig generates client-side tls.Config. +func (tlsConfig *TLSConfig) ClientConfig() (*stdlibtls.Config, error) { + ca, err := tlsConfig.certificateProvider.GetCA() + if err != nil { + return nil, fmt.Errorf("failed to get root CA: %w", err) + } + + return tls.New( + tls.WithClientAuthType(tls.Mutual), + tls.WithCACertPEM(ca), + tls.WithClientCertificateProvider(tlsConfig.certificateProvider), + ) +} + +type certificateProvider struct { + mu sync.Mutex + + apiCerts *secrets.API + clientCert, serverCert *stdlibtls.Certificate +} + +func (p *certificateProvider) Update(apiCerts *secrets.API) error { + p.mu.Lock() + defer p.mu.Unlock() + + p.apiCerts = apiCerts + + serverCert, err := stdlibtls.X509KeyPair(p.apiCerts.TypedSpec().Server.Crt, p.apiCerts.TypedSpec().Server.Key) + if err != nil { + return fmt.Errorf("failed to parse server cert and key into a TLS Certificate: %w", err) + } + + p.serverCert = &serverCert + + clientCert, err := stdlibtls.X509KeyPair(p.apiCerts.TypedSpec().Client.Crt, p.apiCerts.TypedSpec().Client.Key) + if err != nil { + return fmt.Errorf("failed to parse client cert and key into a TLS Certificate: %w", err) + } + + p.clientCert = &clientCert + + return nil +} + +func (p *certificateProvider) GetCA() ([]byte, error) { + p.mu.Lock() + defer p.mu.Unlock() + + return p.apiCerts.TypedSpec().CA.Crt, nil +} + +func (p *certificateProvider) GetCertificate(h *stdlibtls.ClientHelloInfo) (*stdlibtls.Certificate, error) { + p.mu.Lock() + defer p.mu.Unlock() + + return p.serverCert, nil +} + +func (p *certificateProvider) GetClientCertificate(*stdlibtls.CertificateRequestInfo) (*stdlibtls.Certificate, error) { + p.mu.Lock() + defer p.mu.Unlock() + + return p.clientCert, nil +} diff --git a/internal/app/apid/pkg/provider/tls.go b/internal/app/apid/pkg/provider/tls.go deleted file mode 100644 index 037717b904..0000000000 --- a/internal/app/apid/pkg/provider/tls.go +++ /dev/null @@ -1,148 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -// Package provider provides TLS config for client & server. -package provider - -import ( - stdlibtls "crypto/tls" - "fmt" - "log" - stdlibnet "net" - "reflect" - "sort" - "time" - - "github.com/talos-systems/crypto/tls" - "github.com/talos-systems/crypto/x509" - "github.com/talos-systems/net" - - "github.com/talos-systems/talos/pkg/grpc/gen" - "github.com/talos-systems/talos/pkg/machinery/config" - "github.com/talos-systems/talos/pkg/machinery/role" -) - -// TLSConfig provides client & server TLS configs for apid. -type TLSConfig struct { - endpoints Endpoints - lastEndpointList []string - generator *gen.RemoteGenerator - certificateProvider tls.CertificateProvider -} - -// NewTLSConfig builds provider from configuration and endpoints. -func NewTLSConfig(config config.Provider, endpoints Endpoints) (*TLSConfig, error) { - ips, err := net.IPAddrs() - if err != nil { - return nil, fmt.Errorf("failed to discover IP addresses: %w", err) - } - - dnsNames, err := net.DNSNames() - if err != nil { - return nil, err - } - - for _, san := range config.Machine().Security().CertSANs() { - if ip := stdlibnet.ParseIP(san); ip != nil { - ips = append(ips, ip) - } else { - dnsNames = append(dnsNames, san) - } - } - - endpointList, err := endpoints.GetEndpoints() - if err != nil { - return nil, fmt.Errorf("failed to fetch initial endpoint list: %w", err) - } - - sort.Strings(endpointList) - - tlsConfig := &TLSConfig{ - endpoints: endpoints, - lastEndpointList: endpointList, - } - - tlsConfig.generator, err = gen.NewRemoteGenerator( - config.Machine().Security().Token(), - endpointList, - ) - if err != nil { - return nil, fmt.Errorf("failed to create remote certificate generator: %w", err) - } - - tlsConfig.certificateProvider, err = tls.NewRenewingCertificateProvider( - tlsConfig.generator, - x509.DNSNames(dnsNames), - x509.IPAddresses(ips), - x509.Organization(string(role.Impersonator)), - ) - if err != nil { - return nil, err - } - - go tlsConfig.refreshEndpoints() - - return tlsConfig, nil -} - -// ServerConfig generates server-side tls.Config. -func (tlsConfig *TLSConfig) ServerConfig() (*stdlibtls.Config, error) { - ca, err := tlsConfig.certificateProvider.GetCA() - if err != nil { - return nil, fmt.Errorf("failed to get root CA: %w", err) - } - - return tls.New( - tls.WithClientAuthType(tls.Mutual), - tls.WithCACertPEM(ca), - tls.WithServerCertificateProvider(tlsConfig.certificateProvider), - ) -} - -// ClientConfig generates client-side tls.Config. -func (tlsConfig *TLSConfig) ClientConfig() (*stdlibtls.Config, error) { - ca, err := tlsConfig.certificateProvider.GetCA() - if err != nil { - return nil, fmt.Errorf("failed to get root CA: %w", err) - } - - return tls.New( - tls.WithClientAuthType(tls.Mutual), - tls.WithCACertPEM(ca), - tls.WithClientCertificateProvider(tlsConfig.certificateProvider), - ) -} - -func (tlsConfig *TLSConfig) refreshEndpoints() { - // refresh endpoints 1/20 of the default certificate validity time - ticker := time.NewTicker(x509.DefaultCertificateValidityDuration / 20) - defer ticker.Stop() - - for { - <-ticker.C - - endpointList, err := tlsConfig.endpoints.GetEndpoints() - if err != nil { - log.Printf("error refreshing endpoints: %s", err) - - continue - } - - sort.Strings(endpointList) - - if reflect.DeepEqual(tlsConfig.lastEndpointList, endpointList) { - continue - } - - if err = tlsConfig.generator.SetEndpoints(endpointList); err != nil { - log.Printf("error setting new endpoints %v: %s", endpointList, err) - - continue - } - - tlsConfig.lastEndpointList = endpointList - - log.Printf("updated control plane endpoints to %v", endpointList) - } -} diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go index b668522294..8cda51c7ef 100644 --- a/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go @@ -76,6 +76,7 @@ import ( machinetype "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" "github.com/talos-systems/talos/pkg/machinery/constants" "github.com/talos-systems/talos/pkg/machinery/role" + timeresource "github.com/talos-systems/talos/pkg/resources/time" "github.com/talos-systems/talos/pkg/version" ) @@ -313,7 +314,14 @@ func (s *Server) Bootstrap(ctx context.Context, in *machine.BootstrapRequest) (r log.Printf("bootstrap request received") if s.Controller.Runtime().Config().Machine().Type() == machinetype.TypeJoin { - return nil, fmt.Errorf("bootstrap can only be performed on a control plane node") + return nil, status.Error(codes.FailedPrecondition, "bootstrap can only be performed on a control plane node") + } + + timeCtx, timeCtxCancel := context.WithTimeout(ctx, 5*time.Second) + defer timeCtxCancel() + + if err := timeresource.NewSyncCondition(s.Controller.Runtime().State().V1Alpha2().Resources()).Wait(timeCtx); err != nil { + return nil, status.Error(codes.FailedPrecondition, "time is not in sync yet") } go func() { diff --git a/internal/app/machined/pkg/controllers/k8s/endpoint.go b/internal/app/machined/pkg/controllers/k8s/endpoint.go new file mode 100644 index 0000000000..4380f76a1c --- /dev/null +++ b/internal/app/machined/pkg/controllers/k8s/endpoint.go @@ -0,0 +1,148 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "context" + "fmt" + "reflect" + "sort" + "time" + + "github.com/AlekSi/pointer" + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + "inet.af/netaddr" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/talos-systems/talos/pkg/conditions" + "github.com/talos-systems/talos/pkg/kubernetes" + "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" + "github.com/talos-systems/talos/pkg/machinery/constants" + "github.com/talos-systems/talos/pkg/resources/config" + "github.com/talos-systems/talos/pkg/resources/k8s" +) + +// EndpointController looks up control plane endpoints. +type EndpointController struct{} + +// Name implements controller.Controller interface. +func (ctrl *EndpointController) Name() string { + return "k8s.EndpointController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *EndpointController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: pointer.ToString(config.MachineTypeID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *EndpointController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: k8s.EndpointType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *EndpointController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + machineTypeRes, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine type: %w", err) + } + + machineType := machineTypeRes.(*config.MachineType).MachineType() + + if machineType != machine.TypeJoin { + // TODO: implemented only for machine.TypeJoin for now, should be extended to support control plane machines (for etcd join). + continue + } + + logger.Debug("waiting for kubelet client config", zap.String("file", constants.KubeletKubeconfig)) + + if err = conditions.WaitForFileToExist(constants.KubeletKubeconfig).Wait(ctx); err != nil { + return err + } + + client, err := kubernetes.NewClientFromKubeletKubeconfig() + if err != nil { + return fmt.Errorf("error building Kubernetes client: %w", err) + } + + if err = ctrl.watchEndpoints(ctx, r, logger, client); err != nil { + return err + } + } +} + +func (ctrl *EndpointController) watchEndpoints(ctx context.Context, r controller.Runtime, logger *zap.Logger, client *kubernetes.Client) error { + ticker := time.NewTicker(10 * time.Minute) + defer ticker.Stop() + + for { + // unfortunately we can't use Watch or CachedInformer here as system:node role is only allowed verb 'Get' + endpoints, err := client.CoreV1().Endpoints(corev1.NamespaceDefault).Get(ctx, "kubernetes", v1.GetOptions{}) + if err != nil { + return fmt.Errorf("error getting endpoints: %w", err) + } + + addrs := []netaddr.IP{} + + for _, endpoint := range endpoints.Subsets { + for _, addr := range endpoint.Addresses { + ip, err := netaddr.ParseIP(addr.IP) + if err == nil { + addrs = append(addrs, ip) + } + } + } + + sort.Slice(addrs, func(i, j int) bool { return addrs[i].Compare(addrs[j]) < 0 }) + + if err := r.Modify(ctx, + k8s.NewEndpoint(k8s.ControlPlaneNamespaceName, k8s.ControlPlaneEndpointsID), + func(r resource.Resource) error { + if !reflect.DeepEqual(r.(*k8s.Endpoint).TypedSpec().Addresses, addrs) { + logger.Debug("updated controlplane endpoints", zap.Any("endpoints", addrs)) + } + + r.(*k8s.Endpoint).TypedSpec().Addresses = addrs + + return nil + }, + ); err != nil { + return fmt.Errorf("error updating endpoints: %w", err) + } + + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} diff --git a/internal/app/machined/pkg/controllers/secrets/api.go b/internal/app/machined/pkg/controllers/secrets/api.go new file mode 100644 index 0000000000..c6e745cb47 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/api.go @@ -0,0 +1,436 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "fmt" + "net" + "time" + + "github.com/AlekSi/pointer" + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/talos-systems/crypto/x509" + "go.uber.org/zap" + + "github.com/talos-systems/talos/pkg/grpc/gen" + "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" + "github.com/talos-systems/talos/pkg/machinery/role" + "github.com/talos-systems/talos/pkg/resources/config" + "github.com/talos-systems/talos/pkg/resources/k8s" + "github.com/talos-systems/talos/pkg/resources/network" + "github.com/talos-systems/talos/pkg/resources/secrets" + timeresource "github.com/talos-systems/talos/pkg/resources/time" + "github.com/talos-systems/talos/pkg/resources/v1alpha1" +) + +// APIController manages secrets.API based on configuration to provide apid certificate. +type APIController struct{} + +// Name implements controller.Controller interface. +func (ctrl *APIController) Name() string { + return "secrets.APIController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *APIController) Inputs() []controller.Input { + // initial set of inputs: wait for machine type to be known and network to be partially configured + return []controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.StatusType, + ID: pointer.ToString(network.StatusID), + Kind: controller.InputWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: pointer.ToString(config.MachineTypeID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *APIController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.APIType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *APIController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + machineTypeRes, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine type: %w", err) + } + + machineType := machineTypeRes.(*config.MachineType).MachineType() + + networkResource, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.StatusType, network.StatusID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + networkStatus := networkResource.(*network.Status).TypedSpec() + + if !(networkStatus.AddressReady && networkStatus.HostnameReady) { + continue + } + + // machine type is known and network is ready, we can now proceed to one or another reconcile loop + switch machineType { + case machine.TypeInit, machine.TypeControlPlane: + if err = ctrl.reconcile(ctx, r, logger, true); err != nil { + return err + } + case machine.TypeJoin: + if err = ctrl.reconcile(ctx, r, logger, false); err != nil { + return err + } + case machine.TypeUnknown: // nothing to do + } + + if err = ctrl.teardownAll(ctx, r); err != nil { + return err + } + } +} + +//nolint:gocyclo,cyclop +func (ctrl *APIController) reconcile(ctx context.Context, r controller.Runtime, logger *zap.Logger, isControlplane bool) error { + inputs := []controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.RootType, + ID: pointer.ToString(secrets.RootOSID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.HostnameStatusType, + ID: pointer.ToString(network.HostnameID), + Kind: controller.InputWeak, + }, + { + Namespace: network.NamespaceName, + Type: network.NodeAddressType, + ID: pointer.ToString(network.NodeAddressAccumulativeID), + Kind: controller.InputWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: pointer.ToString(config.MachineTypeID), + Kind: controller.InputWeak, + }, + // time status isn't fetched, but the fact that it is in dependencies means + // that certs will be regenerated on time sync/jump (as reconcile will be triggered) + { + Namespace: v1alpha1.NamespaceName, + Type: timeresource.StatusType, + ID: pointer.ToString(timeresource.StatusID), + Kind: controller.InputWeak, + }, + } + + if !isControlplane { + // worker nodes depend on endpoint list + inputs = append(inputs, controller.Input{ + Namespace: k8s.ControlPlaneNamespaceName, + Type: k8s.EndpointType, + ID: pointer.ToString(k8s.ControlPlaneEndpointsID), + Kind: controller.InputWeak, + }) + } + + if err := r.UpdateInputs(inputs); err != nil { + return fmt.Errorf("error updating inputs: %w", err) + } + + r.QueueReconcile() + + refreshTicker := time.NewTicker(x509.DefaultCertificateValidityDuration / 2) + defer refreshTicker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-refreshTicker.C: + } + + machineTypeRes, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine type: %w", err) + } + + machineType := machineTypeRes.(*config.MachineType).MachineType() + + switch machineType { + case machine.TypeInit, machine.TypeControlPlane: + if !isControlplane { + return fmt.Errorf("machine type changed") + } + case machine.TypeJoin: + if isControlplane { + return fmt.Errorf("machine type changed") + } + case machine.TypeUnknown: + return fmt.Errorf("machine type changed") + } + + rootResource, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, secrets.RootOSID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying resources: %w", err) + } + + continue + } + + return fmt.Errorf("error getting etcd root secrets: %w", err) + } + + rootSpec := rootResource.(*secrets.Root).OSSpec() + + hostnameResource, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.HostnameStatusType, network.HostnameID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + hostnameStatus := hostnameResource.(*network.HostnameStatus).TypedSpec() + + addressesResource, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.NodeAddressType, network.NodeAddressAccumulativeID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + nodeAddresses := addressesResource.(*network.NodeAddress).TypedSpec() + + var endpointsStr []string + + if !isControlplane { + endpointResource, err := r.Get(ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.EndpointType, k8s.ControlPlaneEndpointsID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting endpoints resource: %w", err) + } + + endpoints := endpointResource.(*k8s.Endpoint).TypedSpec() + + if len(endpoints.Addresses) == 0 { + continue + } + + endpointsStr = make([]string, 0, len(endpoints.Addresses)) + + for _, ip := range endpoints.Addresses { + endpointsStr = append(endpointsStr, ip.String()) + } + } + + ips := make([]net.IP, 0, len(rootSpec.CertSANIPs)+len(nodeAddresses.Addresses)) + + for _, ip := range rootSpec.CertSANIPs { + ips = append(ips, ip.IPAddr().IP) + } + + for _, ip := range nodeAddresses.Addresses { + ips = append(ips, ip.IPAddr().IP) + } + + dnsNames := make([]string, 0, len(rootSpec.CertSANDNSNames)+2) + + dnsNames = append(dnsNames, rootSpec.CertSANDNSNames...) + dnsNames = append(dnsNames, hostnameStatus.Hostname) + + if hostnameStatus.FQDN() != hostnameStatus.Hostname { + dnsNames = append(dnsNames, hostnameStatus.FQDN()) + } + + if isControlplane { + if err := ctrl.generateControlPlane(ctx, r, logger, rootSpec, ips, dnsNames, hostnameStatus.FQDN()); err != nil { + return err + } + } else { + if err := ctrl.generateJoin(ctx, r, logger, rootSpec, endpointsStr, ips, dnsNames, hostnameStatus.FQDN()); err != nil { + return err + } + } + } +} + +func (ctrl *APIController) generateControlPlane(ctx context.Context, r controller.Runtime, logger *zap.Logger, rootSpec *secrets.RootOSSpec, ips []net.IP, dnsNames []string, fqdn string) error { + // TODO: add keyusage + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(rootSpec.CA) + if err != nil { + return fmt.Errorf("failed to parse CA certificate: %w", err) + } + + serverCert, err := x509.NewKeyPair(ca, + x509.IPAddresses(ips), + x509.DNSNames(dnsNames), + x509.CommonName(fqdn), + x509.NotAfter(time.Now().Add(x509.DefaultCertificateValidityDuration)), + ) + if err != nil { + return fmt.Errorf("failed to generate API server cert: %w", err) + } + + clientCert, err := x509.NewKeyPair(ca, + x509.CommonName(fqdn), + x509.Organization(string(role.Impersonator)), + x509.NotAfter(time.Now().Add(x509.DefaultCertificateValidityDuration)), + ) + if err != nil { + return fmt.Errorf("failed to generate API client cert: %w", err) + } + + if err := r.Modify(ctx, secrets.NewAPI(), + func(r resource.Resource) error { + apiSecrets := r.(*secrets.API).TypedSpec() + + apiSecrets.CA = &x509.PEMEncodedCertificateAndKey{ + Crt: rootSpec.CA.Crt, + } + apiSecrets.Server = x509.NewCertificateAndKeyFromKeyPair(serverCert) + apiSecrets.Client = x509.NewCertificateAndKeyFromKeyPair(clientCert) + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + clientFingerprint, _ := x509.SPKIFingerprintFromDER(clientCert.Certificate.Certificate[0]) //nolint:errcheck + serverFingerprint, _ := x509.SPKIFingerprintFromDER(serverCert.Certificate.Certificate[0]) //nolint:errcheck + + logger.Debug("generated new certificates", + zap.Stringer("client", clientFingerprint), + zap.Stringer("server", serverFingerprint), + ) + + return nil +} + +func (ctrl *APIController) generateJoin(ctx context.Context, r controller.Runtime, logger *zap.Logger, + rootSpec *secrets.RootOSSpec, endpointsStr []string, ips []net.IP, dnsNames []string, fqdn string) error { + remoteGen, err := gen.NewRemoteGenerator(rootSpec.Token, endpointsStr) + if err != nil { + return fmt.Errorf("failed creating trustd client: %w", err) + } + + defer remoteGen.Close() //nolint:errcheck + + serverCSR, serverCert, err := x509.NewEd25519CSRAndIdentity( + x509.IPAddresses(ips), + x509.DNSNames(dnsNames), + x509.CommonName(fqdn), + ) + if err != nil { + return fmt.Errorf("failed to generate API server CSR: %w", err) + } + + var ca []byte + + ca, serverCert.Crt, err = remoteGen.Identity(serverCSR) + if err != nil { + return fmt.Errorf("failed to sign API server CSR: %w", err) + } + + clientCSR, clientCert, err := x509.NewEd25519CSRAndIdentity( + x509.CommonName(fqdn), + x509.Organization(string(role.Impersonator)), + ) + if err != nil { + return fmt.Errorf("failed to generate API client CSR: %w", err) + } + + _, clientCert.Crt, err = remoteGen.Identity(clientCSR) + if err != nil { + return fmt.Errorf("failed to sign API client CSR: %w", err) + } + + if err := r.Modify(ctx, secrets.NewAPI(), + func(r resource.Resource) error { + apiSecrets := r.(*secrets.API).TypedSpec() + + apiSecrets.CA = &x509.PEMEncodedCertificateAndKey{ + Crt: ca, + } + apiSecrets.Server = serverCert + apiSecrets.Client = clientCert + + return nil + }); err != nil { + return fmt.Errorf("error modifying resource: %w", err) + } + + clientFingerprint, _ := x509.SPKIFingerprintFromPEM(clientCert.Crt) //nolint:errcheck + serverFingerprint, _ := x509.SPKIFingerprintFromPEM(serverCert.Crt) //nolint:errcheck + + logger.Debug("generated new certificates", + zap.Stringer("client", clientFingerprint), + zap.Stringer("server", serverFingerprint), + ) + + return nil +} + +func (ctrl *APIController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.APIType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + // TODO: change this to proper teardown sequence + + for _, res := range list.Items { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/secrets/root.go b/internal/app/machined/pkg/controllers/secrets/root.go index bd58683b4b..6776dc9be5 100644 --- a/internal/app/machined/pkg/controllers/secrets/root.go +++ b/internal/app/machined/pkg/controllers/secrets/root.go @@ -13,6 +13,7 @@ import ( "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/state" "go.uber.org/zap" + "inet.af/netaddr" talosconfig "github.com/talos-systems/talos/pkg/machinery/config" "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" @@ -70,8 +71,8 @@ func (ctrl *RootController) Run(ctx context.Context, r controller.Runtime, logge cfg, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.MachineConfigType, config.V1Alpha1ID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { - if err = ctrl.teardownAll(ctx, r); err != nil { - return fmt.Errorf("error destroying static pods: %w", err) + if err = ctrl.teardown(ctx, r, secrets.RootOSID, secrets.RootEtcdID, secrets.RootKubernetesID); err != nil { + return fmt.Errorf("error destroying secrets: %w", err) } continue @@ -93,8 +94,15 @@ func (ctrl *RootController) Run(ctx context.Context, r controller.Runtime, logge machineType := machineTypeRes.(*config.MachineType).MachineType() + if err = r.Modify(ctx, secrets.NewRoot(secrets.RootOSID), func(r resource.Resource) error { + return ctrl.updateOSSecrets(cfgProvider, r.(*secrets.Root).OSSpec()) + }); err != nil { + return err + } + + // TODO: k8s secrets (partial) should be valid for the worker nodes as well, worker node should have machine (OS) CA cert (?) if machineType != machine.TypeControlPlane && machineType != machine.TypeInit { - if err = ctrl.teardownAll(ctx, r); err != nil { + if err = ctrl.teardown(ctx, r, secrets.RootEtcdID, secrets.RootKubernetesID); err != nil { return fmt.Errorf("error destroying secrets: %w", err) } @@ -115,6 +123,25 @@ func (ctrl *RootController) Run(ctx context.Context, r controller.Runtime, logge } } +func (ctrl *RootController) updateOSSecrets(cfgProvider talosconfig.Provider, osSecrets *secrets.RootOSSpec) error { + osSecrets.CA = cfgProvider.Machine().Security().CA() + + osSecrets.CertSANIPs = nil + osSecrets.CertSANDNSNames = nil + + for _, san := range cfgProvider.Machine().Security().CertSANs() { + if ip, err := netaddr.ParseIP(san); err == nil { + osSecrets.CertSANIPs = append(osSecrets.CertSANIPs, ip) + } else { + osSecrets.CertSANDNSNames = append(osSecrets.CertSANDNSNames, san) + } + } + + osSecrets.Token = cfgProvider.Machine().Security().Token() + + return nil +} + func (ctrl *RootController) updateEtcdSecrets(cfgProvider talosconfig.Provider, etcdSecrets *secrets.RootEtcdSpec) error { etcdSecrets.EtcdCA = cfgProvider.Cluster().Etcd().CA() @@ -160,17 +187,13 @@ func (ctrl *RootController) updateK8sSecrets(cfgProvider talosconfig.Provider, k return nil } -func (ctrl *RootController) teardownAll(ctx context.Context, r controller.Runtime) error { - list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, "", resource.VersionUndefined)) - if err != nil { - return err - } - +func (ctrl *RootController) teardown(ctx context.Context, r controller.Runtime, ids ...resource.ID) error { // TODO: change this to proper teardown sequence - - for _, res := range list.Items { - if err = r.Destroy(ctx, res.Metadata()); err != nil { - return err + for _, id := range ids { + if err := r.Destroy(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, id, resource.VersionUndefined)); err != nil { + if !state.IsNotFoundError(err) { + return err + } } } diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go index bef2c1b90e..bd8aa9b6a0 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go @@ -69,8 +69,6 @@ func (r *Runtime) SetConfig(b []byte) error { } // CanApplyImmediate implements the Runtime interface. -// -//nolint:gocyclo func (r *Runtime) CanApplyImmediate(b []byte) error { cfg, err := r.ValidateConfig(b) if err != nil { @@ -104,14 +102,13 @@ func (r *Runtime) CanApplyImmediate(b []byte) error { // * .machine.debug // * .machine.time // * .machine.network + // * .machine.certCANs newConfig.ClusterConfig = currentConfig.ClusterConfig newConfig.ConfigDebug = currentConfig.ConfigDebug if newConfig.MachineConfig != nil && currentConfig.MachineConfig != nil { newConfig.MachineConfig.MachineTime = currentConfig.MachineConfig.MachineTime - } - - if newConfig.MachineConfig != nil && currentConfig.MachineConfig != nil { + newConfig.MachineConfig.MachineCertSANs = currentConfig.MachineConfig.MachineCertSANs newConfig.MachineConfig.MachineNetwork = currentConfig.MachineConfig.MachineNetwork } diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go index 41be651884..0988ee6a09 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go @@ -81,6 +81,7 @@ func (ctrl *Controller) Run(ctx context.Context) error { ShadowPath: constants.SystemEtcPath, }, &k8s.ControlPlaneStaticPodController{}, + &k8s.EndpointController{}, &k8s.ExtraManifestController{}, &k8s.KubeletStaticPodController{}, &k8s.ManifestController{}, @@ -137,6 +138,7 @@ func (ctrl *Controller) Run(ctx context.Context) error { &network.TimeServerMergeController{}, &perf.StatsController{}, &network.TimeServerSpecController{}, + &secrets.APIController{}, &secrets.EtcdController{}, &secrets.KubernetesController{}, &secrets.RootController{}, diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go index 8421796c1a..6cb96c0239 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go @@ -78,6 +78,7 @@ func NewState() (*State, error) { &config.K8sControlPlane{}, &files.EtcFileSpec{}, &files.EtcFileStatus{}, + &k8s.Endpoint{}, &k8s.Manifest{}, &k8s.ManifestStatus{}, &k8s.Nodename{}, @@ -102,6 +103,7 @@ func NewState() (*State, error) { &network.TimeServerSpec{}, &perf.CPU{}, &perf.Memory{}, + &secrets.API{}, &secrets.Etcd{}, &secrets.Kubernetes{}, &secrets.Root{}, diff --git a/internal/app/machined/pkg/system/services/apid.go b/internal/app/machined/pkg/system/services/apid.go index c34aa90275..6462887889 100644 --- a/internal/app/machined/pkg/system/services/apid.go +++ b/internal/app/machined/pkg/system/services/apid.go @@ -6,20 +6,19 @@ package services import ( - "bytes" "context" "fmt" - "log" "net" "os" "path/filepath" "strings" - "sync" "github.com/containerd/containerd/oci" - "github.com/fsnotify/fsnotify" + "github.com/cosi-project/runtime/api/v1alpha1" + "github.com/cosi-project/runtime/pkg/state/protobuf/server" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/talos-systems/go-debug" + "google.golang.org/grpc" "github.com/talos-systems/talos/internal/app/machined/pkg/runtime" "github.com/talos-systems/talos/internal/app/machined/pkg/system/events" @@ -28,19 +27,14 @@ import ( "github.com/talos-systems/talos/internal/app/machined/pkg/system/runner/containerd" "github.com/talos-systems/talos/internal/app/machined/pkg/system/runner/restart" "github.com/talos-systems/talos/pkg/conditions" - "github.com/talos-systems/talos/pkg/copy" - "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" "github.com/talos-systems/talos/pkg/machinery/constants" - "github.com/talos-systems/talos/pkg/resources/network" - "github.com/talos-systems/talos/pkg/resources/time" + "github.com/talos-systems/talos/pkg/resources/secrets" ) // APID implements the Service interface. It serves as the concrete type with // the required methods. type APID struct { - ctx context.Context - cancel context.CancelFunc - wg sync.WaitGroup + runtimeServer *grpc.Server } // ID implements the Service interface. @@ -50,36 +44,35 @@ func (o *APID) ID(r runtime.Runtime) string { // PreFunc implements the Service interface. func (o *APID) PreFunc(ctx context.Context, r runtime.Runtime) error { - if r.Config().Machine().Type() == machine.TypeJoin { - o.syncKubeletPKI() + // Ensure socket dir exists + if err := os.MkdirAll(filepath.Dir(constants.APIRuntimeSocketPath), 0o750); err != nil { + return err + } + + listener, err := net.Listen("unix", constants.APIRuntimeSocketPath) + if err != nil { + return err } + // TODO: filter State to return only resources apid needs + o.runtimeServer = grpc.NewServer() + v1alpha1.RegisterStateServer(o.runtimeServer, server.NewState(r.State().V1Alpha2().Resources())) + + go o.runtimeServer.Serve(listener) //nolint:errcheck + return prepareRootfs(o.ID(r)) } // PostFunc implements the Service interface. func (o *APID) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { - if o.cancel != nil { - o.cancel() - } - - o.wg.Wait() + o.runtimeServer.Stop() - return nil + return os.RemoveAll(constants.APIRuntimeSocketPath) } // Condition implements the Service interface. func (o *APID) Condition(r runtime.Runtime) conditions.Condition { - conds := []conditions.Condition{ - time.NewSyncCondition(r.State().V1Alpha2().Resources()), - network.NewReadyCondition(r.State().V1Alpha2().Resources(), network.AddressReady, network.HostnameReady), - } - - if r.Config().Machine().Type() == machine.TypeJoin { - conds = append(conds, conditions.WaitForFileToExist(constants.KubeletKubeconfig)) - } - - return conditions.WaitForAll(conds...) + return secrets.NewAPIReadyCondition(r.State().V1Alpha2().Resources()) } // DependsOn implements the Service interface. @@ -102,12 +95,8 @@ func (o *APID) Runner(r runtime.Runtime) (runner.Runner, error) { }, } - isWorker := r.Config().Machine().Type() == machine.TypeJoin - - if !isWorker { - args.ProcessArgs = append(args.ProcessArgs, "--endpoints="+strings.Join([]string{"127.0.0.1"}, ",")) - } else { - args.ProcessArgs = append(args.ProcessArgs, "--use-kubernetes-endpoints") + if r.Config().Machine().Features().RBACEnabled() { + args.ProcessArgs = append(args.ProcessArgs, "--enable-rbac") } // Set the mounts. @@ -117,14 +106,6 @@ func (o *APID) Runner(r runtime.Runtime) (runner.Runner, error) { {Type: "bind", Destination: filepath.Dir(constants.APISocketPath), Source: filepath.Dir(constants.APISocketPath), Options: []string{"rbind", "rw"}}, } - if isWorker { - // worker requires kubelet config to refresh the certs via Kubernetes - mounts = append(mounts, - specs.Mount{Type: "bind", Destination: filepath.Dir(constants.KubeletKubeconfig), Source: constants.SystemKubeletPKIDir, Options: []string{"rbind", "ro"}}, - specs.Mount{Type: "bind", Destination: constants.KubeletPKIDir, Source: constants.SystemKubeletPKIDir, Options: []string{"rbind", "ro"}}, - ) - } - env := []string{} for key, val := range r.Config().Machine().Env() { @@ -145,17 +126,9 @@ func (o *APID) Runner(r runtime.Runtime) (runner.Runner, error) { env = append(env, "GORACE=halt_on_error=1") } - b, err := r.Config().Bytes() - if err != nil { - return nil, err - } - - stdin := bytes.NewReader(b) - return restart.New(containerd.NewRunner( r.Config().Debug(), &args, - runner.WithStdin(stdin), runner.WithLoggingManager(r.Logging()), runner.WithContainerdAddress(constants.SystemContainerdAddress), runner.WithEnv(env), @@ -188,61 +161,3 @@ func (o *APID) HealthFunc(runtime.Runtime) health.Check { func (o *APID) HealthSettings(runtime.Runtime) *health.Settings { return &health.DefaultSettings } - -func (o *APID) syncKubeletPKI() { - copyAll := func() { - if err := copy.Dir(constants.KubeletPKIDir, constants.SystemKubeletPKIDir, copy.WithMode(0o700)); err != nil { - log.Printf("failed to sync %s dir contents into %s: %s", constants.KubeletPKIDir, constants.SystemKubeletPKIDir, err) - - return - } - - if err := copy.File(constants.KubeletKubeconfig, filepath.Join(constants.SystemKubeletPKIDir, filepath.Base(constants.KubeletKubeconfig)), copy.WithMode(0o700)); err != nil { - log.Printf("failed to sync %s into %s: %s", constants.KubeletKubeconfig, constants.SystemKubeletPKIDir, err) - - return - } - } - - if err := os.MkdirAll(constants.KubeletPKIDir, 0o700); err != nil { - log.Printf("failed creating kubelet PKI directory: %s", err) - - return - } - - copyAll() - - o.ctx, o.cancel = context.WithCancel(context.Background()) - o.wg.Add(1) - - go func() { - defer o.wg.Done() - - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Printf("failed to create directory watcher %s", err) - - return - } - - defer watcher.Close() //nolint:errcheck - - err = watcher.Add(constants.KubeletPKIDir) - if err != nil { - log.Printf("failed to watch dir %s %s", constants.KubeletPKIDir, err) - - return - } - - for { - select { - case <-o.ctx.Done(): - return - case <-watcher.Events: - copyAll() - case err = <-watcher.Errors: - log.Printf("directory watch error %s", err) - } - } - }() -} diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index 337e1f9467..bc78dc6a4b 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -14,6 +14,8 @@ import ( "github.com/talos-systems/go-retry/retry" "google.golang.org/grpc/backoff" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" machineapi "github.com/talos-systems/talos/pkg/machinery/api/machine" "github.com/talos-systems/talos/pkg/machinery/client" @@ -69,7 +71,7 @@ func (s *APIBootstrapper) Bootstrap(ctx context.Context, out io.Writer) error { defer cancel() if err = cli.Bootstrap(retryCtx, &machineapi.BootstrapRequest{}); err != nil { - if strings.Contains(err.Error(), "connection refused") { + if status.Code(err) == codes.FailedPrecondition || strings.Contains(err.Error(), "connection refused") { return retry.ExpectedError(err) } diff --git a/pkg/grpc/gen/remote.go b/pkg/grpc/gen/remote.go index 8ad3879688..9e35026aee 100644 --- a/pkg/grpc/gen/remote.go +++ b/pkg/grpc/gen/remote.go @@ -66,6 +66,10 @@ func (g *RemoteGenerator) SetEndpoints(endpoints []string) error { g.connMu.Lock() defer g.connMu.Unlock() + if g.conn != nil { + g.conn.Close() //nolint:errcheck + } + g.conn = conn g.client = securityapi.NewSecurityServiceClient(g.conn) @@ -104,6 +108,7 @@ func (g *RemoteGenerator) certificate(in *securityapi.CertificateRequest) (resp } func (g *RemoteGenerator) poll(in *securityapi.CertificateRequest) (ca, crt []byte, err error) { + // TODO: rewrite with retry package timeout := time.NewTimer(time.Minute * 5) defer timeout.Stop() diff --git a/pkg/machinery/api/resource/secrets/secrets.pb.go b/pkg/machinery/api/resource/secrets/secrets.pb.go new file mode 100644 index 0000000000..9b6c74d516 --- /dev/null +++ b/pkg/machinery/api/resource/secrets/secrets.pb.go @@ -0,0 +1,248 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.15.6 +// source: resource/secrets/secrets.proto + +package secrets + +import ( + reflect "reflect" + sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type CertAndKeyPEM struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Cert []byte `protobuf:"bytes,1,opt,name=cert,proto3" json:"cert,omitempty"` + Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` +} + +func (x *CertAndKeyPEM) Reset() { + *x = CertAndKeyPEM{} + if protoimpl.UnsafeEnabled { + mi := &file_resource_secrets_secrets_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CertAndKeyPEM) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CertAndKeyPEM) ProtoMessage() {} + +func (x *CertAndKeyPEM) ProtoReflect() protoreflect.Message { + mi := &file_resource_secrets_secrets_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CertAndKeyPEM.ProtoReflect.Descriptor instead. +func (*CertAndKeyPEM) Descriptor() ([]byte, []int) { + return file_resource_secrets_secrets_proto_rawDescGZIP(), []int{0} +} + +func (x *CertAndKeyPEM) GetCert() []byte { + if x != nil { + return x.Cert + } + return nil +} + +func (x *CertAndKeyPEM) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} + +// APISpec describes secrets.API. +type APISpec struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + CaPem []byte `protobuf:"bytes,1,opt,name=ca_pem,json=caPem,proto3" json:"ca_pem,omitempty"` + Server *CertAndKeyPEM `protobuf:"bytes,2,opt,name=server,proto3" json:"server,omitempty"` + Client *CertAndKeyPEM `protobuf:"bytes,3,opt,name=client,proto3" json:"client,omitempty"` +} + +func (x *APISpec) Reset() { + *x = APISpec{} + if protoimpl.UnsafeEnabled { + mi := &file_resource_secrets_secrets_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *APISpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*APISpec) ProtoMessage() {} + +func (x *APISpec) ProtoReflect() protoreflect.Message { + mi := &file_resource_secrets_secrets_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use APISpec.ProtoReflect.Descriptor instead. +func (*APISpec) Descriptor() ([]byte, []int) { + return file_resource_secrets_secrets_proto_rawDescGZIP(), []int{1} +} + +func (x *APISpec) GetCaPem() []byte { + if x != nil { + return x.CaPem + } + return nil +} + +func (x *APISpec) GetServer() *CertAndKeyPEM { + if x != nil { + return x.Server + } + return nil +} + +func (x *APISpec) GetClient() *CertAndKeyPEM { + if x != nil { + return x.Client + } + return nil +} + +var File_resource_secrets_secrets_proto protoreflect.FileDescriptor + +var file_resource_secrets_secrets_proto_rawDesc = []byte{ + 0x0a, 0x1e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2f, 0x73, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x73, 0x2f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x10, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x73, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x73, 0x22, 0x35, 0x0a, 0x0d, 0x43, 0x65, 0x72, 0x74, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, + 0x50, 0x45, 0x4d, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x92, 0x01, 0x0a, 0x07, 0x41, 0x50, + 0x49, 0x53, 0x70, 0x65, 0x63, 0x12, 0x15, 0x0a, 0x06, 0x63, 0x61, 0x5f, 0x70, 0x65, 0x6d, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x61, 0x50, 0x65, 0x6d, 0x12, 0x37, 0x0a, 0x06, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x72, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x2e, + 0x43, 0x65, 0x72, 0x74, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x50, 0x45, 0x4d, 0x52, 0x06, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x2e, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x2e, 0x43, 0x65, 0x72, 0x74, 0x41, 0x6e, 0x64, + 0x4b, 0x65, 0x79, 0x50, 0x45, 0x4d, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x43, + 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x61, 0x6c, + 0x6f, 0x73, 0x2d, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x73, 0x2f, 0x74, 0x61, 0x6c, 0x6f, 0x73, + 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2f, 0x61, + 0x70, 0x69, 0x2f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2f, 0x73, 0x65, 0x63, 0x72, + 0x65, 0x74, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_resource_secrets_secrets_proto_rawDescOnce sync.Once + file_resource_secrets_secrets_proto_rawDescData = file_resource_secrets_secrets_proto_rawDesc +) + +func file_resource_secrets_secrets_proto_rawDescGZIP() []byte { + file_resource_secrets_secrets_proto_rawDescOnce.Do(func() { + file_resource_secrets_secrets_proto_rawDescData = protoimpl.X.CompressGZIP(file_resource_secrets_secrets_proto_rawDescData) + }) + return file_resource_secrets_secrets_proto_rawDescData +} + +var ( + file_resource_secrets_secrets_proto_msgTypes = make([]protoimpl.MessageInfo, 2) + file_resource_secrets_secrets_proto_goTypes = []interface{}{ + (*CertAndKeyPEM)(nil), // 0: resource.secrets.CertAndKeyPEM + (*APISpec)(nil), // 1: resource.secrets.APISpec + } +) + +var file_resource_secrets_secrets_proto_depIdxs = []int32{ + 0, // 0: resource.secrets.APISpec.server:type_name -> resource.secrets.CertAndKeyPEM + 0, // 1: resource.secrets.APISpec.client:type_name -> resource.secrets.CertAndKeyPEM + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_resource_secrets_secrets_proto_init() } +func file_resource_secrets_secrets_proto_init() { + if File_resource_secrets_secrets_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_resource_secrets_secrets_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CertAndKeyPEM); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_resource_secrets_secrets_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*APISpec); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_resource_secrets_secrets_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_resource_secrets_secrets_proto_goTypes, + DependencyIndexes: file_resource_secrets_secrets_proto_depIdxs, + MessageInfos: file_resource_secrets_secrets_proto_msgTypes, + }.Build() + File_resource_secrets_secrets_proto = out.File + file_resource_secrets_secrets_proto_rawDesc = nil + file_resource_secrets_secrets_proto_goTypes = nil + file_resource_secrets_secrets_proto_depIdxs = nil +} diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go index ea218803f4..6a7ab25442 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go @@ -1741,7 +1741,7 @@ type SystemDiskEncryptionConfig struct { type FeaturesConfig struct { // description: | // Enable role-based access control (RBAC). - RBAC *bool `yaml:"rbac"` + RBAC *bool `yaml:"rbac,omitempty"` } // VolumeMountConfig struct describes extra volume mount for the static pods. diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go index 10e77f95ca..42368e3dc2 100644 --- a/pkg/machinery/constants/constants.go +++ b/pkg/machinery/constants/constants.go @@ -316,6 +316,9 @@ const ( // APISocketPath is the path to file socket of apid. APISocketPath = SystemRunPath + "/apid/apid.sock" + // APIRuntimeSocketPath is the path to file socket of runtime server for apid. + APIRuntimeSocketPath = SystemRunPath + "/apid/runtime.sock" + // MachineSocketPath is the path to file socket of machine API. MachineSocketPath = SystemRunPath + "/machined/machine.sock" diff --git a/pkg/machinery/go.mod b/pkg/machinery/go.mod index fd97d57ee8..9cf428bc31 100644 --- a/pkg/machinery/go.mod +++ b/pkg/machinery/go.mod @@ -11,7 +11,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d github.com/containerd/go-cni v1.0.2 github.com/containernetworking/cni v0.8.1 // indirect; security fix in 0.8.1 - github.com/cosi-project/runtime v0.0.0-20210621171302-3698c5142954 + github.com/cosi-project/runtime v0.0.0-20210623125951-f1649aff7641 github.com/dustin/go-humanize v1.0.0 github.com/evanphx/json-patch v4.11.0+incompatible github.com/ghodss/yaml v1.0.0 diff --git a/pkg/machinery/go.sum b/pkg/machinery/go.sum index 1b70eae7ce..ab85b8efc5 100644 --- a/pkg/machinery/go.sum +++ b/pkg/machinery/go.sum @@ -17,8 +17,8 @@ github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.1 h1:7zpDnQ3T3s4ucOuJ/ZCLrYBxzkg0AELFfII3Epo9TmI= github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/cosi-project/runtime v0.0.0-20210621171302-3698c5142954 h1:IvvTxWEugWa0kbkSELltW7idPl35CSZ7Q+M/yJ2tIFs= -github.com/cosi-project/runtime v0.0.0-20210621171302-3698c5142954/go.mod h1:v/3MIWNuuOSdXXMl3QgCSwZrAk1fTOmQHEnTAfvDqP4= +github.com/cosi-project/runtime v0.0.0-20210623125951-f1649aff7641 h1:InlDrG3Vg+wAwA/V9Uts5h+/upC9cNwmoB6kSYehBPg= +github.com/cosi-project/runtime v0.0.0-20210623125951-f1649aff7641/go.mod h1:v/3MIWNuuOSdXXMl3QgCSwZrAk1fTOmQHEnTAfvDqP4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/pkg/resources/k8s/endpoint.go b/pkg/resources/k8s/endpoint.go new file mode 100644 index 0000000000..6f161173cd --- /dev/null +++ b/pkg/resources/k8s/endpoint.go @@ -0,0 +1,86 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package k8s + +import ( + "fmt" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/meta" + "inet.af/netaddr" +) + +// EndpointType is type of Endpoint resource. +const EndpointType = resource.Type("Endpoints.kubernetes.talos.dev") + +// ControlPlaneEndpointsID is resource ID for controlplane Endpoint. +const ControlPlaneEndpointsID = resource.ID("controlplane") + +// Endpoint resource holds definition of rendered secrets. +type Endpoint struct { + md resource.Metadata + spec EndpointSpec +} + +// EndpointSpec describes status of rendered secrets. +type EndpointSpec struct { + Addresses []netaddr.IP `yaml:"addresses"` +} + +// NewEndpoint initializes a Endpoint resource. +func NewEndpoint(namespace resource.Namespace, id resource.ID) *Endpoint { + r := &Endpoint{ + md: resource.NewMetadata(namespace, EndpointType, id, resource.VersionUndefined), + spec: EndpointSpec{}, + } + + r.md.BumpVersion() + + return r +} + +// Metadata implements resource.Resource. +func (r *Endpoint) Metadata() *resource.Metadata { + return &r.md +} + +// Spec implements resource.Resource. +func (r *Endpoint) Spec() interface{} { + return r.spec +} + +func (r *Endpoint) String() string { + return fmt.Sprintf("k8s.Endpoint(%q)", r.md.ID()) +} + +// DeepCopy implements resource.Resource. +func (r *Endpoint) DeepCopy() resource.Resource { + return &Endpoint{ + md: r.md, + spec: EndpointSpec{ + Addresses: append([]netaddr.IP(nil), r.spec.Addresses...), + }, + } +} + +// ResourceDefinition implements meta.ResourceDefinitionProvider interface. +func (r *Endpoint) ResourceDefinition() meta.ResourceDefinitionSpec { + return meta.ResourceDefinitionSpec{ + Type: EndpointType, + Aliases: []resource.Type{}, + DefaultNamespace: ControlPlaneNamespaceName, + PrintColumns: []meta.PrintColumn{ + { + Name: "Addresses", + JSONPath: "{.addresses}", + }, + }, + } +} + +// TypedSpec allows to access the Spec with the proper type. +func (r *Endpoint) TypedSpec() *EndpointSpec { + return &r.spec +} diff --git a/pkg/resources/k8s/k8s_test.go b/pkg/resources/k8s/k8s_test.go index 14ff1db35b..96f6ff533c 100644 --- a/pkg/resources/k8s/k8s_test.go +++ b/pkg/resources/k8s/k8s_test.go @@ -25,6 +25,7 @@ func TestRegisterResource(t *testing.T) { resourceRegistry := registry.NewResourceRegistry(resources) for _, resource := range []resource.Resource{ + &k8s.Endpoint{}, &k8s.ManifestStatus{}, &k8s.Manifest{}, &k8s.Nodename{}, diff --git a/pkg/resources/secrets/api.go b/pkg/resources/secrets/api.go new file mode 100644 index 0000000000..022595ef7c --- /dev/null +++ b/pkg/resources/secrets/api.go @@ -0,0 +1,137 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "fmt" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/meta" + "github.com/cosi-project/runtime/pkg/resource/protobuf" + "github.com/talos-systems/crypto/x509" + "google.golang.org/protobuf/proto" + + secretspb "github.com/talos-systems/talos/pkg/machinery/api/resource/secrets" +) + +// APIType is type of API resource. +const APIType = resource.Type("ApiCertificates.secrets.talos.dev") + +// APIID is a resource ID of singleton instance. +const APIID = resource.ID("api") + +// API contains apid generated secrets. +type API struct { + md resource.Metadata + spec *APICertsSpec +} + +// APICertsSpec describes etcd certs secrets. +type APICertsSpec struct { + CA *x509.PEMEncodedCertificateAndKey `yaml:"ca"` // only cert is passed, without key + Client *x509.PEMEncodedCertificateAndKey `yaml:"client"` + Server *x509.PEMEncodedCertificateAndKey `yaml:"server"` +} + +// MarshalProto implements ProtoMarshaler. +func (spec *APICertsSpec) MarshalProto() ([]byte, error) { + protoSpec := secretspb.APISpec{ + CaPem: spec.CA.Crt, + Client: &secretspb.CertAndKeyPEM{ + Cert: spec.Client.Crt, + Key: spec.Client.Key, + }, + Server: &secretspb.CertAndKeyPEM{ + Cert: spec.Server.Crt, + Key: spec.Server.Key, + }, + } + + return proto.Marshal(&protoSpec) +} + +// NewAPI initializes a Etc resource. +func NewAPI() *API { + r := &API{ + md: resource.NewMetadata(NamespaceName, APIType, APIID, resource.VersionUndefined), + spec: &APICertsSpec{}, + } + + r.md.BumpVersion() + + return r +} + +// Metadata implements resource.Resource. +func (r *API) Metadata() *resource.Metadata { + return &r.md +} + +// Spec implements resource.Resource. +func (r *API) Spec() interface{} { + return r.spec +} + +func (r *API) String() string { + return fmt.Sprintf("secrets.APICertificates(%q)", r.md.ID()) +} + +// DeepCopy implements resource.Resource. +func (r *API) DeepCopy() resource.Resource { + specCopy := *r.spec + + return &API{ + md: r.md, + spec: &specCopy, + } +} + +// ResourceDefinition implements meta.ResourceDefinitionProvider interface. +func (r *API) ResourceDefinition() meta.ResourceDefinitionSpec { + return meta.ResourceDefinitionSpec{ + Type: APIType, + Aliases: []resource.Type{}, + DefaultNamespace: NamespaceName, + Sensitivity: meta.Sensitive, + } +} + +// TypedSpec returns .spec. +func (r *API) TypedSpec() *APICertsSpec { + return r.spec +} + +// UnmarshalProto implements protobuf.ResourceUnmarshaler. +func (r *API) UnmarshalProto(md *resource.Metadata, protoBytes []byte) error { + r.md = *md + + protoSpec := secretspb.APISpec{} + + if err := proto.Unmarshal(protoBytes, &protoSpec); err != nil { + return err + } + + r.spec = &APICertsSpec{ + CA: &x509.PEMEncodedCertificateAndKey{ + Crt: protoSpec.CaPem, + }, + Client: &x509.PEMEncodedCertificateAndKey{ + Crt: protoSpec.Client.Cert, + Key: protoSpec.Client.Key, + }, + Server: &x509.PEMEncodedCertificateAndKey{ + Crt: protoSpec.Server.Cert, + Key: protoSpec.Server.Key, + }, + } + + return nil +} + +func init() { + if err := protobuf.RegisterResource(APIType, &API{}); err != nil { + panic(err) + } +} diff --git a/pkg/resources/secrets/api_test.go b/pkg/resources/secrets/api_test.go new file mode 100644 index 0000000000..98bf8f72ce --- /dev/null +++ b/pkg/resources/secrets/api_test.go @@ -0,0 +1,45 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + "testing" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/protobuf" + "github.com/stretchr/testify/require" + "github.com/talos-systems/crypto/x509" + + "github.com/talos-systems/talos/pkg/resources/secrets" +) + +func TestAPIProtobufMarshal(t *testing.T) { + r := secrets.NewAPI() + r.TypedSpec().CA = &x509.PEMEncodedCertificateAndKey{ + Crt: []byte("foo"), + } + r.TypedSpec().Client = &x509.PEMEncodedCertificateAndKey{ + Crt: []byte("bar"), + Key: []byte("baz"), + } + r.TypedSpec().Server = &x509.PEMEncodedCertificateAndKey{ + Crt: []byte("car"), + Key: []byte("caz"), + } + + protoR, err := protobuf.FromResource(r) + require.NoError(t, err) + + marshaled, err := protoR.Marshal() + require.NoError(t, err) + + protoR, err = protobuf.Unmarshal(marshaled) + require.NoError(t, err) + + r2, err := protobuf.UnmarshalResource(protoR) + require.NoError(t, err) + + require.True(t, resource.Equal(r, r2)) +} diff --git a/pkg/resources/secrets/condition.go b/pkg/resources/secrets/condition.go new file mode 100644 index 0000000000..7d94372f69 --- /dev/null +++ b/pkg/resources/secrets/condition.go @@ -0,0 +1,45 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" +) + +// APIReadyCondition implements condition which waits for the API certs to be ready. +type APIReadyCondition struct { + state state.State +} + +// NewAPIReadyCondition builds a coondition which waits for the API certs to be ready. +func NewAPIReadyCondition(state state.State) *APIReadyCondition { + return &APIReadyCondition{ + state: state, + } +} + +func (condition *APIReadyCondition) String() string { + return "api certificates" +} + +// Wait implements condition interface. +func (condition *APIReadyCondition) Wait(ctx context.Context) error { + _, err := condition.state.WatchFor( + ctx, + resource.NewMetadata(NamespaceName, APIType, APIID, resource.VersionUndefined), + state.WithCondition(func(r resource.Resource) (bool, error) { + if resource.IsTombstone(r) { + return false, nil + } + + return true, nil + }), + ) + + return err +} diff --git a/pkg/resources/secrets/root.go b/pkg/resources/secrets/root.go index b32205cc7f..26ecb5690c 100644 --- a/pkg/resources/secrets/root.go +++ b/pkg/resources/secrets/root.go @@ -12,6 +12,7 @@ import ( "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/resource/meta" "github.com/talos-systems/crypto/x509" + "inet.af/netaddr" ) // RootType is type of Root secret resource. @@ -19,6 +20,7 @@ const RootType = resource.Type("RootSecrets.secrets.talos.dev") // IDs of various resources of RootType. const ( + RootOSID = resource.ID("os") RootEtcdID = resource.ID("etcd") RootKubernetesID = resource.ID("k8s") ) @@ -29,6 +31,15 @@ type Root struct { spec interface{} } +// RootOSSpec describes operating system CA. +type RootOSSpec struct { + CA *x509.PEMEncodedCertificateAndKey `yaml:"ca"` + CertSANIPs []netaddr.IP `yaml:"certSANIPs"` + CertSANDNSNames []string `yaml:"certSANDNSNames"` + + Token string `yaml:"token"` +} + // RootEtcdSpec describes etcd CA secrets. type RootEtcdSpec struct { EtcdCA *x509.PEMEncodedCertificateAndKey `yaml:"etcdCA"` @@ -59,6 +70,8 @@ func NewRoot(id resource.ID) *Root { } switch id { + case RootOSID: + r.spec = &RootOSSpec{} case RootEtcdID: r.spec = &RootEtcdSpec{} case RootKubernetesID: @@ -89,6 +102,9 @@ func (r *Root) DeepCopy() resource.Resource { var specCopy interface{} switch v := r.spec.(type) { + case *RootOSSpec: + vv := *v + specCopy = &vv case *RootEtcdSpec: vv := *v specCopy = &vv @@ -115,6 +131,11 @@ func (r *Root) ResourceDefinition() meta.ResourceDefinitionSpec { } } +// OSSpec returns .spec. +func (r *Root) OSSpec() *RootOSSpec { + return r.spec.(*RootOSSpec) +} + // EtcdSpec returns .spec. func (r *Root) EtcdSpec() *RootEtcdSpec { return r.spec.(*RootEtcdSpec) diff --git a/pkg/resources/secrets/secrets_test.go b/pkg/resources/secrets/secrets_test.go index d47b9b6169..7c31b214a2 100644 --- a/pkg/resources/secrets/secrets_test.go +++ b/pkg/resources/secrets/secrets_test.go @@ -25,6 +25,7 @@ func TestRegisterResource(t *testing.T) { resourceRegistry := registry.NewResourceRegistry(resources) for _, resource := range []resource.Resource{ + &secrets.API{}, &secrets.Etcd{}, &secrets.Kubernetes{}, &secrets.Root{},