From b1357bed734c09eaa995817ad77496a4ed1d8ba0 Mon Sep 17 00:00:00 2001 From: huiwq1990 Date: Thu, 2 Jun 2022 01:38:22 +0800 Subject: [PATCH] upgrade kubeadm to k8s 1.22.3 --- .../app/apis/bootstraptoken/v1/types.go | 58 ++ .../v1/utils.go} | 65 ++- .../v1/utils_test.go} | 238 ++++++++- .../v1/zz_generated.deepcopy.go | 65 +++ .../app/apis/kubeadm/bootstraptokenstring.go | 91 ---- .../kubeadm/app/apis/kubeadm/types.go | 462 +++++++++++++++- .../kubeadm/app/cmd/options/constant.go | 102 +++- .../kubeadm/app/cmd/phases/workflow/runner.go | 5 +- .../kubeadm/app/constants/constants.go | 505 +++++++++++++++++- .../kubeadm/app/constants/constants_unix.go | 1 - .../app/constants/constants_windows.go | 1 - .../kubeadm/app/discovery/token/token.go | 8 +- .../kubeadm/app/features/features.go | 181 +++++++ .../kubeadm/app/features/features_test.go | 219 ++++++++ .../bootstraptoken/clusterinfo/clusterinfo.go | 8 +- .../app/phases/bootstraptoken/node/token.go | 10 +- .../kubeadm/app/phases/kubelet/flags.go | 9 +- .../kubeadm/app/phases/kubelet/flags_test.go | 92 ---- .../kubeadm/app/phases/kubelet/flags_unix.go | 60 --- .../app/phases/kubelet/flags_windows.go | 28 - .../kubeadm/app/preflight/checks.go | 154 +++++- .../kubeadm/app/util/apiclient/idempotency.go | 1 - .../app/util/apiclient/tryidempotency.go | 232 -------- .../kubeadm/app/util/apiclient/wait.go | 1 - .../kubeadm/app/util/cgroupdriver.go | 53 -- .../app/util/initsystem/initsystem_unix.go | 3 +- .../app/util/initsystem/initsystem_windows.go | 31 +- .../kubeadm/app/util/kubeconfig/kubeconfig.go | 8 +- .../app/util/kubeconfig/kubeconfig_test.go | 8 +- .../kubeadm/app/util/pubkeypin/pubkeypin.go | 16 +- .../app/util/pubkeypin/pubkeypin_test.go | 11 + .../kubeadm/app/util/runtime/runtime.go | 1 - .../kubeadm/app/util/runtime/runtime_test.go | 6 +- .../kubeadm/app/util/runtime/runtime_unix.go | 1 - .../app/util/runtime/runtime_windows.go | 1 - pkg/yurtadm/cmd/join/join.go | 12 +- pkg/yurtadm/cmd/join/phases/preflight.go | 26 +- pkg/yurtadm/constants/join_options.go | 38 ++ pkg/yurtctl/util/kubernetes/util.go | 10 +- 39 files changed, 2121 insertions(+), 700 deletions(-) create mode 100644 pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/types.go rename pkg/util/kubernetes/kubeadm/app/apis/{kubeadm/bootstraptokenhelpers.go => bootstraptoken/v1/utils.go} (69%) rename pkg/util/kubernetes/kubeadm/app/apis/{kubeadm/bootstraptokenhelpers_test.go => bootstraptoken/v1/utils_test.go} (58%) create mode 100644 pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/zz_generated.deepcopy.go delete mode 100644 pkg/util/kubernetes/kubeadm/app/apis/kubeadm/bootstraptokenstring.go create mode 100644 pkg/util/kubernetes/kubeadm/app/features/features.go create mode 100644 pkg/util/kubernetes/kubeadm/app/features/features_test.go delete mode 100644 pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_test.go delete mode 100644 pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_unix.go delete mode 100644 pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_windows.go delete mode 100644 pkg/util/kubernetes/kubeadm/app/util/apiclient/tryidempotency.go delete mode 100644 pkg/util/kubernetes/kubeadm/app/util/cgroupdriver.go create mode 100644 pkg/yurtadm/constants/join_options.go diff --git a/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/types.go b/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/types.go new file mode 100644 index 00000000000..8ebaf77f3ba --- /dev/null +++ b/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/types.go @@ -0,0 +1,58 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// BootstrapToken describes one bootstrap token, stored as a Secret in the cluster +// +k8s:deepcopy-gen=true +type BootstrapToken struct { + // Token is used for establishing bidirectional trust between nodes and control-planes. + // Used for joining nodes in the cluster. + Token *BootstrapTokenString `json:"token" datapolicy:"token"` + // Description sets a human-friendly message why this token exists and what it's used + // for, so other administrators can know its purpose. + // +optional + Description string `json:"description,omitempty"` + // TTL defines the time to live for this token. Defaults to 24h. + // Expires and TTL are mutually exclusive. + // +optional + TTL *metav1.Duration `json:"ttl,omitempty"` + // Expires specifies the timestamp when this token expires. Defaults to being set + // dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive. + // +optional + Expires *metav1.Time `json:"expires,omitempty"` + // Usages describes the ways in which this token can be used. Can by default be used + // for establishing bidirectional trust, but that can be changed here. + // +optional + Usages []string `json:"usages,omitempty"` + // Groups specifies the extra groups that this token will authenticate as when/if + // used for authentication + // +optional + Groups []string `json:"groups,omitempty"` +} + +// BootstrapTokenString is a token of the format abcdef.abcdef0123456789 that is used +// for both validation of the practically of the API server from a joining node's point +// of view and as an authentication method for the node in the bootstrap phase of +// "kubeadm join". This token is and should be short-lived +type BootstrapTokenString struct { + ID string `json:"-"` + Secret string `json:"-" datapolicy:"token"` +} diff --git a/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/bootstraptokenhelpers.go b/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/utils.go similarity index 69% rename from pkg/util/kubernetes/kubeadm/app/apis/kubeadm/bootstraptokenhelpers.go rename to pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/utils.go index 119549c7bb6..58ce744e18f 100644 --- a/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/bootstraptokenhelpers.go +++ b/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/utils.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadm +package v1 import ( + "fmt" "sort" "strings" "time" @@ -29,9 +30,61 @@ import ( bootstrapsecretutil "k8s.io/cluster-bootstrap/util/secrets" ) -// ToSecret converts the given BootstrapToken object to its Secret representation that +// MarshalJSON implements the json.Marshaler interface. +func (bts BootstrapTokenString) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"%s"`, bts.String())), nil +} + +// UnmarshalJSON implements the json.Unmarshaller interface. +func (bts *BootstrapTokenString) UnmarshalJSON(b []byte) error { + // If the token is represented as "", just return quickly without an error + if len(b) == 0 { + return nil + } + + // Remove unnecessary " characters coming from the JSON parser + token := strings.Replace(string(b), `"`, ``, -1) + // Convert the string Token to a BootstrapTokenString object + newbts, err := NewBootstrapTokenString(token) + if err != nil { + return err + } + bts.ID = newbts.ID + bts.Secret = newbts.Secret + return nil +} + +// String returns the string representation of the BootstrapTokenString +func (bts BootstrapTokenString) String() string { + if len(bts.ID) > 0 && len(bts.Secret) > 0 { + return bootstraputil.TokenFromIDAndSecret(bts.ID, bts.Secret) + } + return "" +} + +// NewBootstrapTokenString converts the given Bootstrap Token as a string +// to the BootstrapTokenString object used for serialization/deserialization +// and internal usage. It also automatically validates that the given token +// is of the right format +func NewBootstrapTokenString(token string) (*BootstrapTokenString, error) { + substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token) + // TODO: Add a constant for the 3 value here, and explain better why it's needed (other than because how the regexp parsin works) + if len(substrs) != 3 { + return nil, errors.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern) + } + + return &BootstrapTokenString{ID: substrs[1], Secret: substrs[2]}, nil +} + +// NewBootstrapTokenStringFromIDAndSecret is a wrapper around NewBootstrapTokenString +// that allows the caller to specify the ID and Secret separately +func NewBootstrapTokenStringFromIDAndSecret(id, secret string) (*BootstrapTokenString, error) { + return NewBootstrapTokenString(bootstraputil.TokenFromIDAndSecret(id, secret)) +} + +// BootstrapTokenToSecret converts the given BootstrapToken object to its Secret representation that // may be submitted to the API Server in order to be stored. -func (bt *BootstrapToken) ToSecret() *v1.Secret { +func BootstrapTokenToSecret(bt *BootstrapToken) *v1.Secret { return &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: bootstraputil.BootstrapTokenSecretName(bt.Token.ID), @@ -60,13 +113,13 @@ func encodeTokenSecretData(token *BootstrapToken, now time.Time) map[string][]by if token.Expires != nil { // Format the expiration date accordingly // TODO: This maybe should be a helper function in bootstraputil? - expirationString := token.Expires.Time.Format(time.RFC3339) + expirationString := token.Expires.Time.UTC().Format(time.RFC3339) data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) } else if token.TTL != nil && token.TTL.Duration > 0 { // Only if .Expires is unset, TTL might have an effect // Get the current time, add the specified duration, and format it accordingly - expirationString := now.Add(token.TTL.Duration).Format(time.RFC3339) + expirationString := now.Add(token.TTL.Duration).UTC().Format(time.RFC3339) data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) } diff --git a/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/bootstraptokenhelpers_test.go b/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/utils_test.go similarity index 58% rename from pkg/util/kubernetes/kubeadm/app/apis/kubeadm/bootstraptokenhelpers_test.go rename to pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/utils_test.go index 9d1d97ccdbf..9a12ded26c5 100644 --- a/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/bootstraptokenhelpers_test.go +++ b/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/utils_test.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadm +package v1 import ( "encoding/json" @@ -22,15 +22,239 @@ import ( "testing" "time" + "github.com/pkg/errors" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func TestMarshalJSON(t *testing.T) { + var tests = []struct { + bts BootstrapTokenString + expected string + }{ + {BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, `"abcdef.abcdef0123456789"`}, + {BootstrapTokenString{ID: "foo", Secret: "bar"}, `"foo.bar"`}, + {BootstrapTokenString{ID: "h", Secret: "b"}, `"h.b"`}, + } + for _, rt := range tests { + t.Run(rt.bts.ID, func(t *testing.T) { + b, err := json.Marshal(rt.bts) + if err != nil { + t.Fatalf("json.Marshal returned an unexpected error: %v", err) + } + if string(b) != rt.expected { + t.Errorf( + "failed BootstrapTokenString.MarshalJSON:\n\texpected: %s\n\t actual: %s", + rt.expected, + string(b), + ) + } + }) + } +} + +func TestUnmarshalJSON(t *testing.T) { + var tests = []struct { + input string + bts *BootstrapTokenString + expectedError bool + }{ + {`"f.s"`, &BootstrapTokenString{}, true}, + {`"abcdef."`, &BootstrapTokenString{}, true}, + {`"abcdef:abcdef0123456789"`, &BootstrapTokenString{}, true}, + {`abcdef.abcdef0123456789`, &BootstrapTokenString{}, true}, + {`"abcdef.abcdef0123456789`, &BootstrapTokenString{}, true}, + {`"abcdef.ABCDEF0123456789"`, &BootstrapTokenString{}, true}, + {`"abcdef.abcdef0123456789"`, &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, false}, + {`"123456.aabbccddeeffgghh"`, &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}, false}, + } + for _, rt := range tests { + t.Run(rt.input, func(t *testing.T) { + newbts := &BootstrapTokenString{} + err := json.Unmarshal([]byte(rt.input), newbts) + if (err != nil) != rt.expectedError { + t.Errorf("failed BootstrapTokenString.UnmarshalJSON:\n\texpected error: %t\n\t actual error: %v", rt.expectedError, err) + } else if !reflect.DeepEqual(rt.bts, newbts) { + t.Errorf( + "failed BootstrapTokenString.UnmarshalJSON:\n\texpected: %v\n\t actual: %v", + rt.bts, + newbts, + ) + } + }) + } +} + +func TestJSONRoundtrip(t *testing.T) { + var tests = []struct { + input string + bts *BootstrapTokenString + }{ + {`"abcdef.abcdef0123456789"`, nil}, + {"", &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, + } + for _, rt := range tests { + t.Run(rt.input, func(t *testing.T) { + if err := roundtrip(rt.input, rt.bts); err != nil { + t.Errorf("failed BootstrapTokenString JSON roundtrip with error: %v", err) + } + }) + } +} + +func roundtrip(input string, bts *BootstrapTokenString) error { + var b []byte + var err error + newbts := &BootstrapTokenString{} + // If string input was specified, roundtrip like this: string -> (unmarshal) -> object -> (marshal) -> string + if len(input) > 0 { + if err := json.Unmarshal([]byte(input), newbts); err != nil { + return errors.Wrap(err, "expected no unmarshal error, got error") + } + if b, err = json.Marshal(newbts); err != nil { + return errors.Wrap(err, "expected no marshal error, got error") + } + if input != string(b) { + return errors.Errorf( + "expected token: %s\n\t actual: %s", + input, + string(b), + ) + } + } else { // Otherwise, roundtrip like this: object -> (marshal) -> string -> (unmarshal) -> object + if b, err = json.Marshal(bts); err != nil { + return errors.Wrap(err, "expected no marshal error, got error") + } + if err := json.Unmarshal(b, newbts); err != nil { + return errors.Wrap(err, "expected no unmarshal error, got error") + } + if !reflect.DeepEqual(bts, newbts) { + return errors.Errorf( + "expected object: %v\n\t actual: %v", + bts, + newbts, + ) + } + } + return nil +} + +func TestTokenFromIDAndSecret(t *testing.T) { + var tests = []struct { + bts BootstrapTokenString + expected string + }{ + {BootstrapTokenString{ID: "foo", Secret: "bar"}, "foo.bar"}, + {BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, "abcdef.abcdef0123456789"}, + {BootstrapTokenString{ID: "h", Secret: "b"}, "h.b"}, + } + for _, rt := range tests { + t.Run(rt.bts.ID, func(t *testing.T) { + actual := rt.bts.String() + if actual != rt.expected { + t.Errorf( + "failed BootstrapTokenString.String():\n\texpected: %s\n\t actual: %s", + rt.expected, + actual, + ) + } + }) + } +} + +func TestNewBootstrapTokenString(t *testing.T) { + var tests = []struct { + token string + expectedError bool + bts *BootstrapTokenString + }{ + {token: "", expectedError: true, bts: nil}, + {token: ".", expectedError: true, bts: nil}, + {token: "1234567890123456789012", expectedError: true, bts: nil}, // invalid parcel size + {token: "12345.1234567890123456", expectedError: true, bts: nil}, // invalid parcel size + {token: ".1234567890123456", expectedError: true, bts: nil}, // invalid parcel size + {token: "123456.", expectedError: true, bts: nil}, // invalid parcel size + {token: "123456:1234567890.123456", expectedError: true, bts: nil}, // invalid separation + {token: "abcdef:1234567890123456", expectedError: true, bts: nil}, // invalid separation + {token: "Abcdef.1234567890123456", expectedError: true, bts: nil}, // invalid token id + {token: "123456.AABBCCDDEEFFGGHH", expectedError: true, bts: nil}, // invalid token secret + {token: "123456.AABBCCD-EEFFGGHH", expectedError: true, bts: nil}, // invalid character + {token: "abc*ef.1234567890123456", expectedError: true, bts: nil}, // invalid character + {token: "abcdef.1234567890123456", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "1234567890123456"}}, + {token: "123456.aabbccddeeffgghh", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}}, + {token: "abcdef.abcdef0123456789", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, + {token: "123456.1234560123456789", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "1234560123456789"}}, + } + for _, rt := range tests { + t.Run(rt.token, func(t *testing.T) { + actual, err := NewBootstrapTokenString(rt.token) + if (err != nil) != rt.expectedError { + t.Errorf( + "failed NewBootstrapTokenString for the token %q\n\texpected error: %t\n\t actual error: %v", + rt.token, + rt.expectedError, + err, + ) + } else if !reflect.DeepEqual(actual, rt.bts) { + t.Errorf( + "failed NewBootstrapTokenString for the token %q\n\texpected: %v\n\t actual: %v", + rt.token, + rt.bts, + actual, + ) + } + }) + } +} + +func TestNewBootstrapTokenStringFromIDAndSecret(t *testing.T) { + var tests = []struct { + id, secret string + expectedError bool + bts *BootstrapTokenString + }{ + {id: "", secret: "", expectedError: true, bts: nil}, + {id: "1234567890123456789012", secret: "", expectedError: true, bts: nil}, // invalid parcel size + {id: "12345", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid parcel size + {id: "", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid parcel size + {id: "123456", secret: "", expectedError: true, bts: nil}, // invalid parcel size + {id: "Abcdef", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid token id + {id: "123456", secret: "AABBCCDDEEFFGGHH", expectedError: true, bts: nil}, // invalid token secret + {id: "123456", secret: "AABBCCD-EEFFGGHH", expectedError: true, bts: nil}, // invalid character + {id: "abc*ef", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid character + {id: "abcdef", secret: "1234567890123456", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "1234567890123456"}}, + {id: "123456", secret: "aabbccddeeffgghh", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}}, + {id: "abcdef", secret: "abcdef0123456789", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, + {id: "123456", secret: "1234560123456789", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "1234560123456789"}}, + } + for _, rt := range tests { + t.Run(rt.id, func(t *testing.T) { + actual, err := NewBootstrapTokenStringFromIDAndSecret(rt.id, rt.secret) + if (err != nil) != rt.expectedError { + t.Errorf( + "failed NewBootstrapTokenStringFromIDAndSecret for the token with id %q and secret %q\n\texpected error: %t\n\t actual error: %v", + rt.id, + rt.secret, + rt.expectedError, + err, + ) + } else if !reflect.DeepEqual(actual, rt.bts) { + t.Errorf( + "failed NewBootstrapTokenStringFromIDAndSecret for the token with id %q and secret %q\n\texpected: %v\n\t actual: %v", + rt.id, + rt.secret, + rt.bts, + actual, + ) + } + }) + } +} + // This timestamp is used as the reference value when computing expiration dates based on TTLs in these unit tests var refTime = time.Date(1970, time.January, 1, 1, 1, 1, 0, time.UTC) -func TestToSecret(t *testing.T) { - +func TestBootstrapTokenToSecret(t *testing.T) { var tests = []struct { bt *BootstrapToken secret *v1.Secret @@ -65,10 +289,10 @@ func TestToSecret(t *testing.T) { } for _, rt := range tests { t.Run(rt.bt.Token.ID, func(t *testing.T) { - actual := rt.bt.ToSecret() + actual := BootstrapTokenToSecret(rt.bt) if !reflect.DeepEqual(actual, rt.secret) { t.Errorf( - "failed BootstrapToken.ToSecret():\n\texpected: %v\n\t actual: %v", + "failed BootstrapTokenToSecret():\n\texpected: %v\n\t actual: %v", rt.secret, actual, ) @@ -95,7 +319,7 @@ func TestBootstrapTokenToSecretRoundtrip(t *testing.T) { } for _, rt := range tests { t.Run(rt.bt.Token.ID, func(t *testing.T) { - actual, err := BootstrapTokenFromSecret(rt.bt.ToSecret()) + actual, err := BootstrapTokenFromSecret(BootstrapTokenToSecret(rt.bt)) if err != nil { t.Errorf("failed BootstrapToken to Secret roundtrip with error: %v", err) } diff --git a/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/zz_generated.deepcopy.go b/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/zz_generated.deepcopy.go new file mode 100644 index 00000000000..60e9998c35d --- /dev/null +++ b/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1/zz_generated.deepcopy.go @@ -0,0 +1,65 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BootstrapToken) DeepCopyInto(out *BootstrapToken) { + *out = *in + if in.Token != nil { + in, out := &in.Token, &out.Token + *out = new(BootstrapTokenString) + **out = **in + } + if in.TTL != nil { + in, out := &in.TTL, &out.TTL + *out = new(metav1.Duration) + **out = **in + } + if in.Expires != nil { + in, out := &in.Expires, &out.Expires + *out = (*in).DeepCopy() + } + if in.Usages != nil { + in, out := &in.Usages, &out.Usages + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Groups != nil { + in, out := &in.Groups, &out.Groups + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapToken. +func (in *BootstrapToken) DeepCopy() *BootstrapToken { + if in == nil { + return nil + } + out := new(BootstrapToken) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/bootstraptokenstring.go b/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/bootstraptokenstring.go deleted file mode 100644 index 282d305afb8..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/bootstraptokenstring.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package kubeadm holds the internal kubeadm API types -// Note: This file should be kept in sync with the similar one for the external API -// TODO: The BootstrapTokenString object should move out to either k8s.io/client-go or k8s.io/api in the future -// (probably as part of Bootstrap Tokens going GA). It should not be staged under the kubeadm API as it is now. -package kubeadm - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - bootstraputil "k8s.io/cluster-bootstrap/token/util" -) - -// BootstrapTokenString is a token of the format abcdef.abcdef0123456789 that is used -// for both validation of the practically of the API server from a joining node's point -// of view and as an authentication method for the node in the bootstrap phase of -// "kubeadm join". This token is and should be short-lived -type BootstrapTokenString struct { - ID string - Secret string -} - -// MarshalJSON implements the json.Marshaler interface. -func (bts BootstrapTokenString) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf(`"%s"`, bts.String())), nil -} - -// UnmarshalJSON implements the json.Unmarshaller interface. -func (bts *BootstrapTokenString) UnmarshalJSON(b []byte) error { - // If the token is represented as "", just return quickly without an error - if len(b) == 0 { - return nil - } - - // Remove unnecessary " characters coming from the JSON parser - token := strings.Replace(string(b), `"`, ``, -1) - // Convert the string Token to a BootstrapTokenString object - newbts, err := NewBootstrapTokenString(token) - if err != nil { - return err - } - bts.ID = newbts.ID - bts.Secret = newbts.Secret - return nil -} - -// String returns the string representation of the BootstrapTokenString -func (bts BootstrapTokenString) String() string { - if len(bts.ID) > 0 && len(bts.Secret) > 0 { - return bootstraputil.TokenFromIDAndSecret(bts.ID, bts.Secret) - } - return "" -} - -// NewBootstrapTokenString converts the given Bootstrap Token as a string -// to the BootstrapTokenString object used for serialization/deserialization -// and internal usage. It also automatically validates that the given token -// is of the right format -func NewBootstrapTokenString(token string) (*BootstrapTokenString, error) { - substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token) - // TODO: Add a constant for the 3 value here, and explain better why it's needed (other than because how the regexp parsin works) - if len(substrs) != 3 { - return nil, errors.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern) - } - - return &BootstrapTokenString{ID: substrs[1], Secret: substrs[2]}, nil -} - -// NewBootstrapTokenStringFromIDAndSecret is a wrapper around NewBootstrapTokenString -// that allows the caller to specify the ID and Secret separately -func NewBootstrapTokenStringFromIDAndSecret(id, secret string) (*BootstrapTokenString, error) { - return NewBootstrapTokenString(bootstraputil.TokenFromIDAndSecret(id, secret)) -} diff --git a/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/types.go b/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/types.go index ee0c547ad2b..af4a448aa5b 100644 --- a/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/types.go +++ b/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/types.go @@ -1,6 +1,5 @@ /* Copyright 2016 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,31 +17,448 @@ limitations under the License. package kubeadm import ( + "crypto/x509" + + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + bootstraptokenv1 "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1" + "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/features" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// BootstrapToken describes one bootstrap token, stored as a Secret in the cluster -// TODO: The BootstrapToken object should move out to either k8s.io/client-go or k8s.io/api in the future -// (probably as part of Bootstrap Tokens going GA). It should not be staged under the kubeadm API as it is now. -type BootstrapToken struct { - // Token is used for establishing bidirectional trust between nodes and control-planes. - // Used for joining nodes in the cluster. - Token *BootstrapTokenString - // Description sets a human-friendly message why this token exists and what it's used - // for, so other administrators can know its purpose. - Description string - // TTL defines the time to live for this token. Defaults to 24h. - // Expires and TTL are mutually exclusive. - TTL *metav1.Duration - // Expires specifies the timestamp when this token expires. Defaults to being set - // dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive. - Expires *metav1.Time - // Usages describes the ways in which this token can be used. Can by default be used - // for establishing bidirectional trust, but that can be changed here. - Usages []string - // Groups specifies the extra groups that this token will authenticate as when/if - // used for authentication - Groups []string +// InitConfiguration contains a list of fields that are specifically "kubeadm init"-only runtime +// information. The cluster-wide config is stored in ClusterConfiguration. The InitConfiguration +// object IS NOT uploaded to the kubeadm-config ConfigMap in the cluster, only the +// ClusterConfiguration is. +type InitConfiguration struct { + metav1.TypeMeta + + // ClusterConfiguration holds the cluster-wide information, and embeds that struct (which can be (un)marshalled separately as well) + // When InitConfiguration is marshalled to bytes in the external version, this information IS NOT preserved (which can be seen from + // the `json:"-"` tag in the external variant of these API types. + ClusterConfiguration `json:"-"` + + // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. + BootstrapTokens []bootstraptokenv1.BootstrapToken + + // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster + NodeRegistration NodeRegistrationOptions + + // LocalAPIEndpoint represents the endpoint of the API server instance that's deployed on this control plane node + // In HA setups, this differs from ClusterConfiguration.ControlPlaneEndpoint in the sense that ControlPlaneEndpoint + // is the global endpoint for the cluster, which then loadbalances the requests to each individual API server. This + // configuration object lets you customize what IP/DNS name and port the local API server advertises it's accessible + // on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process + // fails you may set the desired value here. + LocalAPIEndpoint APIEndpoint + + // CertificateKey sets the key with which certificates and keys are encrypted prior to being uploaded in + // a secret in the cluster during the uploadcerts init phase. + CertificateKey string + + // SkipPhases is a list of phases to skip during command execution. + // The list of phases can be obtained with the "kubeadm init --help" command. + // The flag "--skip-phases" takes precedence over this field. + SkipPhases []string + + // Patches contains options related to applying patches to components deployed by kubeadm during + // "kubeadm init". + Patches *Patches +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ClusterConfiguration contains cluster-wide configuration for a kubeadm cluster +type ClusterConfiguration struct { + metav1.TypeMeta + + // ComponentConfigs holds component configs known to kubeadm, should long-term only exist in the internal kubeadm API + // +k8s:conversion-gen=false + ComponentConfigs ComponentConfigMap + + // Etcd holds configuration for etcd. + Etcd Etcd + + // Networking holds configuration for the networking topology of the cluster. + Networking Networking + // KubernetesVersion is the target version of the control plane. + KubernetesVersion string + + // ControlPlaneEndpoint sets a stable IP address or DNS name for the control plane; it + // can be a valid IP address or a RFC-1123 DNS subdomain, both with optional TCP port. + // In case the ControlPlaneEndpoint is not specified, the AdvertiseAddress + BindPort + // are used; in case the ControlPlaneEndpoint is specified but without a TCP port, + // the BindPort is used. + // Possible usages are: + // e.g. In a cluster with more than one control plane instances, this field should be + // assigned the address of the external load balancer in front of the + // control plane instances. + // e.g. in environments with enforced node recycling, the ControlPlaneEndpoint + // could be used for assigning a stable DNS to the control plane. + ControlPlaneEndpoint string + + // APIServer contains extra settings for the API server control plane component + APIServer APIServer + + // ControllerManager contains extra settings for the controller manager control plane component + ControllerManager ControlPlaneComponent + + // Scheduler contains extra settings for the scheduler control plane component + Scheduler ControlPlaneComponent + + // DNS defines the options for the DNS add-on installed in the cluster. + DNS DNS + + // CertificatesDir specifies where to store or look for all required certificates. + CertificatesDir string + + // ImageRepository sets the container registry to pull images from. + // If empty, `k8s.gcr.io` will be used by default; in case of kubernetes version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`) + // `gcr.io/k8s-staging-ci-images` will be used as a default for control plane components and for kube-proxy, while `k8s.gcr.io` + // will be used for all the other images. + ImageRepository string + + // CIImageRepository is the container registry for core images generated by CI. + // Useful for running kubeadm with images from CI builds. + // +k8s:conversion-gen=false + CIImageRepository string + + // FeatureGates enabled by the user. + FeatureGates map[string]bool + + // The cluster name + ClusterName string +} + +// ControlPlaneComponent holds settings common to control plane component of the cluster +type ControlPlaneComponent struct { + // ExtraArgs is an extra set of flags to pass to the control plane component. + // A key in this map is the flag name as it appears on the + // command line except without leading dash(es). + // TODO: This is temporary and ideally we would like to switch all components to + // use ComponentConfig + ConfigMaps. + ExtraArgs map[string]string + + // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. + ExtraVolumes []HostPathMount } + +// APIServer holds settings necessary for API server deployments in the cluster +type APIServer struct { + ControlPlaneComponent + + // CertSANs sets extra Subject Alternative Names for the API Server signing cert. + CertSANs []string + + // TimeoutForControlPlane controls the timeout that we use for API server to appear + TimeoutForControlPlane *metav1.Duration +} + +// DNSAddOnType defines string identifying DNS add-on types +// TODO: Remove with v1beta2 https://github.com/kubernetes/kubeadm/issues/2459 +type DNSAddOnType string + +const ( + // CoreDNS add-on type + // TODO: Remove with v1beta2 https://github.com/kubernetes/kubeadm/issues/2459 + CoreDNS DNSAddOnType = "CoreDNS" +) + +// DNS defines the DNS addon that should be used in the cluster +type DNS struct { + // Type defines the DNS add-on to be used + // TODO: Used only in validation over the internal type. Remove with v1beta2 https://github.com/kubernetes/kubeadm/issues/2459 + Type DNSAddOnType + + // ImageMeta allows to customize the image used for the DNS component + ImageMeta `json:",inline"` +} + +// ImageMeta allows to customize the image used for components that are not +// originated from the Kubernetes/Kubernetes release process +type ImageMeta struct { + // ImageRepository sets the container registry to pull images from. + // if not set, the ImageRepository defined in ClusterConfiguration will be used instead. + ImageRepository string + + // ImageTag allows to specify a tag for the image. + // In case this value is set, kubeadm does not change automatically the version of the above components during upgrades. + ImageTag string + + //TODO: evaluate if we need also a ImageName based on user feedbacks +} + +// APIEndpoint struct contains elements of API server instance deployed on a node. +type APIEndpoint struct { + // AdvertiseAddress sets the IP address for the API server to advertise. + AdvertiseAddress string + + // BindPort sets the secure port for the API Server to bind to. + // Defaults to 6443. + BindPort int32 +} + +// NodeRegistrationOptions holds fields that relate to registering a new control-plane or node to the cluster, either via "kubeadm init" or "kubeadm join" +type NodeRegistrationOptions struct { + + // Name is the `.Metadata.Name` field of the Node API object that will be created in this `kubeadm init` or `kubeadm join` operation. + // This field is also used in the CommonName field of the kubelet's client certificate to the API server. + // Defaults to the hostname of the node if not provided. + Name string + + // CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use + CRISocket string + + // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process + // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your control-plane node, set this field to an + // empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. + Taints []v1.Taint + + // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file + // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap + // Flags have higher priority when parsing. These values are local and specific to the node kubeadm is executing on. + // A key in this map is the flag name as it appears on the + // command line except without leading dash(es). + KubeletExtraArgs map[string]string + + // IgnorePreflightErrors provides a slice of pre-flight errors to be ignored when the current node is registered. + IgnorePreflightErrors []string + + // ImagePullPolicy specifies the policy for image pulling during kubeadm "init" and "join" operations. + // The value of this field must be one of "Always", "IfNotPresent" or "Never". + // If this field is unset kubeadm will default it to "IfNotPresent", or pull the required images if not present on the host. + ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"` +} + +// Networking contains elements describing cluster's networking configuration. +type Networking struct { + // ServiceSubnet is the subnet used by k8s services. Defaults to "10.96.0.0/12". + ServiceSubnet string + // PodSubnet is the subnet used by pods. + PodSubnet string + // DNSDomain is the dns domain used by k8s services. Defaults to "cluster.local". + DNSDomain string +} + +// Etcd contains elements describing Etcd configuration. +type Etcd struct { + + // Local provides configuration knobs for configuring the local etcd instance + // Local and External are mutually exclusive + Local *LocalEtcd + + // External describes how to connect to an external etcd cluster + // Local and External are mutually exclusive + External *ExternalEtcd +} + +// LocalEtcd describes that kubeadm should run an etcd cluster locally +type LocalEtcd struct { + // ImageMeta allows to customize the container used for etcd + ImageMeta `json:",inline"` + + // DataDir is the directory etcd will place its data. + // Defaults to "/var/lib/etcd". + DataDir string + + // ExtraArgs are extra arguments provided to the etcd binary + // when run inside a static pod. + // A key in this map is the flag name as it appears on the + // command line except without leading dash(es). + ExtraArgs map[string]string + + // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. + ServerCertSANs []string + // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. + PeerCertSANs []string +} + +// ExternalEtcd describes an external etcd cluster +type ExternalEtcd struct { + + // Endpoints of etcd members. Useful for using external etcd. + // If not provided, kubeadm will run etcd in a static pod. + Endpoints []string + // CAFile is an SSL Certificate Authority file used to secure etcd communication. + CAFile string + // CertFile is an SSL certification file used to secure etcd communication. + CertFile string + // KeyFile is an SSL key file used to secure etcd communication. + KeyFile string +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// JoinConfiguration contains elements describing a particular node. +type JoinConfiguration struct { + metav1.TypeMeta + + // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster + NodeRegistration NodeRegistrationOptions + + // CACertPath is the path to the SSL certificate authority used to + // secure comunications between node and control-plane. + // Defaults to "/etc/kubernetes/pki/ca.crt". + CACertPath string + + // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process + Discovery Discovery + + // ControlPlane defines the additional control plane instance to be deployed on the joining node. + // If nil, no additional control plane instance will be deployed. + ControlPlane *JoinControlPlane + + // SkipPhases is a list of phases to skip during command execution. + // The list of phases can be obtained with the "kubeadm join --help" command. + // The flag "--skip-phases" takes precedence over this field. + SkipPhases []string + + // Patches contains options related to applying patches to components deployed by kubeadm during + // "kubeadm join". + Patches *Patches +} + +// JoinControlPlane contains elements describing an additional control plane instance to be deployed on the joining node. +type JoinControlPlane struct { + // LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node. + LocalAPIEndpoint APIEndpoint + + // CertificateKey is the key that is used for decryption of certificates after they are downloaded from the secret + // upon joining a new control plane node. The corresponding encryption key is in the InitConfiguration. + CertificateKey string +} + +// Discovery specifies the options for the kubelet to use during the TLS Bootstrap process +type Discovery struct { + // BootstrapToken is used to set the options for bootstrap token based discovery + // BootstrapToken and File are mutually exclusive + BootstrapToken *BootstrapTokenDiscovery + + // File is used to specify a file or URL to a kubeconfig file from which to load cluster information + // BootstrapToken and File are mutually exclusive + File *FileDiscovery + + // TLSBootstrapToken is a token used for TLS bootstrapping. + // If .BootstrapToken is set, this field is defaulted to .BootstrapToken.Token, but can be overridden. + // If .File is set, this field **must be set** in case the KubeConfigFile does not contain any other authentication information + TLSBootstrapToken string + + // Timeout modifies the discovery timeout + Timeout *metav1.Duration +} + +// BootstrapTokenDiscovery is used to set the options for bootstrap token based discovery +type BootstrapTokenDiscovery struct { + // Token is a token used to validate cluster information + // fetched from the control-plane. + Token string + + // APIServerEndpoint is an IP or domain name to the API server from which info will be fetched. + APIServerEndpoint string + + // CACertHashes specifies a set of public key pins to verify + // when token-based discovery is used. The root CA found during discovery + // must match one of these values. Specifying an empty set disables root CA + // pinning, which can be unsafe. Each hash is specified as ":", + // where the only currently supported type is "sha256". This is a hex-encoded + // SHA-256 hash of the Subject Public Key Info (SPKI) object in DER-encoded + // ASN.1. These hashes can be calculated using, for example, OpenSSL. + CACertHashes []string + + // UnsafeSkipCAVerification allows token-based discovery + // without CA verification via CACertHashes. This can weaken + // the security of kubeadm since other nodes can impersonate the control-plane. + UnsafeSkipCAVerification bool +} + +// FileDiscovery is used to specify a file or URL to a kubeconfig file from which to load cluster information +type FileDiscovery struct { + // KubeConfigPath is used to specify the actual file path or URL to the kubeconfig file from which to load cluster information + KubeConfigPath string +} + +// GetControlPlaneImageRepository returns name of image repository +// for control plane images (API,Controller Manager,Scheduler and Proxy) +// It will override location with CI registry name in case user requests special +// Kubernetes version from CI build area. +// (See: kubeadmconstants.DefaultCIImageRepository) +func (cfg *ClusterConfiguration) GetControlPlaneImageRepository() string { + if cfg.CIImageRepository != "" { + return cfg.CIImageRepository + } + return cfg.ImageRepository +} + +// PublicKeyAlgorithm returns the type of encryption keys used in the cluster. +func (cfg *ClusterConfiguration) PublicKeyAlgorithm() x509.PublicKeyAlgorithm { + if features.Enabled(cfg.FeatureGates, features.PublicKeysECDSA) { + return x509.ECDSA + } + + return x509.RSA +} + +// HostPathMount contains elements describing volumes that are mounted from the +// host. +type HostPathMount struct { + // Name of the volume inside the pod template. + Name string + // HostPath is the path in the host that will be mounted inside + // the pod. + HostPath string + // MountPath is the path inside the pod where hostPath will be mounted. + MountPath string + // ReadOnly controls write access to the volume + ReadOnly bool + // PathType is the type of the HostPath. + PathType v1.HostPathType +} + +// Patches contains options related to applying patches to components deployed by kubeadm. +type Patches struct { + // Directory is a path to a directory that contains files named "target[suffix][+patchtype].extension". + // For example, "kube-apiserver0+merge.yaml" or just "etcd.json". "target" can be one of + // "kube-apiserver", "kube-controller-manager", "kube-scheduler", "etcd". "patchtype" can be one + // of "strategic" "merge" or "json" and they match the patch formats supported by kubectl. + // The default "patchtype" is "strategic". "extension" must be either "json" or "yaml". + // "suffix" is an optional string that can be used to determine which patches are applied + // first alpha-numerically. + Directory string +} + +// DocumentMap is a convenient way to describe a map between a YAML document and its GVK type +// +k8s:deepcopy-gen=false +type DocumentMap map[schema.GroupVersionKind][]byte + +// ComponentConfig holds a known component config +type ComponentConfig interface { + // DeepCopy should create a new deep copy of the component config in place + DeepCopy() ComponentConfig + + // Marshal is marshalling the config into a YAML document returned as a byte slice + Marshal() ([]byte, error) + + // Unmarshal loads the config from a document map. No config in the document map is no error. + Unmarshal(docmap DocumentMap) error + + // Default patches the component config with kubeadm preferred defaults + Default(cfg *ClusterConfiguration, localAPIEndpoint *APIEndpoint, nodeRegOpts *NodeRegistrationOptions) + + // IsUserSupplied indicates if the component config was supplied or modified by a user or was kubeadm generated + IsUserSupplied() bool + + // SetUserSupplied sets the state of the component config "user supplied" flag to, either true, or false. + SetUserSupplied(userSupplied bool) + + // Set can be used to set the internal configuration in the ComponentConfig + Set(interface{}) + + // Get can be used to get the internal configuration in the ComponentConfig + Get() interface{} +} + +// ComponentConfigMap is a map between a group name (as in GVK group) and a ComponentConfig +type ComponentConfigMap map[string]ComponentConfig diff --git a/pkg/util/kubernetes/kubeadm/app/cmd/options/constant.go b/pkg/util/kubernetes/kubeadm/app/cmd/options/constant.go index 31cf0cefddf..dba685a7c62 100644 --- a/pkg/util/kubernetes/kubeadm/app/cmd/options/constant.go +++ b/pkg/util/kubernetes/kubeadm/app/cmd/options/constant.go @@ -1,6 +1,5 @@ /* Copyright 2019 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,57 +23,130 @@ const ( // APIServerBindPort flag sets the port for the API Server to bind to. APIServerBindPort = "apiserver-bind-port" + // APIServerCertSANs flag sets extra Subject Alternative Names (SANs) to use for the API Server serving certificate. Can be both IP addresses and DNS names. + APIServerCertSANs = "apiserver-cert-extra-sans" + + // APIServerExtraArgs flag sets a extra flags to pass to the API Server or override default ones in form of =. + APIServerExtraArgs = "apiserver-extra-args" + // CertificatesDir flag sets the path where to save and read the certificates. CertificatesDir = "cert-dir" // CfgPath flag sets the path to kubeadm config file. CfgPath = "config" + // ControllerManagerExtraArgs flag sets extra flags to pass to the Controller Manager or override default ones in form of =. + ControllerManagerExtraArgs = "controller-manager-extra-args" + + // ControlPlaneEndpoint flag sets a stable IP address or DNS name for the control plane. + ControlPlaneEndpoint = "control-plane-endpoint" + // DryRun flag instruct kubeadm to don't apply any changes; just output what would be done. DryRun = "dry-run" + // FeatureGatesString flag sets key=value pairs that describe feature gates for various features. + FeatureGatesString = "feature-gates" + // IgnorePreflightErrors sets the path a list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks. IgnorePreflightErrors = "ignore-preflight-errors" + // ImageRepository sets the container registry to pull control plane images from. + ImageRepository = "image-repository" + + // KubeconfigDir flag sets the path where to save the kubeconfig file. + KubeconfigDir = "kubeconfig-dir" + // KubeconfigPath flag sets the kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations are searched for an existing KubeConfig file. KubeconfigPath = "kubeconfig" // KubernetesVersion flag sets the Kubernetes version for the control plane. KubernetesVersion = "kubernetes-version" + // KubeletVersion flag sets the version for the kubelet config. + KubeletVersion = "kubelet-version" + + // NetworkingDNSDomain flag sets the domain for services, e.g. "myorg.internal". + NetworkingDNSDomain = "service-dns-domain" + + // NetworkingServiceSubnet flag sets the range of IP address for service VIPs. + NetworkingServiceSubnet = "service-cidr" + + // NetworkingPodSubnet flag sets the range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node. + NetworkingPodSubnet = "pod-network-cidr" + // NodeCRISocket flag sets the CRI socket to connect to. NodeCRISocket = "cri-socket" // NodeName flag sets the node name. NodeName = "node-name" + // SchedulerExtraArgs flag sets extra flags to pass to the Scheduler or override default ones in form of =". + SchedulerExtraArgs = "scheduler-extra-args" + + // SkipTokenPrint flag instruct kubeadm to skip printing of the default bootstrap token generated by 'kubeadm init'. + SkipTokenPrint = "skip-token-print" + + // CSROnly flag instructs kubeadm to create CSRs instead of automatically creating or renewing certs + CSROnly = "csr-only" + + // CSRDir flag sets the location for CSRs and flags to be output + CSRDir = "csr-dir" + // TokenStr flags sets both the discovery-token and the tls-bootstrap-token when those values are not provided TokenStr = "token" + // TokenTTL flag sets the time to live for token + TokenTTL = "token-ttl" + + // TokenUsages flag sets the usages of the token + TokenUsages = "usages" + + // TokenGroups flag sets the authentication groups of the token + TokenGroups = "groups" + + // TokenDescription flag sets the description of the token + TokenDescription = "description" + + // TLSBootstrapToken flag sets the token used to temporarily authenticate with the Kubernetes Control Plane to submit a certificate signing request (CSR) for a locally created key pair + TLSBootstrapToken = "tls-bootstrap-token" + + // TokenDiscovery flag sets the token used to validate cluster information fetched from the API server (for token-based discovery) + TokenDiscovery = "discovery-token" + // TokenDiscoveryCAHash flag instruct kubeadm to validate that the root CA public key matches this hash (for token-based discovery) TokenDiscoveryCAHash = "discovery-token-ca-cert-hash" // TokenDiscoverySkipCAHash flag instruct kubeadm to skip CA hash verification (for token-based discovery) TokenDiscoverySkipCAHash = "discovery-token-unsafe-skip-ca-verification" - // ForceReset flag instruct kubeadm to reset the node without prompting for confirmation - ForceReset = "force" + // FileDiscovery flag sets the file or URL from which to load cluster information (for file-based discovery) + FileDiscovery = "discovery-file" + + // ControlPlane flag instruct kubeadm to create a new control plane instance on this node + ControlPlane = "control-plane" + + // UploadCerts flag instruct kubeadm to upload certificates + UploadCerts = "upload-certs" - // NodeType flag sets the type of worker node to edge or cloud. - NodeType = "node-type" + // CertificateKey flag sets the key used to encrypt and decrypt certificate secrets + CertificateKey = "certificate-key" - // Organizations flag sets the extra organizations of hub agent client certificate. - Organizations = "organizations" + // SkipCertificateKeyPrint flag instruct kubeadm to skip printing certificate key used to encrypt certs by 'kubeadm init'. + SkipCertificateKeyPrint = "skip-certificate-key-print" + + // ForceReset flag instruct kubeadm to reset the node without prompting for confirmation + ForceReset = "force" - // NodeLabels flag sets the labels for worker node. - NodeLabels = "node-labels" + // CertificateRenewal flag instruct kubeadm to execute certificate renewal during upgrades + CertificateRenewal = "certificate-renewal" - // PauseImage flag sets the pause image for worker node. - PauseImage = "pause-image" + // EtcdUpgrade flag instruct kubeadm to execute etcd upgrade during upgrades + EtcdUpgrade = "etcd-upgrade" - // YurtHubImage flag sets the yurthub image for worker node. - YurtHubImage = "yurthub-image" + // Patches flag sets the folder where kubeadm component patches are stored + Patches = "patches" - // KubernetesResourceServer flag sets the address for download k8s node resources. - KubernetesResourceServer = "kubernetes-resource-server" + // ExperimentalPatches (DEPRECATED) is the same as Patches + // TODO: https://github.com/kubernetes/kubeadm/issues/2046 remove in 1.23 + ExperimentalPatches = "experimental-patches" ) diff --git a/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/runner.go b/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/runner.go index 60cd89e980a..32e44e5f1d6 100644 --- a/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/runner.go +++ b/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/runner.go @@ -213,7 +213,7 @@ func (e *Runner) Run(args []string) error { // Errors if phases that are meant to create special subcommands only // are wrongly assigned Run Methods if p.RunAllSiblings && (p.RunIf != nil || p.Run != nil) { - return errors.Wrapf(err, "phase marked as RunAllSiblings can not have Run functions %s", p.generatedName) + return errors.Errorf("phase marked as RunAllSiblings can not have Run functions %s", p.generatedName) } // If the phase defines a condition to be checked before executing the phase action. @@ -338,8 +338,7 @@ func (e *Runner) BindToCommand(cmd *cobra.Command) { RunE: func(cmd *cobra.Command, args []string) error { // if the phase has subphases, print the help and exits if len(p.Phases) > 0 { - cmd.Help() - return nil + return cmd.Help() } // overrides the command triggering the Runner using the phaseCmd diff --git a/pkg/util/kubernetes/kubeadm/app/constants/constants.go b/pkg/util/kubernetes/kubeadm/app/constants/constants.go index 3e357f8bbe5..12c40d6a6e3 100644 --- a/pkg/util/kubernetes/kubeadm/app/constants/constants.go +++ b/pkg/util/kubernetes/kubeadm/app/constants/constants.go @@ -18,11 +18,20 @@ package constants import ( "fmt" + "io/ioutil" + "net" + "os" + "path" "path/filepath" + "strings" "time" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/util/wait" bootstrapapi "k8s.io/cluster-bootstrap/token/api" + utilnet "k8s.io/utils/net" ) const ( @@ -30,14 +39,113 @@ const ( KubernetesDir = "/etc/kubernetes" // ManifestsSubDirName defines directory name to store manifests ManifestsSubDirName = "manifests" + // TempDirForKubeadm defines temporary directory for kubeadm + // should be joined with KubernetesDir. + TempDirForKubeadm = "tmp" + + // CertificateValidity defines the validity for all the signed certificates generated by kubeadm + CertificateValidity = time.Hour * 24 * 365 // CACertAndKeyBaseName defines certificate authority base name CACertAndKeyBaseName = "ca" // CACertName defines certificate name CACertName = "ca.crt" + // CAKeyName defines certificate name + CAKeyName = "ca.key" + + // APIServerCertAndKeyBaseName defines API's server certificate and key base name + APIServerCertAndKeyBaseName = "apiserver" + // APIServerCertName defines API's server certificate name + APIServerCertName = "apiserver.crt" + // APIServerKeyName defines API's server key name + APIServerKeyName = "apiserver.key" + // APIServerCertCommonName defines API's server certificate common name (CN) + APIServerCertCommonName = "kube-apiserver" + + // APIServerKubeletClientCertAndKeyBaseName defines kubelet client certificate and key base name + APIServerKubeletClientCertAndKeyBaseName = "apiserver-kubelet-client" + // APIServerKubeletClientCertName defines kubelet client certificate name + APIServerKubeletClientCertName = "apiserver-kubelet-client.crt" + // APIServerKubeletClientKeyName defines kubelet client key name + APIServerKubeletClientKeyName = "apiserver-kubelet-client.key" + // APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN) + APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client" + + // EtcdCACertAndKeyBaseName defines etcd's CA certificate and key base name + EtcdCACertAndKeyBaseName = "etcd/ca" + // EtcdCACertName defines etcd's CA certificate name + EtcdCACertName = "etcd/ca.crt" + // EtcdCAKeyName defines etcd's CA key name + EtcdCAKeyName = "etcd/ca.key" + + // EtcdServerCertAndKeyBaseName defines etcd's server certificate and key base name + EtcdServerCertAndKeyBaseName = "etcd/server" + // EtcdServerCertName defines etcd's server certificate name + EtcdServerCertName = "etcd/server.crt" + // EtcdServerKeyName defines etcd's server key name + EtcdServerKeyName = "etcd/server.key" + + // EtcdListenClientPort defines the port etcd listen on for client traffic + EtcdListenClientPort = 2379 + // EtcdMetricsPort is the port at which to obtain etcd metrics and health status + EtcdMetricsPort = 2381 + + // EtcdPeerCertAndKeyBaseName defines etcd's peer certificate and key base name + EtcdPeerCertAndKeyBaseName = "etcd/peer" + // EtcdPeerCertName defines etcd's peer certificate name + EtcdPeerCertName = "etcd/peer.crt" + // EtcdPeerKeyName defines etcd's peer key name + EtcdPeerKeyName = "etcd/peer.key" + + // EtcdListenPeerPort defines the port etcd listen on for peer traffic + EtcdListenPeerPort = 2380 + + // EtcdHealthcheckClientCertAndKeyBaseName defines etcd's healthcheck client certificate and key base name + EtcdHealthcheckClientCertAndKeyBaseName = "etcd/healthcheck-client" + // EtcdHealthcheckClientCertName defines etcd's healthcheck client certificate name + EtcdHealthcheckClientCertName = "etcd/healthcheck-client.crt" + // EtcdHealthcheckClientKeyName defines etcd's healthcheck client key name + EtcdHealthcheckClientKeyName = "etcd/healthcheck-client.key" + // EtcdHealthcheckClientCertCommonName defines etcd's healthcheck client certificate common name (CN) + EtcdHealthcheckClientCertCommonName = "kube-etcd-healthcheck-client" + + // APIServerEtcdClientCertAndKeyBaseName defines apiserver's etcd client certificate and key base name + APIServerEtcdClientCertAndKeyBaseName = "apiserver-etcd-client" + // APIServerEtcdClientCertName defines apiserver's etcd client certificate name + APIServerEtcdClientCertName = "apiserver-etcd-client.crt" + // APIServerEtcdClientKeyName defines apiserver's etcd client key name + APIServerEtcdClientKeyName = "apiserver-etcd-client.key" + // APIServerEtcdClientCertCommonName defines apiserver's etcd client certificate common name (CN) + APIServerEtcdClientCertCommonName = "kube-apiserver-etcd-client" + + // ServiceAccountKeyBaseName defines SA key base name + ServiceAccountKeyBaseName = "sa" + // ServiceAccountPublicKeyName defines SA public key base name + ServiceAccountPublicKeyName = "sa.pub" + // ServiceAccountPrivateKeyName defines SA private key base name + ServiceAccountPrivateKeyName = "sa.key" + + // FrontProxyCACertAndKeyBaseName defines front proxy CA certificate and key base name + FrontProxyCACertAndKeyBaseName = "front-proxy-ca" + // FrontProxyCACertName defines front proxy CA certificate name + FrontProxyCACertName = "front-proxy-ca.crt" + // FrontProxyCAKeyName defines front proxy CA key name + FrontProxyCAKeyName = "front-proxy-ca.key" + + // FrontProxyClientCertAndKeyBaseName defines front proxy certificate and key base name + FrontProxyClientCertAndKeyBaseName = "front-proxy-client" + // FrontProxyClientCertName defines front proxy certificate name + FrontProxyClientCertName = "front-proxy-client.crt" + // FrontProxyClientKeyName defines front proxy key name + FrontProxyClientKeyName = "front-proxy-client.key" + // FrontProxyClientCertCommonName defines front proxy certificate common name + FrontProxyClientCertCommonName = "front-proxy-client" //used as subject.commonname attribute (CN) // AdminKubeConfigFileName defines name for the kubeconfig aimed to be used by the superuser/admin of the cluster AdminKubeConfigFileName = "admin.conf" + // KubeletBootstrapKubeConfigFileName defines the file name for the kubeconfig that the kubelet will use to do + // the TLS bootstrap to get itself an unique credential + KubeletBootstrapKubeConfigFileName = "bootstrap-kubelet.conf" // KubeletKubeConfigFileName defines the file name for the kubeconfig that the control-plane kubelet will use for talking // to the API server @@ -49,6 +157,22 @@ const ( // Some well-known users and groups in the core Kubernetes authorization system + // ControllerManagerUser defines the well-known user the controller-manager should be authenticated as + ControllerManagerUser = "system:kube-controller-manager" + // SchedulerUser defines the well-known user the scheduler should be authenticated as + SchedulerUser = "system:kube-scheduler" + // SystemPrivilegedGroup defines the well-known group for the apiservers. This group is also superuser by default + // (i.e. bound to the cluster-admin ClusterRole) + SystemPrivilegedGroup = "system:masters" + // NodesGroup defines the well-known group for all nodes. + NodesGroup = "system:nodes" + // NodesUserPrefix defines the user name prefix as requested by the Node authorizer. + NodesUserPrefix = "system:node:" + // NodesClusterRoleBinding defines the well-known ClusterRoleBinding which binds the too permissive system:node + // ClusterRole to the system:nodes group. Since kubeadm is using the Node Authorizer, this ClusterRoleBinding's + // system:nodes group subject is removed if present. + NodesClusterRoleBinding = "system:node" + // APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation APICallRetryInterval = 500 * time.Millisecond // DiscoveryRetryInterval specifies how long kubeadm should wait before retrying to connect to the control-plane when doing discovery @@ -66,22 +190,78 @@ const ( // PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed PullImageRetry = 5 + // DefaultControlPlaneTimeout specifies the default control plane (actually API Server) timeout for use by kubeadm + DefaultControlPlaneTimeout = 4 * time.Minute + + // MinimumAddressesInServiceSubnet defines minimum amount of nodes the Service subnet should allow. + // We need at least ten, because the DNS service is always at the tenth cluster clusterIP + MinimumAddressesInServiceSubnet = 10 + + // MaximumBitsForServiceSubnet defines maximum possible size of the service subnet in terms of bits. + // For example, if the value is 20, then the largest supported service subnet is /12 for IPv4 and /108 for IPv6. + // Note however that anything in between /108 and /112 will be clamped to /112 due to the limitations of the underlying allocation logic. + // TODO: https://github.com/kubernetes/enhancements/pull/1881 + MaximumBitsForServiceSubnet = 20 + + // MinimumAddressesInPodSubnet defines minimum amount of pods in the cluster. + // We need at least more than services, an IPv4 /28 or IPv6 /128 subnet means 14 util addresses + MinimumAddressesInPodSubnet = 14 + + // PodSubnetNodeMaskMaxDiff is limited to 16 due to an issue with uncompressed IP bitmap in core: + // xref: #44918 + // The node subnet mask size must be no more than the pod subnet mask size + 16 + PodSubnetNodeMaskMaxDiff = 16 + + // DefaultTokenDuration specifies the default amount of time that a bootstrap token will be valid + // Default behaviour is 24 hours + DefaultTokenDuration = 24 * time.Hour + + // DefaultCertTokenDuration specifies the default amount of time that the token used by upload certs will be valid + // Default behaviour is 2 hours + DefaultCertTokenDuration = 2 * time.Hour + + // CertificateKeySize specifies the size of the key used to encrypt certificates on uploadcerts phase + CertificateKeySize = 32 + + // LabelNodeRoleOldControlPlane specifies that a node hosts control-plane components + // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 + LabelNodeRoleOldControlPlane = "node-role.kubernetes.io/master" + + // LabelNodeRoleControlPlane specifies that a node hosts control-plane components + LabelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane" + + // LabelExcludeFromExternalLB can be set on a node to exclude it from external load balancers. + // This is added to control plane nodes to preserve backwards compatibility with a legacy behavior. + LabelExcludeFromExternalLB = "node.kubernetes.io/exclude-from-external-load-balancers" + // AnnotationKubeadmCRISocket specifies the annotation kubeadm uses to preserve the crisocket information given to kubeadm at // init/join time for use later. kubeadm annotates the node object with this information AnnotationKubeadmCRISocket = "kubeadm.alpha.kubernetes.io/cri-socket" + // UnknownCRISocket defines the undetected or unknown CRI socket + UnknownCRISocket = "/var/run/unknown.sock" + // KubeadmConfigConfigMap specifies in what ConfigMap in the kube-system namespace the `kubeadm init` configuration should be stored KubeadmConfigConfigMap = "kubeadm-config" // ClusterConfigurationConfigMapKey specifies in what ConfigMap key the cluster configuration should be stored ClusterConfigurationConfigMapKey = "ClusterConfiguration" + // KubeProxyConfigMap specifies in what ConfigMap in the kube-system namespace the kube-proxy configuration should be stored + KubeProxyConfigMap = "kube-proxy" + + // KubeProxyConfigMapKey specifies in what ConfigMap key the component config of kube-proxy should be stored + KubeProxyConfigMapKey = "config.conf" + // KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-" // KubeletBaseConfigurationConfigMapKey specifies in what ConfigMap key the initial remote configuration of kubelet should be stored KubeletBaseConfigurationConfigMapKey = "kubelet" + // KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap. + KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-" + // KubeletRunDirectory specifies the directory where the kubelet runtime information is stored. KubeletRunDirectory = "/var/lib/kubelet" @@ -101,22 +281,162 @@ const ( // KubeletHealthzPort is the port of the kubelet healthz endpoint KubeletHealthzPort = 10248 - // NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in - NodeBootstrapTokenAuthGroup = "system:bootstrappers:kubeadm:default-node-token" + // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports + MinExternalEtcdVersion = "3.2.18" + // DefaultEtcdVersion indicates the default etcd version that kubeadm uses + DefaultEtcdVersion = "3.5.0-0" + + // Etcd defines variable used internally when referring to etcd component + Etcd = "etcd" // KubeAPIServer defines variable used internally when referring to kube-apiserver component KubeAPIServer = "kube-apiserver" // KubeControllerManager defines variable used internally when referring to kube-controller-manager component KubeControllerManager = "kube-controller-manager" // KubeScheduler defines variable used internally when referring to kube-scheduler component KubeScheduler = "kube-scheduler" + // KubeProxy defines variable used internally when referring to kube-proxy component + KubeProxy = "kube-proxy" + // CoreDNS defines variable used internally when referring to the CoreDNS component + CoreDNS = "CoreDNS" + // Kubelet defines variable used internally when referring to the Kubelet + Kubelet = "kubelet" + + // KubeCertificatesVolumeName specifies the name for the Volume that is used for injecting certificates to control plane components (can be both a hostPath volume or a projected, all-in-one volume) + KubeCertificatesVolumeName = "k8s-certs" + + // KubeConfigVolumeName specifies the name for the Volume that is used for injecting the kubeconfig to talk securely to the api server for a control plane component if applicable + KubeConfigVolumeName = "kubeconfig" + + // NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in + NodeBootstrapTokenAuthGroup = "system:bootstrappers:kubeadm:default-node-token" + + // DefaultCIImageRepository points to image registry where CI uploads images from ci-cross build job + DefaultCIImageRepository = "gcr.io/k8s-staging-ci-images" + + // CoreDNSConfigMap specifies in what ConfigMap in the kube-system namespace the CoreDNS config should be stored + CoreDNSConfigMap = "coredns" + + // CoreDNSDeploymentName specifies the name of the Deployment for CoreDNS add-on + CoreDNSDeploymentName = "coredns" + + // CoreDNSImageName specifies the name of the image for CoreDNS add-on + CoreDNSImageName = "coredns" + + // CoreDNSVersion is the version of CoreDNS to be deployed if it is used + CoreDNSVersion = "v1.8.4" + + // ClusterConfigurationKind is the string kind value for the ClusterConfiguration struct + ClusterConfigurationKind = "ClusterConfiguration" + + // InitConfigurationKind is the string kind value for the InitConfiguration struct + InitConfigurationKind = "InitConfiguration" + + // JoinConfigurationKind is the string kind value for the JoinConfiguration struct + JoinConfigurationKind = "JoinConfiguration" + + // YAMLDocumentSeparator is the separator for YAML documents + // TODO: Find a better place for this constant + YAMLDocumentSeparator = "---\n" + + // DefaultAPIServerBindAddress is the default bind address for the API Server + DefaultAPIServerBindAddress = "0.0.0.0" + + // ControlPlaneNumCPU is the number of CPUs required on control-plane + ControlPlaneNumCPU = 2 + + // ControlPlaneMem is the number of megabytes of memory required on the control-plane + // Below that amount of RAM running a stable control plane would be difficult. + ControlPlaneMem = 1700 + + // KubeadmCertsSecret specifies in what Secret in the kube-system namespace the certificates should be stored + KubeadmCertsSecret = "kubeadm-certs" // KubeletPort is the default port for the kubelet server on each host machine. // May be overridden by a flag at startup. KubeletPort = 10250 + // KubeSchedulerPort is the default port for the scheduler status server. + // May be overridden by a flag at startup. + KubeSchedulerPort = 10259 + // KubeControllerManagerPort is the default port for the controller manager status server. + // May be overridden by a flag at startup. + KubeControllerManagerPort = 10257 + + // EtcdAdvertiseClientUrlsAnnotationKey is the annotation key on every etcd pod, describing the + // advertise client URLs + EtcdAdvertiseClientUrlsAnnotationKey = "kubeadm.kubernetes.io/etcd.advertise-client-urls" + // KubeAPIServerAdvertiseAddressEndpointAnnotationKey is the annotation key on every apiserver pod, + // describing the API endpoint (advertise address and bind port of the api server) + KubeAPIServerAdvertiseAddressEndpointAnnotationKey = "kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint" + // ComponentConfigHashAnnotationKey holds the config map annotation key that kubeadm uses to store + // a SHA256 sum to check for user changes + ComponentConfigHashAnnotationKey = "kubeadm.kubernetes.io/component-config.hash" + + // ControlPlaneTier is the value used in the tier label to identify control plane components + ControlPlaneTier = "control-plane" + + // Mode* constants were copied from pkg/kubeapiserver/authorizer/modes + // to avoid kubeadm dependency on the internal module + // TODO: share Mode* constants in component config + + // ModeAlwaysAllow is the mode to set all requests as authorized + ModeAlwaysAllow string = "AlwaysAllow" + // ModeAlwaysDeny is the mode to set no requests as authorized + ModeAlwaysDeny string = "AlwaysDeny" + // ModeABAC is the mode to use Attribute Based Access Control to authorize + ModeABAC string = "ABAC" + // ModeWebhook is the mode to make an external webhook call to authorize + ModeWebhook string = "Webhook" + // ModeRBAC is the mode to use Role Based Access Control to authorize + ModeRBAC string = "RBAC" + // ModeNode is an authorization mode that authorizes API requests made by kubelets. + ModeNode string = "Node" + + // PauseVersion indicates the default pause image version for kubeadm + PauseVersion = "3.5" + + // CgroupDriverSystemd holds the systemd driver type + CgroupDriverSystemd = "systemd" + + // The username of the user that kube-controller-manager runs as. + KubeControllerManagerUserName string = "kubeadm-kcm" + // The username of the user that kube-apiserver runs as. + KubeAPIServerUserName string = "kubeadm-kas" + // The username of the user that kube-scheduler runs as. + KubeSchedulerUserName string = "kubeadm-ks" + // The username of the user that etcd runs as. + EtcdUserName string = "kubeadm-etcd" + // The group of users that are allowed to read the service account private key. + ServiceAccountKeyReadersGroupName string = "kubeadm-sa-key-readers" ) var ( + // OldControlPlaneTaint is the taint to apply on the PodSpec for being able to run that Pod on the control-plane + // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 + OldControlPlaneTaint = v1.Taint{ + Key: LabelNodeRoleOldControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + + // OldControlPlaneToleration is the toleration to apply on the PodSpec for being able to run that Pod on the control-plane + // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 + OldControlPlaneToleration = v1.Toleration{ + Key: LabelNodeRoleOldControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + + // ControlPlaneTaint is the taint to apply on the PodSpec for being able to run that Pod on the control-plane + ControlPlaneTaint = v1.Taint{ + Key: LabelNodeRoleControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + + // ControlPlaneToleration is the toleration to apply on the PodSpec for being able to run that Pod on the control-plane + ControlPlaneToleration = v1.Toleration{ + Key: LabelNodeRoleControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + // DefaultTokenUsages specifies the default functions a token will get DefaultTokenUsages = bootstrapapi.KnownTokenUsages @@ -126,10 +446,86 @@ var ( // ControlPlaneComponents defines the control-plane component names ControlPlaneComponents = []string{KubeAPIServer, KubeControllerManager, KubeScheduler} + // MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy + MinimumControlPlaneVersion = version.MustParseSemantic("v1.21.0") + // MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports - MinimumKubeletVersion = version.MustParseSemantic("v1.17.0") + MinimumKubeletVersion = version.MustParseSemantic("v1.21.0") + + // CurrentKubernetesVersion specifies current Kubernetes version supported by kubeadm + CurrentKubernetesVersion = version.MustParseSemantic("v1.22.0") + + // SupportedEtcdVersion lists officially supported etcd versions with corresponding Kubernetes releases + SupportedEtcdVersion = map[uint8]string{ + 13: "3.2.24", + 14: "3.3.10", + 15: "3.3.10", + 16: "3.3.17-0", + 17: "3.4.3-0", + 18: "3.4.3-0", + 19: "3.4.13-0", + 20: "3.4.13-0", + 21: "3.4.13-0", + 22: "3.5.0-0", + 23: "3.5.0-0", + } + + // KubeadmCertsClusterRoleName sets the name for the ClusterRole that allows + // the bootstrap tokens to access the kubeadm-certs Secret during the join of a new control-plane + KubeadmCertsClusterRoleName = fmt.Sprintf("kubeadm:%s", KubeadmCertsSecret) + + // StaticPodMirroringDefaultRetry is used a backoff strategy for + // waiting for static pods to be mirrored to the apiserver. + StaticPodMirroringDefaultRetry = wait.Backoff{ + Steps: 30, + Duration: 1 * time.Second, + Factor: 1.0, + Jitter: 0.1, + } ) +// EtcdSupportedVersion returns officially supported version of etcd for a specific Kubernetes release +// If passed version is not in the given list, the function returns the nearest version with a warning +func EtcdSupportedVersion(supportedEtcdVersion map[uint8]string, versionString string) (etcdVersion *version.Version, warning, err error) { + kubernetesVersion, err := version.ParseSemantic(versionString) + if err != nil { + return nil, nil, err + } + desiredVersion, etcdStringVersion := uint8(kubernetesVersion.Minor()), "" + + min, max := ^uint8(0), uint8(0) + for k, v := range supportedEtcdVersion { + if desiredVersion == k { + etcdStringVersion = v + break + } + if k < min { + min = k + } + if k > max { + max = k + } + } + + if len(etcdStringVersion) == 0 { + if desiredVersion < min { + etcdStringVersion = supportedEtcdVersion[min] + } + if desiredVersion > max { + etcdStringVersion = supportedEtcdVersion[max] + } + warning = fmt.Errorf("could not find officially supported version of etcd for Kubernetes %s, falling back to the nearest etcd version (%s)", + versionString, etcdStringVersion) + } + + etcdVersion, err = version.ParseSemantic(etcdStringVersion) + if err != nil { + return nil, nil, err + } + + return etcdVersion, warning, nil +} + // GetStaticPodDirectory returns the location on the disk where the Static Pod should be present func GetStaticPodDirectory() string { return filepath.Join(KubernetesDir, ManifestsSubDirName) @@ -145,6 +541,109 @@ func GetAdminKubeConfigPath() string { return filepath.Join(KubernetesDir, AdminKubeConfigFileName) } +// GetBootstrapKubeletKubeConfigPath returns the location on the disk where bootstrap kubelet kubeconfig is located by default +func GetBootstrapKubeletKubeConfigPath() string { + return filepath.Join(KubernetesDir, KubeletBootstrapKubeConfigFileName) +} + +// GetKubeletKubeConfigPath returns the location on the disk where kubelet kubeconfig is located by default +func GetKubeletKubeConfigPath() string { + return filepath.Join(KubernetesDir, KubeletKubeConfigFileName) +} + +// CreateTempDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp (not using /tmp as that would potentially be dangerous) +func CreateTempDirForKubeadm(kubernetesDir, dirName string) (string, error) { + tempDir := path.Join(KubernetesDir, TempDirForKubeadm) + if len(kubernetesDir) != 0 { + tempDir = path.Join(kubernetesDir, TempDirForKubeadm) + } + + // creates target folder if not already exists + if err := os.MkdirAll(tempDir, 0700); err != nil { + return "", errors.Wrapf(err, "failed to create directory %q", tempDir) + } + + tempDir, err := ioutil.TempDir(tempDir, dirName) + if err != nil { + return "", errors.Wrap(err, "couldn't create a temporary directory") + } + return tempDir, nil +} + +// CreateTimestampDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp formatted with the current date +func CreateTimestampDirForKubeadm(kubernetesDir, dirName string) (string, error) { + tempDir := path.Join(KubernetesDir, TempDirForKubeadm) + if len(kubernetesDir) != 0 { + tempDir = path.Join(kubernetesDir, TempDirForKubeadm) + } + + // creates target folder if not already exists + if err := os.MkdirAll(tempDir, 0700); err != nil { + return "", errors.Wrapf(err, "failed to create directory %q", tempDir) + } + + timestampDirName := fmt.Sprintf("%s-%s", dirName, time.Now().Format("2006-01-02-15-04-05")) + timestampDir := path.Join(tempDir, timestampDirName) + if err := os.Mkdir(timestampDir, 0700); err != nil { + return "", errors.Wrap(err, "could not create timestamp directory") + } + + return timestampDir, nil +} + +// GetDNSIP returns a dnsIP, which is 10th IP in svcSubnet CIDR range +func GetDNSIP(svcSubnetList string, isDualStack bool) (net.IP, error) { + // Get the service subnet CIDR + svcSubnetCIDR, err := GetKubernetesServiceCIDR(svcSubnetList, isDualStack) + if err != nil { + return nil, errors.Wrapf(err, "unable to get internal Kubernetes Service IP from the given service CIDR (%s)", svcSubnetList) + } + + // Selects the 10th IP in service subnet CIDR range as dnsIP + dnsIP, err := utilnet.GetIndexedIP(svcSubnetCIDR, 10) + if err != nil { + return nil, errors.Wrap(err, "unable to get internal Kubernetes Service IP from the given service CIDR") + } + + return dnsIP, nil +} + +// GetKubernetesServiceCIDR returns the default Service CIDR for the Kubernetes internal service +func GetKubernetesServiceCIDR(svcSubnetList string, isDualStack bool) (*net.IPNet, error) { + if isDualStack { + // The default service address family for the cluster is the address family of the first + // service cluster IP range configured via the `--service-cluster-ip-range` flag + // of the kube-controller-manager and kube-apiserver. + svcSubnets, err := utilnet.ParseCIDRs(strings.Split(svcSubnetList, ",")) + if err != nil { + return nil, errors.Wrapf(err, "unable to parse ServiceSubnet %v", svcSubnetList) + } + if len(svcSubnets) == 0 { + return nil, errors.New("received empty ServiceSubnet for dual-stack") + } + return svcSubnets[0], nil + } + // internal IP address for the API server + _, svcSubnet, err := net.ParseCIDR(svcSubnetList) + if err != nil { + return nil, errors.Wrapf(err, "unable to parse ServiceSubnet %v", svcSubnetList) + } + return svcSubnet, nil +} + +// GetAPIServerVirtualIP returns the IP of the internal Kubernetes API service +func GetAPIServerVirtualIP(svcSubnetList string, isDualStack bool) (net.IP, error) { + svcSubnet, err := GetKubernetesServiceCIDR(svcSubnetList, isDualStack) + if err != nil { + return nil, errors.Wrap(err, "unable to get internal Kubernetes Service IP from the given service CIDR") + } + internalAPIServerVirtualIP, err := utilnet.GetIndexedIP(svcSubnet, 1) + if err != nil { + return nil, errors.Wrapf(err, "unable to get the first IP address from the given CIDR: %s", svcSubnet.String()) + } + return internalAPIServerVirtualIP, nil +} + // GetKubeletConfigMapName returns the right ConfigMap name for the right branch of k8s func GetKubeletConfigMapName(k8sVersion *version.Version) string { return fmt.Sprintf("%s%d.%d", KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor()) diff --git a/pkg/util/kubernetes/kubeadm/app/constants/constants_unix.go b/pkg/util/kubernetes/kubeadm/app/constants/constants_unix.go index 812faf064b5..16ff72d5556 100644 --- a/pkg/util/kubernetes/kubeadm/app/constants/constants_unix.go +++ b/pkg/util/kubernetes/kubeadm/app/constants/constants_unix.go @@ -1,4 +1,3 @@ -//go:build !windows // +build !windows /* diff --git a/pkg/util/kubernetes/kubeadm/app/constants/constants_windows.go b/pkg/util/kubernetes/kubeadm/app/constants/constants_windows.go index 1a44a82723d..6daae0a1fff 100644 --- a/pkg/util/kubernetes/kubeadm/app/constants/constants_windows.go +++ b/pkg/util/kubernetes/kubeadm/app/constants/constants_windows.go @@ -1,4 +1,3 @@ -//go:build windows // +build windows /* diff --git a/pkg/util/kubernetes/kubeadm/app/discovery/token/token.go b/pkg/util/kubernetes/kubeadm/app/discovery/token/token.go index f2246d89754..8389d98b03d 100644 --- a/pkg/util/kubernetes/kubeadm/app/discovery/token/token.go +++ b/pkg/util/kubernetes/kubeadm/app/discovery/token/token.go @@ -35,7 +35,7 @@ import ( bootstrap "k8s.io/cluster-bootstrap/token/jws" "k8s.io/klog/v2" - kubeadmapi "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/apis/kubeadm" + bootstraptokenv1 "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1" "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/constants" kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/kubeconfig" "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/pubkeypin" @@ -68,7 +68,7 @@ func RetrieveBootstrapConfig(data joindata.YurtJoinData) (*clientcmdapi.Config, // retrieveValidatedConfigInfo is a private implementation of RetrieveValidatedConfigInfo. // It accepts an optional clientset that can be used for testing purposes. func retrieveValidatedConfigInfo(client clientset.Interface, data joindata.YurtJoinData) (*clientcmdapi.Config, error) { - token, err := kubeadmapi.NewBootstrapTokenString(data.JoinToken()) + token, err := bootstraptokenv1.NewBootstrapTokenString(data.JoinToken()) if err != nil { return nil, err } @@ -159,7 +159,7 @@ func buildSecureBootstrapKubeConfig(endpoint string, caCert []byte, clustername } // validateClusterInfoToken validates that the JWS token present in the cluster info ConfigMap is valid -func validateClusterInfoToken(insecureClusterInfo *v1.ConfigMap, token *kubeadmapi.BootstrapTokenString) ([]byte, error) { +func validateClusterInfoToken(insecureClusterInfo *v1.ConfigMap, token *bootstraptokenv1.BootstrapTokenString) ([]byte, error) { insecureKubeconfigString, ok := insecureClusterInfo.Data[bootstrapapi.KubeConfigKey] if !ok || len(insecureKubeconfigString) == 0 { return nil, errors.Errorf("there is no %s key in the %s ConfigMap. This API Server isn't set up for token bootstrapping, can't connect", @@ -202,7 +202,7 @@ func validateClusterCA(insecureConfig *clientcmdapi.Config, pubKeyPins *pubkeypi // getClusterInfo creates a client from the given kubeconfig if the given client is nil, // and requests the cluster info ConfigMap using PollImmediate. // If a client is provided it will be used instead. -func getClusterInfo(client clientset.Interface, kubeconfig *clientcmdapi.Config, token *kubeadmapi.BootstrapTokenString, interval, duration time.Duration) (*v1.ConfigMap, error) { +func getClusterInfo(client clientset.Interface, kubeconfig *clientcmdapi.Config, token *bootstraptokenv1.BootstrapTokenString, interval, duration time.Duration) (*v1.ConfigMap, error) { var cm *v1.ConfigMap var err error diff --git a/pkg/util/kubernetes/kubeadm/app/features/features.go b/pkg/util/kubernetes/kubeadm/app/features/features.go new file mode 100644 index 00000000000..1288cb5e5d8 --- /dev/null +++ b/pkg/util/kubernetes/kubeadm/app/features/features.go @@ -0,0 +1,181 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" +) + +const ( + // IPv6DualStack is expected to be beta in v1.21 + IPv6DualStack = "IPv6DualStack" + // PublicKeysECDSA is expected to be alpha in v1.19 + PublicKeysECDSA = "PublicKeysECDSA" + // RootlessControlPlane is expected to be in alpha in v1.22 + RootlessControlPlane = "RootlessControlPlane" +) + +// InitFeatureGates are the default feature gates for the init command +var InitFeatureGates = FeatureList{ + IPv6DualStack: {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}}, + PublicKeysECDSA: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, + RootlessControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, +} + +// Feature represents a feature being gated +type Feature struct { + featuregate.FeatureSpec + MinimumVersion *version.Version + HiddenInHelpText bool + DeprecationMessage string +} + +// FeatureList represents a list of feature gates +type FeatureList map[string]Feature + +// ValidateVersion ensures that a feature gate list is compatible with the chosen Kubernetes version +func ValidateVersion(allFeatures FeatureList, requestedFeatures map[string]bool, requestedVersion string) error { + if requestedVersion == "" { + return nil + } + parsedExpVersion, err := version.ParseSemantic(requestedVersion) + if err != nil { + return errors.Wrapf(err, "error parsing version %s", requestedVersion) + } + for k := range requestedFeatures { + if minVersion := allFeatures[k].MinimumVersion; minVersion != nil { + if !parsedExpVersion.AtLeast(minVersion) { + return errors.Errorf( + "the requested Kubernetes version (%s) is incompatible with the %s feature gate, which needs %s as a minimum", + requestedVersion, k, minVersion) + } + } + } + return nil +} + +// Enabled indicates whether a feature name has been enabled +func Enabled(featureList map[string]bool, featureName string) bool { + if enabled, ok := featureList[string(featureName)]; ok { + return enabled + } + return InitFeatureGates[string(featureName)].Default +} + +// Supports indicates whether a feature name is supported on the given +// feature set +func Supports(featureList FeatureList, featureName string) bool { + for k, v := range featureList { + if featureName == string(k) { + return v.PreRelease != featuregate.Deprecated + } + } + return false +} + +// Keys returns a slice of feature names for a given feature set +func Keys(featureList FeatureList) []string { + var list []string + for k := range featureList { + list = append(list, string(k)) + } + return list +} + +// KnownFeatures returns a slice of strings describing the FeatureList features. +func KnownFeatures(f *FeatureList) []string { + var known []string + for k, v := range *f { + if v.HiddenInHelpText { + continue + } + + pre := "" + if v.PreRelease != featuregate.GA { + pre = fmt.Sprintf("%s - ", v.PreRelease) + } + known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default)) + } + sort.Strings(known) + return known +} + +// NewFeatureGate parses a string of the form "key1=value1,key2=value2,..." into a +// map[string]bool of known keys or returns an error. +func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) { + featureGate := map[string]bool{} + for _, s := range strings.Split(value, ",") { + if len(s) == 0 { + continue + } + + arr := strings.SplitN(s, "=", 2) + if len(arr) != 2 { + return nil, errors.Errorf("missing bool value for feature-gate key:%s", s) + } + + k := strings.TrimSpace(arr[0]) + v := strings.TrimSpace(arr[1]) + + featureSpec, ok := (*f)[k] + if !ok { + return nil, errors.Errorf("unrecognized feature-gate key: %s", k) + } + + if featureSpec.PreRelease == featuregate.Deprecated { + return nil, errors.Errorf("feature-gate key is deprecated: %s", k) + } + + boolValue, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k) + } + featureGate[k] = boolValue + } + + return featureGate, nil +} + +// CheckDeprecatedFlags takes a list of existing feature gate flags and validates against the current feature flag set. +// It used during upgrades for ensuring consistency of feature gates used in an existing cluster, that might +// be created with a previous version of kubeadm, with the set of features currently supported by kubeadm +func CheckDeprecatedFlags(f *FeatureList, features map[string]bool) map[string]string { + deprecatedMsg := map[string]string{} + for k := range features { + featureSpec, ok := (*f)[k] + if !ok { + // This case should never happen, it is implemented only as a sentinel + // for removal of flags executed when flags are still in use (always before deprecate, then after one cycle remove) + deprecatedMsg[k] = fmt.Sprintf("Unknown feature gate flag: %s", k) + } + + if featureSpec.PreRelease == featuregate.Deprecated { + if _, ok := deprecatedMsg[k]; !ok { + deprecatedMsg[k] = featureSpec.DeprecationMessage + } + } + } + + return deprecatedMsg +} diff --git a/pkg/util/kubernetes/kubeadm/app/features/features_test.go b/pkg/util/kubernetes/kubeadm/app/features/features_test.go new file mode 100644 index 00000000000..81d29f3e6a8 --- /dev/null +++ b/pkg/util/kubernetes/kubeadm/app/features/features_test.go @@ -0,0 +1,219 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "reflect" + "testing" + + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" +) + +func TestKnownFeatures(t *testing.T) { + var someFeatures = FeatureList{ + "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}}, + "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, + "feature3": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.GA}}, + "hidden": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.GA}, HiddenInHelpText: true}, + } + + r := KnownFeatures(&someFeatures) + + if len(r) != 3 { + t.Errorf("KnownFeatures returned %d values, expected 3", len(r)) + } + + // check the first value is feature1 (the list should be sorted); prerelease and default should be present + f1 := "feature1=true|false (BETA - default=false)" + if r[0] != f1 { + t.Errorf("KnownFeatures returned %s values, expected %s", r[0], f1) + } + // check the second value is feature2; prerelease and default should be present + f2 := "feature2=true|false (ALPHA - default=true)" + if r[1] != f2 { + t.Errorf("KnownFeatures returned %s values, expected %s", r[1], f2) + } + // check the second value is feature3; prerelease should not be shown for GA features; default should be present + f3 := "feature3=true|false (default=false)" + if r[2] != f3 { + t.Errorf("KnownFeatures returned %s values, expected %s", r[2], f3) + } +} + +func TestNewFeatureGate(t *testing.T) { + var someFeatures = FeatureList{ + "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, + "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}}, + "deprecated": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Deprecated}}, + } + + var tests = []struct { + value string + expectedError bool + expectedFeaturesGate map[string]bool + }{ + { //invalid value (missing =) + value: "invalidValue", + expectedError: true, + }, + { //invalid value (missing =) + value: "feature1=true,invalidValue", + expectedError: true, + }, + { //invalid value (not a boolean) + value: "feature1=notABoolean", + expectedError: true, + }, + { //invalid value (not a boolean) + value: "feature1=true,feature2=notABoolean", + expectedError: true, + }, + { //unrecognized feature-gate key + value: "unknownFeature=false", + expectedError: true, + }, + { //unrecognized feature-gate key + value: "feature1=true,unknownFeature=false", + expectedError: true, + }, + { //deprecated feature-gate key + value: "deprecated=true", + expectedError: true, + }, + { //one feature + value: "feature1=true", + expectedError: false, + expectedFeaturesGate: map[string]bool{"feature1": true}, + }, + { //two features + value: "feature1=true,feature2=false", + expectedError: false, + expectedFeaturesGate: map[string]bool{"feature1": true, "feature2": false}, + }, + } + + for _, test := range tests { + t.Run(test.value, func(t *testing.T) { + r, err := NewFeatureGate(&someFeatures, test.value) + + if !test.expectedError && err != nil { + t.Errorf("NewFeatureGate failed when not expected: %v", err) + return + } else if test.expectedError && err == nil { + t.Error("NewFeatureGate didn't failed when expected") + return + } + + if !reflect.DeepEqual(r, test.expectedFeaturesGate) { + t.Errorf("NewFeatureGate returned a unexpected value") + } + }) + } +} + +func TestValidateVersion(t *testing.T) { + var someFeatures = FeatureList{ + "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, + "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}, MinimumVersion: version.MustParseSemantic("v1.17.0").WithPreRelease("alpha.1")}, + } + + var tests = []struct { + name string + requestedVersion string + requestedFeatures map[string]bool + expectedError bool + }{ + { + name: "no min version", + requestedFeatures: map[string]bool{"feature1": true}, + expectedError: false, + }, + { + name: "min version but correct value given", + requestedFeatures: map[string]bool{"feature2": true}, + requestedVersion: "v1.17.0", + expectedError: false, + }, + { + name: "min version and incorrect value given", + requestedFeatures: map[string]bool{"feature2": true}, + requestedVersion: "v1.11.2", + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := ValidateVersion(someFeatures, test.requestedFeatures, test.requestedVersion) + if !test.expectedError && err != nil { + t.Errorf("ValidateVersion failed when not expected: %v", err) + return + } else if test.expectedError && err == nil { + t.Error("ValidateVersion didn't failed when expected") + return + } + }) + } +} + +// TestEnabledDefaults tests that Enabled returns the default values for +// each feature gate when no feature gates are specified. +func TestEnabledDefaults(t *testing.T) { + for featureName, feature := range InitFeatureGates { + featureList := make(map[string]bool) + + enabled := Enabled(featureList, featureName) + if enabled != feature.Default { + t.Errorf("Enabled returned %v instead of default value %v for feature %s", enabled, feature.Default, featureName) + } + } +} + +func TestCheckDeprecatedFlags(t *testing.T) { + dummyMessage := "dummy message" + var someFeatures = FeatureList{ + "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, + "deprecated": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Deprecated}, DeprecationMessage: dummyMessage}, + } + + var tests = []struct { + name string + features map[string]bool + expectedMsg map[string]string + }{ + { + name: "deprecated feature", + features: map[string]bool{"deprecated": true}, + expectedMsg: map[string]string{"deprecated": dummyMessage}, + }, + { + name: "valid feature", + features: map[string]bool{"feature1": true}, + expectedMsg: map[string]string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + msg := CheckDeprecatedFlags(&someFeatures, test.features) + if !reflect.DeepEqual(test.expectedMsg, msg) { + t.Error("CheckDeprecatedFlags didn't returned expected message") + } + }) + } +} diff --git a/pkg/util/kubernetes/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go b/pkg/util/kubernetes/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go index d39ca01532e..5d3b2c31956 100644 --- a/pkg/util/kubernetes/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go +++ b/pkg/util/kubernetes/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go @@ -1,6 +1,5 @@ /* Copyright 2017 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -49,6 +48,9 @@ func CreateBootstrapConfigMapIfNotExists(client clientset.Interface, file string if err != nil { return errors.Wrap(err, "failed to load admin kubeconfig") } + if err = clientcmdapi.FlattenConfig(adminConfig); err != nil { + return err + } adminCluster := adminConfig.Contexts[adminConfig.CurrentContext].Cluster // Copy the cluster from admin.conf to the bootstrap kubeconfig, contains the CA cert and the server URL @@ -65,7 +67,7 @@ func CreateBootstrapConfigMapIfNotExists(client clientset.Interface, file string // Create or update the ConfigMap in the kube-public namespace klog.V(1).Infoln("[bootstrap-token] creating/updating ConfigMap in kube-public namespace") - return apiclient.CreateOrUpdateConfigMapWithTry(client, &v1.ConfigMap{ + return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: bootstrapapi.ConfigMapClusterInfo, Namespace: metav1.NamespacePublic, @@ -79,7 +81,7 @@ func CreateBootstrapConfigMapIfNotExists(client clientset.Interface, file string // CreateClusterInfoRBACRules creates the RBAC rules for exposing the cluster-info ConfigMap in the kube-public namespace to unauthenticated users func CreateClusterInfoRBACRules(client clientset.Interface) error { klog.V(1).Infoln("creating the RBAC rules for exposing the cluster-info ConfigMap in the kube-public namespace") - err := apiclient.CreateOrUpdateRoleWithTry(client, &rbac.Role{ + err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ ObjectMeta: metav1.ObjectMeta{ Name: BootstrapSignerClusterRoleName, Namespace: metav1.NamespacePublic, diff --git a/pkg/util/kubernetes/kubeadm/app/phases/bootstraptoken/node/token.go b/pkg/util/kubernetes/kubeadm/app/phases/bootstraptoken/node/token.go index 39052da6cb0..24fe53f719b 100644 --- a/pkg/util/kubernetes/kubeadm/app/phases/bootstraptoken/node/token.go +++ b/pkg/util/kubernetes/kubeadm/app/phases/bootstraptoken/node/token.go @@ -24,19 +24,17 @@ import ( clientset "k8s.io/client-go/kubernetes" bootstraputil "k8s.io/cluster-bootstrap/token/util" - kubeadmapi "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/apis/kubeadm" + bootstraptokenv1 "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1" "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/apiclient" ) -// TODO(mattmoyer): Move CreateNewTokens, UpdateOrCreateTokens out of this package to client-go for a generic abstraction and client for a Bootstrap Token - // CreateNewTokens tries to create a token and fails if one with the same ID already exists -func CreateNewTokens(client clientset.Interface, tokens []kubeadmapi.BootstrapToken) error { +func CreateNewTokens(client clientset.Interface, tokens []bootstraptokenv1.BootstrapToken) error { return UpdateOrCreateTokens(client, true, tokens) } // UpdateOrCreateTokens attempts to update a token with the given ID, or create if it does not already exist. -func UpdateOrCreateTokens(client clientset.Interface, failIfExists bool, tokens []kubeadmapi.BootstrapToken) error { +func UpdateOrCreateTokens(client clientset.Interface, failIfExists bool, tokens []bootstraptokenv1.BootstrapToken) error { for _, token := range tokens { @@ -46,7 +44,7 @@ func UpdateOrCreateTokens(client clientset.Interface, failIfExists bool, tokens return errors.Errorf("a token with id %q already exists", token.Token.ID) } - updatedOrNewSecret := token.ToSecret() + updatedOrNewSecret := bootstraptokenv1.BootstrapTokenToSecret(&token) // Try to create or update the token with an exponential backoff err = apiclient.TryRunCommand(func() error { if err := apiclient.CreateOrUpdateSecret(client, updatedOrNewSecret); err != nil { diff --git a/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags.go b/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags.go index 8b8d482ef1b..e32149e2c09 100644 --- a/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags.go +++ b/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags.go @@ -19,6 +19,7 @@ package kubelet import ( "fmt" + "io/ioutil" "os" "path/filepath" "strings" @@ -108,8 +109,14 @@ func writeKubeletFlagBytesToDisk(b []byte, kubeletDir string) error { if err := os.MkdirAll(kubeletDir, 0700); err != nil { return errors.Wrapf(err, "failed to create directory %q", kubeletDir) } - if err := os.WriteFile(kubeletEnvFilePath, b, 0644); err != nil { + if err := ioutil.WriteFile(kubeletEnvFilePath, b, 0644); err != nil { return errors.Wrapf(err, "failed to write kubelet configuration to the file %q", kubeletEnvFilePath) } return nil } + +// buildKubeletArgMap takes a kubeletFlagsOpts object and builds based on that a string-string map with flags +// that should be given to the local kubelet daemon. +func buildKubeletArgMap(opts joindata.YurtJoinData) map[string]string { + return buildKubeletArgMapCommon(opts) +} diff --git a/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_test.go b/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_test.go deleted file mode 100644 index fb64fd67ac0..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_test.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "reflect" - "strings" - "testing" -) - -func TestConstructNodeLabels(t *testing.T) { - edgeWorkerLabel := "openyurt.io/is-edge-worker" - testcases := map[string]struct { - nodeLabels map[string]string - mode string - result map[string]string - }{ - "no input node labels with cloud mode": { - mode: "cloud", - result: map[string]string{ - "openyurt.io/is-edge-worker": "false", - }, - }, - "one input node labels with cloud mode": { - nodeLabels: map[string]string{"foo": "bar"}, - mode: "cloud", - result: map[string]string{ - "openyurt.io/is-edge-worker": "false", - "foo": "bar", - }, - }, - "more than one input node labels with cloud mode": { - nodeLabels: map[string]string{ - "foo": "bar", - "foo2": "bar2", - }, - mode: "cloud", - result: map[string]string{ - "openyurt.io/is-edge-worker": "false", - "foo": "bar", - "foo2": "bar2", - }, - }, - "no input node labels with edge mode": { - mode: "edge", - result: map[string]string{ - "openyurt.io/is-edge-worker": "true", - }, - }, - "one input node labels with edge mode": { - nodeLabels: map[string]string{"foo": "bar"}, - mode: "edge", - result: map[string]string{ - "openyurt.io/is-edge-worker": "true", - "foo": "bar", - }, - }, - } - - for k, tc := range testcases { - t.Run(k, func(t *testing.T) { - constructedLabelsStr := constructNodeLabels(tc.nodeLabels, tc.mode, edgeWorkerLabel) - constructedLabels := make(map[string]string) - parts := strings.Split(constructedLabelsStr, ",") - for i := range parts { - kv := strings.Split(parts[i], "=") - if len(kv) == 2 { - constructedLabels[kv[0]] = kv[1] - } - } - - if !reflect.DeepEqual(constructedLabels, tc.result) { - t.Errorf("expected node labels: %v, but got %v", tc.result, constructedLabels) - } - }) - } - -} diff --git a/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_unix.go b/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_unix.go deleted file mode 100644 index 5f3e8337564..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_unix.go +++ /dev/null @@ -1,60 +0,0 @@ -//go:build !windows -// +build !windows - -/* -Copyright 2020 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "k8s.io/klog/v2" - utilsexec "k8s.io/utils/exec" - - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/constants" - kubeadmutil "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/initsystem" - "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" -) - -// buildKubeletArgMap takes a kubeletFlagsOpts object and builds based on that a string-string map with flags -// that should be given to the local Linux kubelet daemon. -func buildKubeletArgMap(data joindata.YurtJoinData) map[string]string { - kubeletFlags := buildKubeletArgMapCommon(data) - - // TODO: Conditionally set `--cgroup-driver` to either `systemd` or `cgroupfs` for CRI other than Docker - nodeReg := data.NodeRegistration() - if nodeReg.CRISocket == constants.DefaultDockerCRISocket { - driver, err := kubeadmutil.GetCgroupDriverDocker(utilsexec.New()) - if err != nil { - klog.Warningf("cannot automatically assign a '--cgroup-driver' value when starting the Kubelet: %v\n", err) - } else { - kubeletFlags["cgroup-driver"] = driver - } - } - - initSystem, err := initsystem.GetInitSystem() - if err != nil { - klog.Warningf("cannot get init system: %v\n", err) - return kubeletFlags - } - - if initSystem.ServiceIsActive("systemd-resolved") { - kubeletFlags["resolv-conf"] = "/run/systemd/resolve/resolv.conf" - } - - return kubeletFlags -} diff --git a/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_windows.go b/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_windows.go deleted file mode 100644 index 71cec754948..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/phases/kubelet/flags_windows.go +++ /dev/null @@ -1,28 +0,0 @@ -//go:build windows -// +build windows - -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" - -// buildKubeletArgMap takes a kubeletFlagsOpts object and builds based on that a string-string map with flags -// that should be given to the local Windows kubelet daemon. -func buildKubeletArgMap(data joindata.YurtJoinData) map[string]string { - return buildKubeletArgMapCommon(data) -} diff --git a/pkg/util/kubernetes/kubeadm/app/preflight/checks.go b/pkg/util/kubernetes/kubeadm/app/preflight/checks.go index 824a2ec7b48..b06ac7f0586 100644 --- a/pkg/util/kubernetes/kubeadm/app/preflight/checks.go +++ b/pkg/util/kubernetes/kubeadm/app/preflight/checks.go @@ -1,6 +1,5 @@ /* Copyright 2016 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,6 +22,8 @@ import ( "fmt" "io" "net" + "net/http" + "net/url" "os" "path/filepath" "runtime" @@ -30,22 +31,26 @@ import ( "github.com/pkg/errors" v1 "k8s.io/api/core/v1" + netutil "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" versionutil "k8s.io/apimachinery/pkg/util/version" "k8s.io/klog/v2" system "k8s.io/system-validators/validators" utilsexec "k8s.io/utils/exec" + utilsnet "k8s.io/utils/net" + kubeadmapi "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/apis/kubeadm" kubeadmconstants "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/constants" "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/initsystem" utilruntime "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/runtime" - "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" ) const ( - bridgenf = "/proc/sys/net/bridge/bridge-nf-call-iptables" - ipv4Forward = "/proc/sys/net/ipv4/ip_forward" + bridgenf = "/proc/sys/net/bridge/bridge-nf-call-iptables" + bridgenf6 = "/proc/sys/net/bridge/bridge-nf-call-ip6tables" + ipv4Forward = "/proc/sys/net/ipv4/ip_forward" + ipv6DefaultForwarding = "/proc/sys/net/ipv6/conf/default/forwarding" ) // Error defines struct for communicating error messages generated by preflight checks @@ -394,6 +399,96 @@ func (hc HostnameCheck) Check() (warnings, errorList []error) { return warnings, errorList } +// HTTPProxyCheck checks if https connection to specific host is going +// to be done directly or over proxy. If proxy detected, it will return warning. +type HTTPProxyCheck struct { + Proto string + Host string +} + +// Name returns HTTPProxy as name for HTTPProxyCheck +func (hst HTTPProxyCheck) Name() string { + return "HTTPProxy" +} + +// Check validates http connectivity type, direct or via proxy. +func (hst HTTPProxyCheck) Check() (warnings, errorList []error) { + klog.V(1).Infoln("validating if the connectivity type is via proxy or direct") + u := &url.URL{Scheme: hst.Proto, Host: hst.Host} + if utilsnet.IsIPv6String(hst.Host) { + u.Host = net.JoinHostPort(hst.Host, "1234") + } + + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, []error{err} + } + + proxy, err := netutil.SetOldTransportDefaults(&http.Transport{}).Proxy(req) + if err != nil { + return nil, []error{err} + } + if proxy != nil { + return []error{errors.Errorf("Connection to %q uses proxy %q. If that is not intended, adjust your proxy settings", u, proxy)}, nil + } + return nil, nil +} + +// HTTPProxyCIDRCheck checks if https connection to specific subnet is going +// to be done directly or over proxy. If proxy detected, it will return warning. +// Similar to HTTPProxyCheck above, but operates with subnets and uses API +// machinery transport defaults to simulate kube-apiserver accessing cluster +// services and pods. +type HTTPProxyCIDRCheck struct { + Proto string + CIDR string +} + +// Name will return HTTPProxyCIDR as name for HTTPProxyCIDRCheck +func (HTTPProxyCIDRCheck) Name() string { + return "HTTPProxyCIDR" +} + +// Check validates http connectivity to first IP address in the CIDR. +// If it is not directly connected and goes via proxy it will produce warning. +func (subnet HTTPProxyCIDRCheck) Check() (warnings, errorList []error) { + klog.V(1).Infoln("validating http connectivity to first IP address in the CIDR") + if len(subnet.CIDR) == 0 { + return nil, nil + } + + _, cidr, err := net.ParseCIDR(subnet.CIDR) + if err != nil { + return nil, []error{errors.Wrapf(err, "error parsing CIDR %q", subnet.CIDR)} + } + + testIP, err := utilsnet.GetIndexedIP(cidr, 1) + if err != nil { + return nil, []error{errors.Wrapf(err, "unable to get first IP address from the given CIDR (%s)", cidr.String())} + } + + testIPstring := testIP.String() + if len(testIP) == net.IPv6len { + testIPstring = fmt.Sprintf("[%s]:1234", testIP) + } + url := fmt.Sprintf("%s://%s/", subnet.Proto, testIPstring) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, []error{err} + } + + // Utilize same transport defaults as it will be used by API server + proxy, err := netutil.SetOldTransportDefaults(&http.Transport{}).Proxy(req) + if err != nil { + return nil, []error{err} + } + if proxy != nil { + return []error{errors.Errorf("connection to %q uses proxy %q. This may lead to malfunctional cluster setup. Make sure that Pod and Services IP ranges specified correctly as exceptions in proxy configuration", subnet.CIDR, proxy)}, nil + } + return nil, nil +} + // SystemVerificationCheck defines struct used for running the system verification node check in test/e2e_node/system type SystemVerificationCheck struct { IsDocker bool @@ -623,26 +718,50 @@ func (ncc NumCPUCheck) Check() (warnings, errorList []error) { } // RunJoinNodeChecks executes all individual, applicable to node checks. -func RunJoinNodeChecks(execer utilsexec.Interface, data joindata.YurtJoinData) error { +func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.JoinConfiguration, ignorePreflightErrors sets.String) error { // First, check if we're root separately from the other preflight checks and fail fast - if err := RunRootCheckOnly(data.IgnorePreflightErrors()); err != nil { + if err := RunRootCheckOnly(ignorePreflightErrors); err != nil { return err } checks := []Checker{ DirAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName)}, - FileExistingCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)}, - FileExistingCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, "pki", kubeadmconstants.CACertName)}, + FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)}, + FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName)}, + } + checks = addCommonChecks(execer, "", &cfg.NodeRegistration, checks) + if cfg.ControlPlane == nil { + checks = append(checks, FileAvailableCheck{Path: cfg.CACertPath}) + } + + addIPv6Checks := false + if cfg.Discovery.BootstrapToken != nil { + ipstr, _, err := net.SplitHostPort(cfg.Discovery.BootstrapToken.APIServerEndpoint) + if err == nil { + checks = append(checks, + HTTPProxyCheck{Proto: "https", Host: ipstr}, + ) + if ip := net.ParseIP(ipstr); ip != nil { + if utilsnet.IsIPv6(ip) { + addIPv6Checks = true + } + } + } + } + if addIPv6Checks { + checks = append(checks, + FileContentCheck{Path: bridgenf6, Content: []byte{'1'}}, + FileContentCheck{Path: ipv6DefaultForwarding, Content: []byte{'1'}}, + ) } - checks = addCommonChecks(execer, data, checks) - return RunChecks(checks, os.Stderr, data.IgnorePreflightErrors()) + return RunChecks(checks, os.Stderr, ignorePreflightErrors) } // addCommonChecks is a helper function to duplicate checks that are common between both the // kubeadm init and join commands -func addCommonChecks(execer utilsexec.Interface, data joindata.YurtJoinData, checks []Checker) []Checker { - containerRuntime, err := utilruntime.NewContainerRuntime(execer, data.NodeRegistration().CRISocket) +func addCommonChecks(execer utilsexec.Interface, k8sVersion string, nodeReg *kubeadmapi.NodeRegistrationOptions, checks []Checker) []Checker { + containerRuntime, err := utilruntime.NewContainerRuntime(execer, nodeReg.CRISocket) isDocker := false if err != nil { fmt.Printf("[preflight] WARNING: Couldn't create the interface used for talking to the container runtime: %v\n", err) @@ -654,13 +773,6 @@ func addCommonChecks(execer utilsexec.Interface, data joindata.YurtJoinData, che } } - // add image pull check for openyurt - checks = append(checks, ImagePullCheck{ - runtime: containerRuntime, - imageList: []string{data.PauseImage(), data.YurtHubImage()}, - imagePullPolicy: v1.PullIfNotPresent, - }) - // non-windows checks if runtime.GOOS == "linux" { if !isDocker { @@ -683,8 +795,8 @@ func addCommonChecks(execer utilsexec.Interface, data joindata.YurtJoinData, che } checks = append(checks, SystemVerificationCheck{IsDocker: isDocker}, - HostnameCheck{nodeName: data.NodeRegistration().Name}, - KubeletVersionCheck{KubernetesVersion: data.KubernetesVersion(), exec: execer}, + HostnameCheck{nodeName: nodeReg.Name}, + KubeletVersionCheck{KubernetesVersion: k8sVersion, exec: execer}, ServiceCheck{Service: "kubelet", CheckIfActive: false}, PortOpenCheck{port: kubeadmconstants.KubeletPort}) return checks diff --git a/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go b/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go index a60cb9b9725..6b2e2dfb330 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go +++ b/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go @@ -1,6 +1,5 @@ /* Copyright 2017 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/util/kubernetes/kubeadm/app/util/apiclient/tryidempotency.go b/pkg/util/kubernetes/kubeadm/app/util/apiclient/tryidempotency.go deleted file mode 100644 index 36c43d2e4ea..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/util/apiclient/tryidempotency.go +++ /dev/null @@ -1,232 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiclient - -import ( - "time" - - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" -) - -// CreateOrUpdateConfigMapWithTry runs CreateOrUpdateSecret with try. -func CreateOrUpdateConfigMapWithTry(client clientset.Interface, cm *v1.ConfigMap) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := CreateOrUpdateConfigMap(client, cm) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// CreateOrRetainConfigMapWithTry runs CreateOrRetainConfigMap with try. -func CreateOrRetainConfigMapWithTry(client clientset.Interface, cm *v1.ConfigMap, configMapName string) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := CreateOrRetainConfigMap(client, cm, configMapName) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// CreateOrUpdateSecretWithTry runs CreateOrUpdateSecret with try. -func CreateOrUpdateSecretWithTry(client clientset.Interface, secret *v1.Secret) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := CreateOrUpdateSecret(client, secret) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// CreateOrUpdateServiceAccountWithTry runs CreateOrUpdateServiceAccount with try. -func CreateOrUpdateServiceAccountWithTry(client clientset.Interface, sa *v1.ServiceAccount) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := CreateOrUpdateServiceAccount(client, sa) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// CreateOrUpdateDeploymentWithTry runs CreateOrUpdateDeployment with try. -func CreateOrUpdateDeploymentWithTry(client clientset.Interface, deploy *apps.Deployment) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := CreateOrUpdateDeployment(client, deploy) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// CreateOrUpdateDaemonSetWithTry runs CreateOrUpdateDaemonSet with try. -func CreateOrUpdateDaemonSetWithTry(client clientset.Interface, ds *apps.DaemonSet) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := CreateOrUpdateDaemonSet(client, ds) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// DeleteDaemonSetForegroundWithTry runs DeleteDaemonSetForeground with try. -func DeleteDaemonSetForegroundWithTry(client clientset.Interface, namespace, name string) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := DeleteDaemonSetForeground(client, namespace, name) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// DeleteDeploymentForegroundWithTry runs DeleteDeploymentForeground with try. -func DeleteDeploymentForegroundWithTry(client clientset.Interface, namespace, name string) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := DeleteDeploymentForeground(client, namespace, name) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// CreateOrUpdateRoleWithTry runs CreateOrUpdateRole with try. -func CreateOrUpdateRoleWithTry(client clientset.Interface, role *rbac.Role) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := CreateOrUpdateRole(client, role) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// CreateOrUpdateRoleBindingWithTry runs CreateOrUpdateRoleBinding with try. -func CreateOrUpdateRoleBindingWithTry(client clientset.Interface, roleBinding *rbac.RoleBinding) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := CreateOrUpdateRoleBinding(client, roleBinding) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// CreateOrUpdateClusterRoleWithTry runs CreateOrUpdateClusterRole with try. -func CreateOrUpdateClusterRoleWithTry(client clientset.Interface, clusterRole *rbac.ClusterRole) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := CreateOrUpdateClusterRole(client, clusterRole) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// CreateOrUpdateClusterRoleBindingWithTry runs CreateOrUpdateClusterRoleBinding with try. -func CreateOrUpdateClusterRoleBindingWithTry(client clientset.Interface, clusterRoleBinding *rbac.ClusterRoleBinding) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := CreateOrUpdateClusterRoleBinding(client, clusterRoleBinding) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// CreateOrMutateConfigMapWithTry runs CreateOrUpdateClusterRoleBinding with try. -func CreateOrMutateConfigMapWithTry(client clientset.Interface, cm *v1.ConfigMap, mutator ConfigMapMutator) error { - backoff := getBackOff() - - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := CreateOrMutateConfigMap(client, cm, mutator) - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} - -// try 200 times, the interval is three seconds. -func getBackOff() wait.Backoff { - backoff := wait.Backoff{ - Duration: 3 * time.Second, - Factor: 1, - Steps: 200, - } - return backoff -} diff --git a/pkg/util/kubernetes/kubeadm/app/util/apiclient/wait.go b/pkg/util/kubernetes/kubeadm/app/util/apiclient/wait.go index 2eee776995d..5b27cf2236e 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/apiclient/wait.go +++ b/pkg/util/kubernetes/kubeadm/app/util/apiclient/wait.go @@ -1,6 +1,5 @@ /* Copyright 2018 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/util/kubernetes/kubeadm/app/util/cgroupdriver.go b/pkg/util/kubernetes/kubeadm/app/util/cgroupdriver.go deleted file mode 100644 index b7f8668b4dd..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/util/cgroupdriver.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "strings" - - "github.com/pkg/errors" - utilsexec "k8s.io/utils/exec" -) - -const ( - // CgroupDriverSystemd holds the systemd driver type - CgroupDriverSystemd = "systemd" - // CgroupDriverCgroupfs holds the cgroupfs driver type - CgroupDriverCgroupfs = "cgroupfs" -) - -// TODO: add support for detecting the cgroup driver for CRI other than -// Docker. Currently only Docker driver detection is supported: -// Discussion: -// https://github.com/kubernetes/kubeadm/issues/844 - -// GetCgroupDriverDocker runs 'docker info -f "{{.CgroupDriver}}"' to obtain the docker cgroup driver -func GetCgroupDriverDocker(execer utilsexec.Interface) (string, error) { - driver, err := callDockerInfo(execer) - if err != nil { - return "", err - } - return strings.TrimSuffix(driver, "\n"), nil -} - -func callDockerInfo(execer utilsexec.Interface) (string, error) { - out, err := execer.Command("docker", "info", "-f", "{{.CgroupDriver}}").Output() - if err != nil { - return "", errors.Wrap(err, "cannot execute 'docker info -f {{.CgroupDriver}}'") - } - return string(out), nil -} diff --git a/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_unix.go b/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_unix.go index f1778eff330..5cbf9099e75 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_unix.go +++ b/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_unix.go @@ -1,4 +1,3 @@ -//go:build !windows // +build !windows /* @@ -87,7 +86,7 @@ func (sysd SystemdInitSystem) EnableCommand(service string) string { // reloadSystemd reloads the systemd daemon func (sysd SystemdInitSystem) reloadSystemd() error { if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil { - return fmt.Errorf("failed to reload systemd: %w", err) + return fmt.Errorf("failed to reload systemd: %v", err) } return nil } diff --git a/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_windows.go b/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_windows.go index fb1dd8a8b99..394272ddcb3 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_windows.go +++ b/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_windows.go @@ -1,4 +1,3 @@ -//go:build windows // +build windows /* @@ -46,14 +45,14 @@ func (sysd WindowsInitSystem) ServiceStart(service string) error { s, err := m.OpenService(service) if err != nil { - return fmt.Errorf("could not access service %s: %w", service, err) + return fmt.Errorf("could not access service %s: %v", service, err) } defer s.Close() // Check if service is already started status, err := s.Query() if err != nil { - return fmt.Errorf("could not query service %s: %w", service, err) + return fmt.Errorf("could not query service %s: %v", service, err) } if status.State != svc.Stopped && status.State != svc.StopPending { @@ -68,20 +67,20 @@ func (sysd WindowsInitSystem) ServiceStart(service string) error { time.Sleep(300 * time.Millisecond) status, err = s.Query() if err != nil { - return fmt.Errorf("could not retrieve %s service status: %w", service, err) + return fmt.Errorf("could not retrieve %s service status: %v", service, err) } } // Start the service err = s.Start("is", "manual-started") if err != nil { - return fmt.Errorf("could not start service %s: %w", service, err) + return fmt.Errorf("could not start service %s: %v", service, err) } // Check that the start was successful status, err = s.Query() if err != nil { - return fmt.Errorf("could not query service %s: %w", service, err) + return fmt.Errorf("could not query service %s: %v", service, err) } timeout = time.Now().Add(10 * time.Second) for status.State != svc.Running { @@ -91,7 +90,7 @@ func (sysd WindowsInitSystem) ServiceStart(service string) error { time.Sleep(300 * time.Millisecond) status, err = s.Query() if err != nil { - return fmt.Errorf("could not retrieve %s service status: %w", service, err) + return fmt.Errorf("could not retrieve %s service status: %v", service, err) } } return nil @@ -100,10 +99,10 @@ func (sysd WindowsInitSystem) ServiceStart(service string) error { // ServiceRestart tries to reload the environment and restart the specific service func (sysd WindowsInitSystem) ServiceRestart(service string) error { if err := sysd.ServiceStop(service); err != nil { - return fmt.Errorf("couldn't stop service %s: %w", service, err) + return fmt.Errorf("couldn't stop service %s: %v", service, err) } if err := sysd.ServiceStart(service); err != nil { - return fmt.Errorf("couldn't start service %s: %w", service, err) + return fmt.Errorf("couldn't start service %s: %v", service, err) } return nil @@ -120,14 +119,14 @@ func (sysd WindowsInitSystem) ServiceStop(service string) error { s, err := m.OpenService(service) if err != nil { - return fmt.Errorf("could not access service %s: %w", service, err) + return fmt.Errorf("could not access service %s: %v", service, err) } defer s.Close() // Check if service is already stopped status, err := s.Query() if err != nil { - return fmt.Errorf("could not query service %s: %w", service, err) + return fmt.Errorf("could not query service %s: %v", service, err) } if status.State == svc.Stopped { @@ -144,7 +143,7 @@ func (sysd WindowsInitSystem) ServiceStop(service string) error { time.Sleep(300 * time.Millisecond) status, err = s.Query() if err != nil { - return fmt.Errorf("could not retrieve %s service status: %w", service, err) + return fmt.Errorf("could not retrieve %s service status: %v", service, err) } } return nil @@ -153,13 +152,13 @@ func (sysd WindowsInitSystem) ServiceStop(service string) error { // Stop the service status, err = s.Control(svc.Stop) if err != nil { - return fmt.Errorf("could not stop service %s: %w", service, err) + return fmt.Errorf("could not stop service %s: %v", service, err) } // Check that the stop was successful status, err = s.Query() if err != nil { - return fmt.Errorf("could not query service %s: %w", service, err) + return fmt.Errorf("could not query service %s: %v", service, err) } timeout := time.Now().Add(10 * time.Second) for status.State != svc.Stopped { @@ -169,7 +168,7 @@ func (sysd WindowsInitSystem) ServiceStop(service string) error { time.Sleep(300 * time.Millisecond) status, err = s.Query() if err != nil { - return fmt.Errorf("could not retrieve %s service status: %w", service, err) + return fmt.Errorf("could not retrieve %s service status: %v", service, err) } } return nil @@ -239,7 +238,7 @@ func (sysd WindowsInitSystem) ServiceIsActive(service string) bool { func GetInitSystem() (InitSystem, error) { m, err := mgr.Connect() if err != nil { - return nil, fmt.Errorf("no supported init system detected: %w", err) + return nil, fmt.Errorf("no supported init system detected: %v", err) } defer m.Disconnect() return &WindowsInitSystem{}, nil diff --git a/pkg/util/kubernetes/kubeadm/app/util/kubeconfig/kubeconfig.go b/pkg/util/kubernetes/kubeadm/app/util/kubeconfig/kubeconfig.go index 92221643a12..fead644c60b 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/kubeconfig/kubeconfig.go +++ b/pkg/util/kubernetes/kubeadm/app/util/kubeconfig/kubeconfig.go @@ -18,7 +18,7 @@ package kubeconfig import ( "fmt" - "os" + "io/ioutil" "github.com/pkg/errors" clientset "k8s.io/client-go/kubernetes" @@ -150,7 +150,7 @@ func EnsureAuthenticationInfoAreEmbedded(config *clientcmdapi.Config) error { } if len(authInfo.ClientCertificateData) == 0 && len(authInfo.ClientCertificate) != 0 { - clientCert, err := os.ReadFile(authInfo.ClientCertificate) + clientCert, err := ioutil.ReadFile(authInfo.ClientCertificate) if err != nil { return errors.Wrap(err, "error while reading client cert file defined in kubeconfig") } @@ -158,7 +158,7 @@ func EnsureAuthenticationInfoAreEmbedded(config *clientcmdapi.Config) error { authInfo.ClientCertificate = "" } if len(authInfo.ClientKeyData) == 0 && len(authInfo.ClientKey) != 0 { - clientKey, err := os.ReadFile(authInfo.ClientKey) + clientKey, err := ioutil.ReadFile(authInfo.ClientKey) if err != nil { return errors.Wrap(err, "error while reading client key file defined in kubeconfig") } @@ -177,7 +177,7 @@ func EnsureCertificateAuthorityIsEmbedded(cluster *clientcmdapi.Cluster) error { } if len(cluster.CertificateAuthorityData) == 0 && len(cluster.CertificateAuthority) != 0 { - ca, err := os.ReadFile(cluster.CertificateAuthority) + ca, err := ioutil.ReadFile(cluster.CertificateAuthority) if err != nil { return errors.Wrap(err, "error while reading certificate authority file defined in kubeconfig") } diff --git a/pkg/util/kubernetes/kubeadm/app/util/kubeconfig/kubeconfig_test.go b/pkg/util/kubernetes/kubeadm/app/util/kubeconfig/kubeconfig_test.go index 749fcbe5f92..44cbbaa6795 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/kubeconfig/kubeconfig_test.go +++ b/pkg/util/kubernetes/kubeadm/app/util/kubeconfig/kubeconfig_test.go @@ -18,8 +18,8 @@ package kubeconfig import ( "bytes" - "errors" "fmt" + "io/ioutil" "os" "testing" @@ -143,7 +143,7 @@ func TestCreateWithToken(t *testing.T) { } func TestWriteKubeconfigToDisk(t *testing.T) { - tmpdir, err := os.MkdirTemp("", "") + tmpdir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("Couldn't create tmpdir") } @@ -170,14 +170,14 @@ func TestWriteKubeconfigToDisk(t *testing.T) { ) configPath := fmt.Sprintf("%s/etc/kubernetes/%s.conf", tmpdir, rt.name) err := WriteToDisk(configPath, c) - if !errors.Is(err, rt.expected) { + if err != rt.expected { t.Errorf( "failed WriteToDisk with an error:\n\texpected: %s\n\t actual: %s", rt.expected, err, ) } - newFile, _ := os.ReadFile(configPath) + newFile, _ := ioutil.ReadFile(configPath) if !bytes.Equal(newFile, rt.file) { t.Errorf( "failed WriteToDisk config write:\n\texpected: %s\n\t actual: %s", diff --git a/pkg/util/kubernetes/kubeadm/app/util/pubkeypin/pubkeypin.go b/pkg/util/kubernetes/kubeadm/app/util/pubkeypin/pubkeypin.go index fb157160d6a..01999d29497 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/pubkeypin/pubkeypin.go +++ b/pkg/util/kubernetes/kubeadm/app/util/pubkeypin/pubkeypin.go @@ -32,6 +32,11 @@ const ( formatSHA256 = "sha256" ) +var ( + // supportedFormats enumerates the supported formats + supportedFormats = strings.Join([]string{formatSHA256}, ", ") +) + // Set is a set of pinned x509 public keys. type Set struct { sha256Hashes map[string]bool @@ -47,15 +52,18 @@ func (s *Set) Allow(pubKeyHashes ...string) error { for _, pubKeyHash := range pubKeyHashes { parts := strings.Split(pubKeyHash, ":") if len(parts) != 2 { - return errors.New("invalid public key hash, expected \"format:value\"") + return errors.Errorf("invalid hash, expected \"format:hex-value\". "+ + "Known format(s) are: %s", supportedFormats) } format, value := parts[0], parts[1] switch strings.ToLower(format) { case "sha256": - return s.allowSHA256(value) + if err := s.allowSHA256(value); err != nil { + return errors.Errorf("invalid hash %q, %v", pubKeyHash, err) + } default: - return errors.Errorf("unknown hash format %q", format) + return errors.Errorf("unknown hash format %q. Known format(s) are: %s", format, supportedFormats) } } return nil @@ -99,7 +107,7 @@ func (s *Set) allowSHA256(hash string) error { // validate that the hash is valid hex _, err := hex.DecodeString(hash) if err != nil { - return err + return errors.Wrap(err, "could not decode SHA-256 from hex") } // in the end, just store the original hex string in memory (in lowercase) diff --git a/pkg/util/kubernetes/kubeadm/app/util/pubkeypin/pubkeypin_test.go b/pkg/util/kubernetes/kubeadm/app/util/pubkeypin/pubkeypin_test.go index 8ca2c6cf776..af726ea9bc5 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/pubkeypin/pubkeypin_test.go +++ b/pkg/util/kubernetes/kubeadm/app/util/pubkeypin/pubkeypin_test.go @@ -143,6 +143,17 @@ func TestSet(t *testing.T) { t.Error("expected the second test cert to be disallowed") return } + + s = NewSet() // keep set empty + hashes := []string{ + `sha256:0000000000000000000000000000000000000000000000000000000000000000`, + `sha256:0000000000000000000000000000000000000000000000000000000000000001`, + } + err = s.Allow(hashes...) + if err != nil || len(s.sha256Hashes) != 2 { + t.Error("expected allowing multiple hashes to succeed") + return + } } func TestHash(t *testing.T) { diff --git a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime.go b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime.go index 9c85ac1cdc3..b4f99377aa7 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime.go +++ b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime.go @@ -1,6 +1,5 @@ /* Copyright 2018 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_test.go b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_test.go index 5382779e16e..d10d05c0c6c 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_test.go +++ b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_test.go @@ -1,6 +1,5 @@ /* Copyright 2018 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,6 +17,7 @@ limitations under the License. package util import ( + "io/ioutil" "net" "os" "reflect" @@ -336,7 +336,7 @@ func TestIsExistingSocket(t *testing.T) { { name: "Valid domain socket is detected as such", proc: func(t *testing.T) { - tmpFile, err := os.CreateTemp(os.TempDir(), tempPrefix) + tmpFile, err := ioutil.TempFile("", tempPrefix) if err != nil { t.Fatalf("unexpected error by TempFile: %v", err) } @@ -358,7 +358,7 @@ func TestIsExistingSocket(t *testing.T) { { name: "Regular file is not a domain socket", proc: func(t *testing.T) { - tmpFile, err := os.CreateTemp(os.TempDir(), tempPrefix) + tmpFile, err := ioutil.TempFile("", tempPrefix) if err != nil { t.Fatalf("unexpected error by TempFile: %v", err) } diff --git a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_unix.go b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_unix.go index 11bc059dea7..b15c3037313 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_unix.go +++ b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_unix.go @@ -1,4 +1,3 @@ -//go:build !windows // +build !windows /* diff --git a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_windows.go b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_windows.go index 35a84cd3855..0c6a7b496dc 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_windows.go +++ b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_windows.go @@ -1,4 +1,3 @@ -//go:build windows // +build windows /* diff --git a/pkg/yurtadm/cmd/join/join.go b/pkg/yurtadm/cmd/join/join.go index 0d930b388d2..1092f1ab71a 100644 --- a/pkg/yurtadm/cmd/join/join.go +++ b/pkg/yurtadm/cmd/join/join.go @@ -122,7 +122,7 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) { "Use this token for both discovery-token and tls-bootstrap-token when those values are not provided.", ) flagSet.StringVar( - &joinOptions.nodeType, options.NodeType, joinOptions.nodeType, + &joinOptions.nodeType, yurtconstants.NodeType, joinOptions.nodeType, "Sets the node is edge or cloud", ) flagSet.StringVar( @@ -134,15 +134,15 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) { "Path to the CRI socket to connect", ) flagSet.StringVar( - &joinOptions.organizations, options.Organizations, joinOptions.organizations, + &joinOptions.organizations, yurtconstants.Organizations, joinOptions.organizations, "Organizations that will be added into hub's client certificate", ) flagSet.StringVar( - &joinOptions.pauseImage, options.PauseImage, joinOptions.pauseImage, + &joinOptions.pauseImage, yurtconstants.PauseImage, joinOptions.pauseImage, "Sets the image version of pause container", ) flagSet.StringVar( - &joinOptions.yurthubImage, options.YurtHubImage, joinOptions.yurthubImage, + &joinOptions.yurthubImage, yurtconstants.YurtHubImage, joinOptions.yurthubImage, "Sets the image version of yurthub component", ) flagSet.StringSliceVar( @@ -158,11 +158,11 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) { "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", ) flagSet.StringVar( - &joinOptions.nodeLabels, options.NodeLabels, joinOptions.nodeLabels, + &joinOptions.nodeLabels, yurtconstants.NodeLabels, joinOptions.nodeLabels, "Sets the labels for joining node", ) flagSet.StringVar( - &joinOptions.kubernetesResourceServer, options.KubernetesResourceServer, joinOptions.kubernetesResourceServer, + &joinOptions.kubernetesResourceServer, yurtconstants.KubernetesResourceServer, joinOptions.kubernetesResourceServer, "Sets the address for downloading k8s node resources", ) } diff --git a/pkg/yurtadm/cmd/join/phases/preflight.go b/pkg/yurtadm/cmd/join/phases/preflight.go index c236ef998eb..af1fc040cbb 100644 --- a/pkg/yurtadm/cmd/join/phases/preflight.go +++ b/pkg/yurtadm/cmd/join/phases/preflight.go @@ -18,9 +18,12 @@ package phases import ( "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" utilsexec "k8s.io/utils/exec" + "strings" + kubeadmapi "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/apis/kubeadm" "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/cmd/options" "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow" "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/preflight" @@ -52,7 +55,28 @@ func runPreflight(c workflow.RunData) error { // Start with general checks klog.V(1).Infoln("[preflight] Running general checks") - if err := preflight.RunJoinNodeChecks(utilsexec.New(), data); err != nil { + + // /etc/kubernetes/kubelet.conf /etc/kubernetes/pki/ca.crt already create by yurtadm + ignorePreflightErrors := sets.NewString(strings.ToLower("FileAvailable--etc-kubernetes-kubelet.conf"),strings.ToLower("FileAvailable--etc-kubernetes-pki-ca.crt")) + ignorePreflightErrors = ignorePreflightErrors.Union(data.IgnorePreflightErrors()) + + cfg := &kubeadmapi.JoinConfiguration{ + CACertPath: "/etc/kubernetes/pki/ca.crt", + NodeRegistration: kubeadmapi.NodeRegistrationOptions{ + IgnorePreflightErrors: ignorePreflightErrors.List(), + CRISocket: data.NodeRegistration().CRISocket, + Name: data.NodeRegistration().Name, + }, + Discovery: kubeadmapi.Discovery{ + TLSBootstrapToken: data.JoinToken(), + BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ + APIServerEndpoint: data.ServerAddr(), + Token: data.JoinToken()}, + }, + ControlPlane: nil, + } + + if err := preflight.RunJoinNodeChecks(utilsexec.New(), cfg, ignorePreflightErrors); err != nil { return err } diff --git a/pkg/yurtadm/constants/join_options.go b/pkg/yurtadm/constants/join_options.go new file mode 100644 index 00000000000..750e8c5b00b --- /dev/null +++ b/pkg/yurtadm/constants/join_options.go @@ -0,0 +1,38 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package constants + +const ( + + // NodeType flag sets the type of worker node to edge or cloud. + NodeType = "node-type" + + // Organizations flag sets the extra organizations of hub agent client certificate. + Organizations = "organizations" + + // NodeLabels flag sets the labels for worker node. + NodeLabels = "node-labels" + + // PauseImage flag sets the pause image for worker node. + PauseImage = "pause-image" + + // YurtHubImage flag sets the yurthub image for worker node. + YurtHubImage = "yurthub-image" + + // KubernetesResourceServer flag sets the address for download k8s node resources. + KubernetesResourceServer = "kubernetes-resource-server" +) diff --git a/pkg/yurtctl/util/kubernetes/util.go b/pkg/yurtctl/util/kubernetes/util.go index 86074952d62..cf29b40f725 100644 --- a/pkg/yurtctl/util/kubernetes/util.go +++ b/pkg/yurtctl/util/kubernetes/util.go @@ -55,7 +55,7 @@ import ( "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/projectinfo" - kubeadmapi "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/apis/kubeadm" + bootstraptokenv1 "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1" kubeadmconstants "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/constants" nodetoken "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/phases/bootstraptoken/node" tmplutil "github.com/openyurtio/openyurt/pkg/util/templates" @@ -500,7 +500,7 @@ func GetOrCreateJoinTokenString(cliSet *kubernetes.Clientset) (string, error) { for _, secret := range secrets.Items { // Get the BootstrapToken struct representation from the Secret object - token, err := kubeadmapi.BootstrapTokenFromSecret(&secret) + token, err := bootstraptokenv1.BootstrapTokenFromSecret(&secret) if err != nil { klog.Warningf("%v", err) continue @@ -517,14 +517,14 @@ func GetOrCreateJoinTokenString(cliSet *kubernetes.Clientset) (string, error) { if err != nil { return "", fmt.Errorf("couldn't generate random token, %w", err) } - token, err := kubeadmapi.NewBootstrapTokenString(tokenStr) + token, err := bootstraptokenv1.NewBootstrapTokenString(tokenStr) if err != nil { return "", err } klog.V(1).Infoln("[token] creating token") if err := nodetoken.CreateNewTokens(cliSet, - []kubeadmapi.BootstrapToken{{ + []bootstraptokenv1.BootstrapToken{{ Token: token, Usages: kubeadmconstants.DefaultTokenUsages, Groups: kubeadmconstants.DefaultTokenGroups, @@ -535,7 +535,7 @@ func GetOrCreateJoinTokenString(cliSet *kubernetes.Clientset) (string, error) { } // usagesAndGroupsAreValid checks if the usages and groups in the given bootstrap token are valid -func usagesAndGroupsAreValid(token *kubeadmapi.BootstrapToken) bool { +func usagesAndGroupsAreValid(token *bootstraptokenv1.BootstrapToken) bool { sliceEqual := func(a, b []string) bool { if len(a) != len(b) { return false