diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index e40a295586ed..e8e4c494f85f 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -220,16 +220,20 @@ func getAssetAndDir(dataDir string) (string, string) { // extract checks for and if necessary unpacks the bindata archive, returning the unique path // to the extracted bindata asset. func extract(dataDir string) (string, error) { - // first look for global asset folder so we don't create a HOME version if not needed - _, dir := getAssetAndDir(datadir.DefaultDataDir) + // check if content already exists in requested data-dir + asset, dir := getAssetAndDir(dataDir) if _, err := os.Stat(filepath.Join(dir, "bin", "k3s")); err == nil { return dir, nil } - asset, dir := getAssetAndDir(dataDir) - // check if target content already exists - if _, err := os.Stat(filepath.Join(dir, "bin", "k3s")); err == nil { - return dir, nil + // check if content exists in default path as a fallback, prior + // to extracting. This will prevent re-extracting into the user's home + // dir if the assets already exist in the default path. + if dataDir != datadir.DefaultDataDir { + _, defaultDir := getAssetAndDir(datadir.DefaultDataDir) + if _, err := os.Stat(filepath.Join(defaultDir, "bin", "k3s")); err == nil { + return defaultDir, nil + } } // acquire a data directory lock diff --git a/docs/adrs/etcd-snapshot-cr.md b/docs/adrs/etcd-snapshot-cr.md new file mode 100644 index 000000000000..d4454df7f2aa --- /dev/null +++ b/docs/adrs/etcd-snapshot-cr.md @@ -0,0 +1,60 @@ +# Store etcd snapshot metadata in a Custom Resource + +Date: 2023-07-27 + +## Status + +Accepted + +## Context + +K3s currently stores a list of etcd snapshots and associated metadata in a ConfigMap. Other downstream +projects and controllers consume the content of this ConfigMap in order to present cluster administrators with +a list of snapshots that can be restored. + +On clusters with more than a handful of nodes, and reasonable snapshot intervals and retention periods, the snapshot +list ConfigMap frequently reaches the maximum size allowed by Kubernetes, and fails to store any additional information. +The snapshots are still created, but they cannot be discovered by users or accessed by tools that consume information +from the ConfigMap. + +When this occurs, the K3s service log shows errors such as: +``` +level=error msg="failed to save local snapshot data to configmap: ConfigMap \"k3s-etcd-snapshots\" is invalid: []: Too long: must have at most 1048576 bytes" +``` + +A side-effect of this is that snapshot metadata is lost if the ConfigMap cannot be updated, as the list is the only place that it is stored. + +Reference: +* https://github.com/rancher/rke2/issues/4495 +* https://github.com/k3s-io/k3s/blob/36645e7311e9bdbbf2adb79ecd8bd68556bc86f6/pkg/etcd/etcd.go#L1503-L1516 + +### Existing Work + +Rancher already has a `rke.cattle.io/v1 ETCDSnapshot` Custom Resource that contains the same information after it's been +imported by the management cluster: +* https://github.com/rancher/rancher/blob/027246f77f03b82660dc2e91df6bf2cd549163f0/pkg/apis/rke.cattle.io/v1/etcd.go#L48-L74 + +It is unlikely that we would want to use this custom resource in its current package; we may be able to negotiate moving +it into a neutral project for use by both projects. + +## Decision + +1. Instead of populating snapshots into a ConfigMap using the JSON serialization of the private `snapshotFile` type, K3s + will manage creation of an new Custom Resource Definition with similar fields. +2. Metadata on each snapshot will be stored in a distinct Custom Resource. +3. The new Custom Resource will be cluster-scoped, as etcd and its snapshots are a cluster-level resource. +4. Snapshot metadata will also be written alongside snapshot files created on disk and/or uploaded to S3. The metadata + files will have the same basename as their corresponding snapshot file. +5. A hash of the server token will be stored as an annotation on the Custom Resource, and stored as metadata on snapshots uploaded to S3. + This hash should be compared to a current etcd snapshot's token hash to determine if the server token must be rolled back as part of the + snapshot restore process. +6. Downstream consumers of etcd snapshot lists will migrate to watching Custom Resource types, instead of the ConfigMap. +7. K3s will observe a three minor version transition period, where both the new Custom Resources, and the existing + ConfigMap, will both be used. +8. During the transition period, older snapshot metadata may be removed from the ConfigMap while those snapshots still + exist and are referenced by new Custom Resources, if the ConfigMap exceeds a preset size or key count limit. + +## Consequences + +* Snapshot metadata will no longer be lost when the number of snapshots exceeds what can be stored in the ConfigMap. +* There will be some additional complexity in managing the new Custom Resource, and working with other projects to migrate to using it. diff --git a/go.mod b/go.mod index 208e8ee1968c..0948e75725d5 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ replace ( github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.11.0 github.com/Mirantis/cri-dockerd => github.com/k3s-io/cri-dockerd v0.3.4-k3s1 // k3s/release-1.27 github.com/cloudnativelabs/kube-router/v2 => github.com/k3s-io/kube-router/v2 v2.0.0-20230925161250-364f994b140b - github.com/containerd/containerd => github.com/k3s-io/containerd v1.7.6-k3s1.27 + github.com/containerd/containerd => github.com/k3s-io/containerd v1.7.7-k3s1.27 github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e github.com/docker/distribution => github.com/docker/distribution v2.8.2+incompatible github.com/docker/docker => github.com/docker/docker v24.0.0-rc.2.0.20230801142700-69c9adb7d386+incompatible @@ -43,11 +43,11 @@ replace ( go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.13.0 go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v0.19.0 golang.org/x/crypto => golang.org/x/crypto v0.1.0 - golang.org/x/net => golang.org/x/net v0.8.0 + golang.org/x/net => golang.org/x/net v0.17.0 golang.org/x/sys => golang.org/x/sys v0.6.0 google.golang.org/api => google.golang.org/api v0.60.0 google.golang.org/genproto => google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 - google.golang.org/grpc => google.golang.org/grpc v1.51.0 + google.golang.org/grpc => google.golang.org/grpc v1.58.3 gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.6.0 k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.27.6-k3s1 k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.27.6-k3s1 @@ -141,10 +141,10 @@ require ( go.etcd.io/etcd/etcdutl/v3 v3.5.9 go.etcd.io/etcd/server/v3 v3.5.9 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.14.0 golang.org/x/net v0.14.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.11.0 + golang.org/x/sys v0.13.0 google.golang.org/grpc v1.57.0 gopkg.in/yaml.v2 v2.4.0 inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252 @@ -165,7 +165,7 @@ require ( ) require ( - cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute v1.21.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect @@ -184,7 +184,7 @@ require ( github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.11.0 // indirect + github.com/Microsoft/hcsshim v0.11.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/Rican7/retry v0.1.0 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect @@ -211,7 +211,8 @@ require ( github.com/containerd/go-cni v1.1.9 // indirect github.com/containerd/go-runc v1.0.0 // indirect github.com/containerd/imgcrypt v1.1.7 // indirect - github.com/containerd/nri v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/nri v0.4.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/containerd/ttrpc v1.2.2 // indirect github.com/containerd/typeurl v1.0.2 // indirect @@ -267,12 +268,12 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect github.com/hanwen/go-fuse/v2 v2.3.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -340,7 +341,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect github.com/prometheus/client_golang v1.16.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect github.com/rs/xid v1.5.0 // indirect @@ -382,17 +383,17 @@ require ( go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/mod v0.10.0 // indirect - golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/term v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/mod v0.11.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.9.3 // indirect + golang.org/x/tools v0.10.0 // indirect golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 // indirect - google.golang.org/api v0.108.0 // indirect + google.golang.org/api v0.126.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect + google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/gcfg.v1 v1.2.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index d65368da7a28..aa1df3047aec 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34h cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -33,8 +35,11 @@ cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOt cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= +cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= @@ -45,6 +50,12 @@ cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -57,6 +68,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20221206110420-d395f97c4830/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= @@ -177,7 +190,7 @@ github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -194,10 +207,11 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4= github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/container-orchestrated-devices/container-device-interface v0.5.4 h1:PqQGqJqQttMP5oJ/qNGEg8JttlHqGY3xDbbcKb5T9E8= @@ -234,8 +248,10 @@ github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nN github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/imgcrypt v1.1.7 h1:WSf9o9EQ0KGHiUx2ESFZ+PKf4nxK9BcvV/nJDX8RkB4= github.com/containerd/imgcrypt v1.1.7/go.mod h1:FD8gqIcX5aTotCtOmjeCsi3A1dHmTZpnMISGKSczt4k= -github.com/containerd/nri v0.3.0 h1:2ZM4WImye1ypSnE7COjOvPAiLv84kaPILBDvb1tbDK8= -github.com/containerd/nri v0.3.0/go.mod h1:Zw9q2lP16sdg0zYybemZ9yTDy8g7fPCIB3KXOGlggXI= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/nri v0.4.0 h1:PjgIBm0RtUiFyEO6JqPBQZRQicbsIz41Fz/5VSC0zgw= +github.com/containerd/nri v0.4.0/go.mod h1:Zw9q2lP16sdg0zYybemZ9yTDy8g7fPCIB3KXOGlggXI= github.com/containerd/stargz-snapshotter v0.14.4-0.20230913082252-7275d45b185c h1:Qr2w9ZeMciAfruOt2be10s4W13vQiTD3gAEqz3zxUrg= github.com/containerd/stargz-snapshotter v0.14.4-0.20230913082252-7275d45b185c/go.mod h1:ytZHGHs/q9DsZCyA+27rSYQEsbGgToUwXtl/5znV9qQ= github.com/containerd/stargz-snapshotter/estargz v0.10.0/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0= @@ -346,8 +362,11 @@ github.com/emicklei/go-restful v2.16.0+incompatible h1:rgqiKNjTnFQA6kkhFe16D8epT github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83 h1:ngHdSomn2MyugZYKHiycad2xERwIrmMlET7A0lC0UU4= github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83/go.mod h1:v6o7m/E9bfvm79dE1iFiF+3T7zLBnrjYjkWMa1J+Hv0= github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= @@ -450,8 +469,9 @@ github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -512,6 +532,7 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -523,6 +544,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -533,6 +556,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= @@ -554,8 +578,9 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjdKDqyr/2L+f6U12Fk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/hanwen/go-fuse/v2 v2.3.0 h1:t5ivNIH2PK+zw4OBul/iJjsoG9K6kXo4nMDoBpciC8A= github.com/hanwen/go-fuse/v2 v2.3.0/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= @@ -589,6 +614,7 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -631,8 +657,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k3s-io/containerd v1.7.6-k3s1.27 h1:BGoNTvkH/rxURf5UCvE4hIDY2emZ3o1DPNlTAmCoJpI= -github.com/k3s-io/containerd v1.7.6-k3s1.27/go.mod h1:dWUW/BzVXrFhxzfRZ1Jmr/yLlRvjryZlb1ns2SCHsgs= +github.com/k3s-io/containerd v1.7.7-k3s1.27 h1:obFQjBCahWKETAmAVFdq4Ph8VduSK0Eho9bWjpd6hjs= +github.com/k3s-io/containerd v1.7.7-k3s1.27/go.mod h1:CrxVnLZTD61NLkOVNd7Cedb7E5huJzcmTVpunlITQJY= github.com/k3s-io/cri-dockerd v0.3.4-k3s1 h1:eCeVCeXzf10fyanv1gniSwidBjdO83/akv+M72uEnZc= github.com/k3s-io/cri-dockerd v0.3.4-k3s1/go.mod h1:0KDOU8lLjp+ETJFFCcVBRQbJ8puRoDxaHBDj8C87Fk4= github.com/k3s-io/cri-tools v1.26.0-rc.0-k3s1 h1:yWVy9pS0T1BWBMZBPRy2Q29gaLmaGknQHSnx+HStrVM= @@ -765,6 +791,7 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9 github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -995,8 +1022,9 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -1078,6 +1106,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -1121,6 +1150,7 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -1271,6 +1301,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -1287,10 +1318,11 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1304,13 +1336,16 @@ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1333,22 +1368,25 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1414,8 +1452,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1440,9 +1478,16 @@ google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11K google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= diff --git a/hack/crdgen.go b/hack/crdgen.go new file mode 100644 index 000000000000..fed1083d0b38 --- /dev/null +++ b/hack/crdgen.go @@ -0,0 +1,13 @@ +package main + +import ( + "os" + + k3scrd "github.com/k3s-io/k3s/pkg/crd" + _ "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io/v1" + "github.com/rancher/wrangler/pkg/crd" +) + +func main() { + crd.Print(os.Stdout, k3scrd.List()) +} diff --git a/manifests/local-storage.yaml b/manifests/local-storage.yaml index d912b338e35a..a330cab286b6 100644 --- a/manifests/local-storage.yaml +++ b/manifests/local-storage.yaml @@ -155,5 +155,5 @@ data: spec: containers: - name: helper-pod - image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/mirrored-library-busybox:1.34.1 + image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/mirrored-library-busybox:1.36.1 imagePullPolicy: IfNotPresent diff --git a/manifests/traefik.yaml b/manifests/traefik.yaml index 314592500fcf..aa98abf89da6 100644 --- a/manifests/traefik.yaml +++ b/manifests/traefik.yaml @@ -27,7 +27,7 @@ spec: priorityClassName: "system-cluster-critical" image: repository: "rancher/mirrored-library-traefik" - tag: "2.9.10" + tag: "2.10.5" tolerations: - key: "CriticalAddonsOnly" operator: "Exists" diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index 4d0c52fc42a6..a4313c461f59 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -125,6 +125,7 @@ enable_keychain = true runtime_type = "{{$v.RuntimeType}}" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}".options] BinaryName = "{{$v.BinaryName}}" + SystemdCgroup = {{ .SystemdCgroup }} {{end}} ` diff --git a/pkg/apis/k3s.cattle.io/v1/types.go b/pkg/apis/k3s.cattle.io/v1/types.go index 79ae04f77a0e..c52e8eee518b 100644 --- a/pkg/apis/k3s.cattle.io/v1/types.go +++ b/pkg/apis/k3s.cattle.io/v1/types.go @@ -1,20 +1,105 @@ package v1 import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient +// +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// Addon is used to track application of a manifest file on disk. It mostly exists so that the wrangler DesiredSet +// Apply controller has an object to track as the owner, and ensure that all created resources are tracked when the +// manifest is modified or removed. type Addon struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` + // Spec provides information about the on-disk manifest backing this resource. Spec AddonSpec `json:"spec,omitempty"` } type AddonSpec struct { - Source string `json:"source,omitempty"` - Checksum string `json:"checksum,omitempty"` + // Source is the Path on disk to the manifest file that this Addon tracks. + Source string `json:"source,omitempty" column:""` + // Checksum is the SHA256 checksum of the most recently successfully applied manifest file. + Checksum string `json:"checksum,omitempty" column:""` +} + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ETCDSnapshot tracks a point-in-time snapshot of the etcd datastore. +type ETCDSnapshotFile struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Spec defines properties of an etcd snapshot file + Spec ETCDSnapshotSpec `json:"spec,omitempty"` + // Status represents current information about a snapshot. + Status ETCDSnapshotStatus `json:"status,omitempty"` +} + +// ETCDSnapshotSpec desribes an etcd snapshot file +type ETCDSnapshotSpec struct { + // SnapshotName contains the base name of the snapshot file. CLI actions that act + // on snapshots stored locally or within a pre-configured S3 bucket and + // prefix usually take the snapshot name as their argument. + SnapshotName string `json:"snapshotName" column:""` + // NodeName contains the name of the node that took the snapshot. + NodeName string `json:"nodeName" column:"name=Node"` + // Location is the absolute file:// or s3:// URI address of the snapshot. + Location string `json:"location" column:""` + // Metadata contains point-in-time snapshot of the contents of the + // k3s-etcd-snapshot-extra-metadata ConfigMap's data field, at the time the + // snapshot was taken. This is intended to contain data about cluster state + // that may be important for an external system to have available when restoring + // the snapshot. + Metadata map[string]string `json:"metadata,omitempty"` + // S3 contains extra metadata about the S3 storage system holding the + // snapshot. This is guaranteed to be set for all snapshots uploaded to S3. + // If not specified, the snapshot was not uploaded to S3. + S3 *ETCDSnapshotS3 `json:"s3,omitempty"` +} + +// ETCDSnapshotS3 holds information about the S3 storage system holding the snapshot. +type ETCDSnapshotS3 struct { + // Endpoint is the host or host:port of the S3 service + Endpoint string `json:"endpoint,omitempty"` + // EndpointCA is the path on disk to the S3 service's trusted CA list. Leave empty to use the OS CA bundle. + EndpointCA string `json:"endpointCA,omitempty"` + // SkipSSLVerify is true if TLS certificate verification is disabled + SkipSSLVerify bool `json:"skipSSLVerify,omitempty"` + // Bucket is the bucket holding the snapshot + Bucket string `json:"bucket,omitempty"` + // Region is the region of the S3 service + Region string `json:"region,omitempty"` + // Prefix is the prefix in which the snapshot file is stored. + Prefix string `json:"prefix,omitempty"` + // Insecure is true if the S3 service uses HTTP instead of HTTPS + Insecure bool `json:"insecure,omitempty"` +} + +// ETCDSnapshotStatus is the status of the ETCDSnapshotFile object. +type ETCDSnapshotStatus struct { + // Size is the size of the snapshot file, in bytes. If not specified, the snapshot failed. + Size *resource.Quantity `json:"size,omitempty" column:""` + // CreationTime is the timestamp when the snapshot was taken by etcd. + CreationTime *metav1.Time `json:"creationTime,omitempty" column:""` + // ReadyToUse indicates that the snapshot is available to be restored. + ReadyToUse *bool `json:"readyToUse,omitempty"` + // Error is the last observed error during snapshot creation, if any. + // If the snapshot is retried, this field will be cleared on success. + Error *ETCDSnapshotError `json:"error,omitempty"` +} + +// ETCDSnapshotError describes an error encountered during snapshot creation. +type ETCDSnapshotError struct { + // Time is the timestamp when the error was encountered. + Time *metav1.Time `json:"time,omitempty"` + // Message is a string detailing the encountered error during snapshot creation if specified. + // NOTE: message may be logged, and it should not contain sensitive information. + Message *string `json:"message,omitempty"` } diff --git a/pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go b/pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go index 69011aa78d22..1679c1e7fff5 100644 --- a/pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go +++ b/pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go @@ -100,3 +100,168 @@ func (in *AddonSpec) DeepCopy() *AddonSpec { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDSnapshotError) DeepCopyInto(out *ETCDSnapshotError) { + *out = *in + if in.Time != nil { + in, out := &in.Time, &out.Time + *out = (*in).DeepCopy() + } + if in.Message != nil { + in, out := &in.Message, &out.Message + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotError. +func (in *ETCDSnapshotError) DeepCopy() *ETCDSnapshotError { + if in == nil { + return nil + } + out := new(ETCDSnapshotError) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDSnapshotFile) DeepCopyInto(out *ETCDSnapshotFile) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotFile. +func (in *ETCDSnapshotFile) DeepCopy() *ETCDSnapshotFile { + if in == nil { + return nil + } + out := new(ETCDSnapshotFile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ETCDSnapshotFile) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDSnapshotFileList) DeepCopyInto(out *ETCDSnapshotFileList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ETCDSnapshotFile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotFileList. +func (in *ETCDSnapshotFileList) DeepCopy() *ETCDSnapshotFileList { + if in == nil { + return nil + } + out := new(ETCDSnapshotFileList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ETCDSnapshotFileList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDSnapshotS3) DeepCopyInto(out *ETCDSnapshotS3) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotS3. +func (in *ETCDSnapshotS3) DeepCopy() *ETCDSnapshotS3 { + if in == nil { + return nil + } + out := new(ETCDSnapshotS3) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDSnapshotSpec) DeepCopyInto(out *ETCDSnapshotSpec) { + *out = *in + if in.Metadata != nil { + in, out := &in.Metadata, &out.Metadata + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.S3 != nil { + in, out := &in.S3, &out.S3 + *out = new(ETCDSnapshotS3) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotSpec. +func (in *ETCDSnapshotSpec) DeepCopy() *ETCDSnapshotSpec { + if in == nil { + return nil + } + out := new(ETCDSnapshotSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDSnapshotStatus) DeepCopyInto(out *ETCDSnapshotStatus) { + *out = *in + if in.Size != nil { + in, out := &in.Size, &out.Size + x := (*in).DeepCopy() + *out = &x + } + if in.CreationTime != nil { + in, out := &in.CreationTime, &out.CreationTime + *out = (*in).DeepCopy() + } + if in.ReadyToUse != nil { + in, out := &in.ReadyToUse, &out.ReadyToUse + *out = new(bool) + **out = **in + } + if in.Error != nil { + in, out := &in.Error, &out.Error + *out = new(ETCDSnapshotError) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotStatus. +func (in *ETCDSnapshotStatus) DeepCopy() *ETCDSnapshotStatus { + if in == nil { + return nil + } + out := new(ETCDSnapshotStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apis/k3s.cattle.io/v1/zz_generated_list_types.go b/pkg/apis/k3s.cattle.io/v1/zz_generated_list_types.go index 52955028638c..c00d6ac70ae7 100644 --- a/pkg/apis/k3s.cattle.io/v1/zz_generated_list_types.go +++ b/pkg/apis/k3s.cattle.io/v1/zz_generated_list_types.go @@ -40,3 +40,20 @@ func NewAddon(namespace, name string, obj Addon) *Addon { obj.Namespace = namespace return &obj } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ETCDSnapshotFileList is a list of ETCDSnapshotFile resources +type ETCDSnapshotFileList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []ETCDSnapshotFile `json:"items"` +} + +func NewETCDSnapshotFile(namespace, name string, obj ETCDSnapshotFile) *ETCDSnapshotFile { + obj.APIVersion, obj.Kind = SchemeGroupVersion.WithKind("ETCDSnapshotFile").ToAPIVersionAndKind() + obj.Name = name + obj.Namespace = namespace + return &obj +} diff --git a/pkg/apis/k3s.cattle.io/v1/zz_generated_register.go b/pkg/apis/k3s.cattle.io/v1/zz_generated_register.go index 80e8f3629831..90761711f75d 100644 --- a/pkg/apis/k3s.cattle.io/v1/zz_generated_register.go +++ b/pkg/apis/k3s.cattle.io/v1/zz_generated_register.go @@ -28,7 +28,8 @@ import ( ) var ( - AddonResourceName = "addons" + AddonResourceName = "addons" + ETCDSnapshotFileResourceName = "etcdsnapshotfiles" ) // SchemeGroupVersion is group version used to register these objects @@ -54,6 +55,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &Addon{}, &AddonList{}, + &ETCDSnapshotFile{}, + &ETCDSnapshotFileList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/pkg/cli/etcdsnapshot/etcd_snapshot.go b/pkg/cli/etcdsnapshot/etcd_snapshot.go index 714ccc982a4b..97e8c696fa65 100644 --- a/pkg/cli/etcdsnapshot/etcd_snapshot.go +++ b/pkg/cli/etcdsnapshot/etcd_snapshot.go @@ -1,19 +1,20 @@ package etcdsnapshot import ( + "context" "encoding/json" "errors" "fmt" "os" "path/filepath" + "sort" "strings" "text/tabwriter" "time" "github.com/erikdubbelboer/gspt" "github.com/k3s-io/k3s/pkg/cli/cmds" - "github.com/k3s-io/k3s/pkg/cluster" - "github.com/k3s-io/k3s/pkg/daemons/config" + daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/etcd" "github.com/k3s-io/k3s/pkg/server" util2 "github.com/k3s-io/k3s/pkg/util" @@ -22,16 +23,22 @@ import ( "gopkg.in/yaml.v2" ) +type etcdCommand struct { + etcd *etcd.ETCD + ctx context.Context +} + // commandSetup setups up common things needed // for each etcd command. -func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) error { +func commandSetup(app *cli.Context, cfg *cmds.Server, config *server.Config) (*etcdCommand, error) { + ctx := signals.SetupSignalContext() gspt.SetProcTitle(os.Args[0]) nodeName := app.String("node-name") if nodeName == "" { h, err := os.Hostname() if err != nil { - return err + return nil, err } nodeName = h } @@ -40,33 +47,53 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) error { dataDir, err := server.ResolveDataDir(cfg.DataDir) if err != nil { - return err + return nil, err + } + + config.DisableAgent = true + config.ControlConfig.DataDir = dataDir + config.ControlConfig.EtcdSnapshotName = cfg.EtcdSnapshotName + config.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir + config.ControlConfig.EtcdSnapshotCompress = cfg.EtcdSnapshotCompress + config.ControlConfig.EtcdListFormat = strings.ToLower(cfg.EtcdListFormat) + config.ControlConfig.EtcdS3 = cfg.EtcdS3 + config.ControlConfig.EtcdS3Endpoint = cfg.EtcdS3Endpoint + config.ControlConfig.EtcdS3EndpointCA = cfg.EtcdS3EndpointCA + config.ControlConfig.EtcdS3SkipSSLVerify = cfg.EtcdS3SkipSSLVerify + config.ControlConfig.EtcdS3AccessKey = cfg.EtcdS3AccessKey + config.ControlConfig.EtcdS3SecretKey = cfg.EtcdS3SecretKey + config.ControlConfig.EtcdS3BucketName = cfg.EtcdS3BucketName + config.ControlConfig.EtcdS3Region = cfg.EtcdS3Region + config.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder + config.ControlConfig.EtcdS3Insecure = cfg.EtcdS3Insecure + config.ControlConfig.EtcdS3Timeout = cfg.EtcdS3Timeout + config.ControlConfig.Runtime = daemonconfig.NewRuntime(nil) + config.ControlConfig.Runtime.ETCDServerCA = filepath.Join(dataDir, "tls", "etcd", "server-ca.crt") + config.ControlConfig.Runtime.ClientETCDCert = filepath.Join(dataDir, "tls", "etcd", "client.crt") + config.ControlConfig.Runtime.ClientETCDKey = filepath.Join(dataDir, "tls", "etcd", "client.key") + config.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig") + + e := etcd.NewETCD() + if err := e.SetControlConfig(&config.ControlConfig); err != nil { + return nil, err } - sc.DisableAgent = true - sc.ControlConfig.DataDir = dataDir - sc.ControlConfig.EtcdSnapshotName = cfg.EtcdSnapshotName - sc.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir - sc.ControlConfig.EtcdSnapshotCompress = cfg.EtcdSnapshotCompress - sc.ControlConfig.EtcdListFormat = strings.ToLower(cfg.EtcdListFormat) - sc.ControlConfig.EtcdS3 = cfg.EtcdS3 - sc.ControlConfig.EtcdS3Endpoint = cfg.EtcdS3Endpoint - sc.ControlConfig.EtcdS3EndpointCA = cfg.EtcdS3EndpointCA - sc.ControlConfig.EtcdS3SkipSSLVerify = cfg.EtcdS3SkipSSLVerify - sc.ControlConfig.EtcdS3AccessKey = cfg.EtcdS3AccessKey - sc.ControlConfig.EtcdS3SecretKey = cfg.EtcdS3SecretKey - sc.ControlConfig.EtcdS3BucketName = cfg.EtcdS3BucketName - sc.ControlConfig.EtcdS3Region = cfg.EtcdS3Region - sc.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder - sc.ControlConfig.EtcdS3Insecure = cfg.EtcdS3Insecure - sc.ControlConfig.EtcdS3Timeout = cfg.EtcdS3Timeout - sc.ControlConfig.Runtime = config.NewRuntime(nil) - sc.ControlConfig.Runtime.ETCDServerCA = filepath.Join(dataDir, "tls", "etcd", "server-ca.crt") - sc.ControlConfig.Runtime.ClientETCDCert = filepath.Join(dataDir, "tls", "etcd", "client.crt") - sc.ControlConfig.Runtime.ClientETCDKey = filepath.Join(dataDir, "tls", "etcd", "client.key") - sc.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig") + initialized, err := e.IsInitialized() + if err != nil { + return nil, err + } + if !initialized { + return nil, fmt.Errorf("etcd database not found in %s", config.ControlConfig.DataDir) + } - return nil + sc, err := server.NewContext(ctx, config.ControlConfig.Runtime.KubeConfigAdmin, false) + if err != nil { + return nil, err + } + config.ControlConfig.Runtime.K3s = sc.K3s + config.ControlConfig.Runtime.Core = sc.Core + + return &etcdCommand{etcd: e, ctx: ctx}, nil } // Save triggers an on-demand etcd snapshot operation @@ -80,43 +107,18 @@ func Save(app *cli.Context) error { func save(app *cli.Context, cfg *cmds.Server) error { var serverConfig server.Config - if err := commandSetup(app, cfg, &serverConfig); err != nil { - return err - } - if len(app.Args()) > 0 { return util2.ErrCommandNoArgs } - serverConfig.ControlConfig.EtcdSnapshotRetention = 0 // disable retention check - - ctx := signals.SetupSignalContext() - e := etcd.NewETCD() - if err := e.SetControlConfig(&serverConfig.ControlConfig); err != nil { - return err - } - - initialized, err := e.IsInitialized() + ec, err := commandSetup(app, cfg, &serverConfig) if err != nil { return err } - if !initialized { - return fmt.Errorf("etcd database not found in %s", serverConfig.ControlConfig.DataDir) - } - - cluster := cluster.New(&serverConfig.ControlConfig) - - if err := cluster.Bootstrap(ctx, true); err != nil { - return err - } - sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin, false) - if err != nil { - return err - } - serverConfig.ControlConfig.Runtime.Core = sc.Core + serverConfig.ControlConfig.EtcdSnapshotRetention = 0 // disable retention check - return cluster.Snapshot(ctx, &serverConfig.ControlConfig) + return ec.etcd.Snapshot(ec.ctx) } func Delete(app *cli.Context) error { @@ -129,7 +131,8 @@ func Delete(app *cli.Context) error { func delete(app *cli.Context, cfg *cmds.Server) error { var serverConfig server.Config - if err := commandSetup(app, cfg, &serverConfig); err != nil { + ec, err := commandSetup(app, cfg, &serverConfig) + if err != nil { return err } @@ -138,19 +141,7 @@ func delete(app *cli.Context, cfg *cmds.Server) error { return errors.New("no snapshots given for removal") } - ctx := signals.SetupSignalContext() - e := etcd.NewETCD() - if err := e.SetControlConfig(&serverConfig.ControlConfig); err != nil { - return err - } - - sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin, false) - if err != nil { - return err - } - serverConfig.ControlConfig.Runtime.Core = sc.Core - - return e.DeleteSnapshots(ctx, app.Args()) + return ec.etcd.DeleteSnapshots(ec.ctx, app.Args()) } func List(app *cli.Context) error { @@ -160,7 +151,7 @@ func List(app *cli.Context) error { return list(app, &cmds.ServerConfig) } -var etcdListFormats = []string{"json", "yaml"} +var etcdListFormats = []string{"json", "yaml", "table"} func validEtcdListFormat(format string) bool { for _, supportedFormat := range etcdListFormats { @@ -174,17 +165,12 @@ func validEtcdListFormat(format string) bool { func list(app *cli.Context, cfg *cmds.Server) error { var serverConfig server.Config - if err := commandSetup(app, cfg, &serverConfig); err != nil { - return err - } - - ctx := signals.SetupSignalContext() - e := etcd.NewETCD() - if err := e.SetControlConfig(&serverConfig.ControlConfig); err != nil { + ec, err := commandSetup(app, cfg, &serverConfig) + if err != nil { return err } - sf, err := e.ListSnapshots(ctx) + sf, err := ec.etcd.ListSnapshots(ec.ctx) if err != nil { return err } @@ -208,20 +194,23 @@ func list(app *cli.Context, cfg *cmds.Server) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) defer w.Flush() - if cfg.EtcdS3 { - fmt.Fprint(w, "Name\tSize\tCreated\n") - for _, s := range sf { - if s.NodeName == "s3" { - fmt.Fprintf(w, "%s\t%d\t%s\n", s.Name, s.Size, s.CreatedAt.Format(time.RFC3339)) - } - } - } else { - fmt.Fprint(w, "Name\tLocation\tSize\tCreated\n") - for _, s := range sf { - if s.NodeName != "s3" { - fmt.Fprintf(w, "%s\t%s\t%d\t%s\n", s.Name, s.Location, s.Size, s.CreatedAt.Format(time.RFC3339)) - } + // Sort snapshots by creation time and key + sfKeys := make([]string, 0, len(sf)) + for k := range sf { + sfKeys = append(sfKeys, k) + } + sort.Slice(sfKeys, func(i, j int) bool { + iKey := sfKeys[i] + jKey := sfKeys[j] + if sf[iKey].CreatedAt.Equal(sf[jKey].CreatedAt) { + return iKey < jKey } + return sf[iKey].CreatedAt.Before(sf[jKey].CreatedAt) + }) + + fmt.Fprint(w, "Name\tLocation\tSize\tCreated\n") + for _, k := range sfKeys { + fmt.Fprintf(w, "%s\t%s\t%d\t%s\n", sf[k].Name, sf[k].Location, sf[k].Size, sf[k].CreatedAt.Format(time.RFC3339)) } } @@ -238,23 +227,12 @@ func Prune(app *cli.Context) error { func prune(app *cli.Context, cfg *cmds.Server) error { var serverConfig server.Config - if err := commandSetup(app, cfg, &serverConfig); err != nil { + ec, err := commandSetup(app, cfg, &serverConfig) + if err != nil { return err } serverConfig.ControlConfig.EtcdSnapshotRetention = cfg.EtcdSnapshotRetention - ctx := signals.SetupSignalContext() - e := etcd.NewETCD() - if err := e.SetControlConfig(&serverConfig.ControlConfig); err != nil { - return err - } - - sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin, false) - if err != nil { - return err - } - serverConfig.ControlConfig.Runtime.Core = sc.Core - - return e.PruneSnapshots(ctx) + return ec.etcd.PruneSnapshots(ec.ctx) } diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index a2c63a974e49..a0f804564931 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -19,6 +19,7 @@ import ( "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/etcd" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/k3s-io/kine/pkg/client" "github.com/k3s-io/kine/pkg/endpoint" @@ -248,7 +249,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, if c.managedDB != nil && !isHTTP { token := c.config.Token if token == "" { - tokenFromFile, err := readTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir) + tokenFromFile, err := util.ReadTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir) if err != nil { return err } @@ -260,7 +261,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, token = tokenFromFile } - normalizedToken, err := normalizeToken(token) + normalizedToken, err := util.NormalizeToken(token) if err != nil { return err } @@ -424,15 +425,6 @@ func (c *Cluster) bootstrap(ctx context.Context) error { return c.storageBootstrap(ctx) } -// Snapshot is a proxy method to call the snapshot method on the managedb -// interface for etcd clusters. -func (c *Cluster) Snapshot(ctx context.Context, config *config.Control) error { - if c.managedDB == nil { - return errors.New("unable to perform etcd snapshot on non-etcd system") - } - return c.managedDB.Snapshot(ctx) -} - // compareConfig verifies that the config of the joining control plane node coincides with the cluster's config func (c *Cluster) compareConfig() error { token := c.config.AgentToken diff --git a/pkg/cluster/bootstrap_test.go b/pkg/cluster/bootstrap_test.go index b20a36fd6841..3531fcab25f2 100644 --- a/pkg/cluster/bootstrap_test.go +++ b/pkg/cluster/bootstrap_test.go @@ -197,50 +197,3 @@ func TestCluster_migrateBootstrapData(t *testing.T) { }) } } - -func TestCluster_Snapshot(t *testing.T) { - type fields struct { - clientAccessInfo *clientaccess.Info - config *config.Control - managedDB managed.Driver - joining bool - storageStarted bool - saveBootstrap bool - shouldBootstrap bool - } - type args struct { - ctx context.Context - config *config.Control - } - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - { - name: "Fail on non etcd cluster", - fields: fields{}, - args: args{ - ctx: context.Background(), - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := &Cluster{ - clientAccessInfo: tt.fields.clientAccessInfo, - config: tt.fields.config, - managedDB: tt.fields.managedDB, - joining: tt.fields.joining, - storageStarted: tt.fields.storageStarted, - saveBootstrap: tt.fields.saveBootstrap, - shouldBootstrap: tt.fields.shouldBootstrap, - } - if err := c.Snapshot(tt.args.ctx, tt.args.config); (err != nil) != tt.wantErr { - t.Errorf("Cluster.Snapshot() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/pkg/cluster/encrypt.go b/pkg/cluster/encrypt.go index 1046d61e1a8b..b39fdc151370 100644 --- a/pkg/cluster/encrypt.go +++ b/pkg/cluster/encrypt.go @@ -5,9 +5,7 @@ import ( "crypto/cipher" "crypto/rand" "crypto/sha1" - "crypto/sha256" "encoding/base64" - "encoding/hex" "fmt" "io" "strings" @@ -19,14 +17,7 @@ import ( // storageKey returns the etcd key for storing bootstrap data for a given passphrase. // The key is derived from the sha256 hash of the passphrase. func storageKey(passphrase string) string { - return "/bootstrap/" + keyHash(passphrase) -} - -// keyHash returns the first 12 characters of the sha256 sum of the passphrase. -func keyHash(passphrase string) string { - d := sha256.New() - d.Write([]byte(passphrase)) - return hex.EncodeToString(d.Sum(nil)[:])[:12] + return "/bootstrap/" + util.ShortHash(passphrase, 12) } // encrypt encrypts a byte slice using aes+gcm with a pbkdf2 key derived from the passphrase and a random salt. diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 70e3961fdd23..549291961253 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -4,13 +4,11 @@ import ( "bytes" "context" "errors" - "os" - "path/filepath" "time" "github.com/k3s-io/k3s/pkg/bootstrap" - "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/kine/pkg/client" "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" @@ -23,12 +21,12 @@ const maxBootstrapWaitAttempts = 5 func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error { - token, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) + token, err := util.ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) if err != nil { return err } - normalizedToken, err := normalizeToken(token) + normalizedToken, err := util.NormalizeToken(token) if err != nil { return err } @@ -52,7 +50,7 @@ func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken return err } - normalizedOldToken, err := normalizeToken(oldToken) + normalizedOldToken, err := util.NormalizeToken(oldToken) if err != nil { return err } @@ -76,13 +74,13 @@ func Save(ctx context.Context, config *config.Control, override bool) error { } token := config.Token if token == "" { - tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) + tokenFromFile, err := util.ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) if err != nil { return err } token = tokenFromFile } - normalizedToken, err := normalizeToken(token) + normalizedToken, err := util.NormalizeToken(token) if err != nil { return err } @@ -165,7 +163,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { token := c.config.Token if token == "" { - tokenFromFile, err := readTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir) + tokenFromFile, err := util.ReadTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir) if err != nil { return err } @@ -181,7 +179,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { } token = tokenFromFile } - normalizedToken, err := normalizeToken(token) + normalizedToken, err := util.NormalizeToken(token) if err != nil { return err } @@ -288,39 +286,6 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client return nil, false, errors.New("bootstrap data already found and encrypted with different token") } -// readTokenFromFile will attempt to get the token from /token if it the file not found -// in case of fresh installation it will try to use the runtime serverToken saved in memory -// after stripping it from any additional information like the username or cahash, if the file -// found then it will still strip the token from any additional info -func readTokenFromFile(serverToken, certs, dataDir string) (string, error) { - tokenFile := filepath.Join(dataDir, "token") - - b, err := os.ReadFile(tokenFile) - if err != nil { - if os.IsNotExist(err) { - token, err := clientaccess.FormatToken(serverToken, certs) - if err != nil { - return token, err - } - return token, nil - } - return "", err - } - - // strip the token from any new line if its read from file - return string(bytes.TrimRight(b, "\n")), nil -} - -// normalizeToken will normalize the token read from file or passed as a cli flag -func normalizeToken(token string) (string, error) { - _, password, ok := clientaccess.ParseUsernamePassword(token) - if !ok { - return password, errors.New("failed to normalize server token; must be in format K10::: or ") - } - - return password, nil -} - // migrateTokens will list all keys that has prefix /bootstrap and will check for key that is // hashed with empty string and keys that is hashed with old token format before normalizing // then migrate those and resave only with the normalized token diff --git a/pkg/codegen/main.go b/pkg/codegen/main.go index e9b6e370de65..afb2d622ec2b 100644 --- a/pkg/codegen/main.go +++ b/pkg/codegen/main.go @@ -74,6 +74,7 @@ func main() { "k3s.cattle.io": { Types: []interface{}{ v1.Addon{}, + v1.ETCDSnapshotFile{}, }, GenerateTypes: true, GenerateClients: true, diff --git a/pkg/crd/crds.go b/pkg/crd/crds.go index 634f555087ea..0a1a918dbe24 100644 --- a/pkg/crd/crds.go +++ b/pkg/crd/crds.go @@ -6,10 +6,19 @@ import ( ) func List() []crd.CRD { - addon := crd.NamespacedType("Addon.k3s.cattle.io/v1"). - WithSchemaFromStruct(v1.Addon{}). - WithColumn("Source", ".spec.source"). - WithColumn("Checksum", ".spec.checksum") - - return []crd.CRD{addon} + addon := v1.Addon{} + etcdSnapshotFile := v1.ETCDSnapshotFile{} + return []crd.CRD{ + crd.NamespacedType("Addon.k3s.cattle.io/v1"). + WithSchemaFromStruct(addon). + WithColumn("Source", ".spec.source"). + WithColumn("Checksum", ".spec.checksum"), + crd.NonNamespacedType("ETCDSnapshotFile.k3s.cattle.io/v1"). + WithSchemaFromStruct(etcdSnapshotFile). + WithColumn("SnapshotName", ".spec.snapshotName"). + WithColumn("Node", ".spec.nodeName"). + WithColumn("Location", ".spec.location"). + WithColumn("Size", ".status.size"). + WithColumn("CreationTime", ".status.creationTime"), + } } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 22dba3ed4a85..25bf0f5cafb3 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io" "github.com/k3s-io/kine/pkg/endpoint" "github.com/rancher/wrangler/pkg/generated/controllers/core" "github.com/rancher/wrangler/pkg/leader" @@ -342,6 +343,7 @@ type ControlRuntime struct { ClientETCDCert string ClientETCDKey string + K3s *k3s.Factory Core *core.Factory Event record.EventRecorder EtcdConfig endpoint.ETCDConfig diff --git a/pkg/daemons/executor/executor.go b/pkg/daemons/executor/executor.go index c8ff45aaff24..e59b81ea99b8 100644 --- a/pkg/daemons/executor/executor.go +++ b/pkg/daemons/executor/executor.go @@ -37,6 +37,7 @@ type ETCDConfig struct { InitialOptions `json:",inline"` Name string `json:"name,omitempty"` ListenClientURLs string `json:"listen-client-urls,omitempty"` + ListenClientHTTPURLs string `json:"listen-client-http-urls,omitempty"` ListenMetricsURLs string `json:"listen-metrics-urls,omitempty"` ListenPeerURLs string `json:"listen-peer-urls,omitempty"` AdvertiseClientURLs string `json:"advertise-client-urls,omitempty"` diff --git a/pkg/deploy/zz_generated_bindata.go b/pkg/deploy/zz_generated_bindata.go index 1e4a5e8f1a35..53803324f925 100644 --- a/pkg/deploy/zz_generated_bindata.go +++ b/pkg/deploy/zz_generated_bindata.go @@ -131,7 +131,7 @@ func corednsYaml() (*asset, error) { return a, nil } -var _localStorageYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x56\x5f\x6f\xdb\xb6\x16\x7f\xd7\xa7\x38\x57\xb7\x79\xb8\x17\xa5\x9d\x6c\x05\x32\xb0\xd8\x83\x9b\x38\x69\x80\xc4\x36\x6c\x77\x43\x51\x14\x06\x2d\x1d\xdb\x6c\x28\x92\x20\x29\xb7\x6a\x96\xef\x3e\x90\x94\x1d\xc9\x71\x13\x07\xdb\xde\xa6\x17\x81\x87\xe7\xef\xef\xfc\x23\xd3\xfc\x37\x34\x96\x2b\x49\x61\x7d\x92\xdc\x72\x99\x53\x98\xa0\x59\xf3\x0c\x7b\x59\xa6\x4a\xe9\x92\x02\x1d\xcb\x99\x63\x34\x01\x90\xac\x40\x0a\x42\x65\x4c\x10\xcd\xdc\x8a\x68\xa3\xd6\xdc\xcb\xa3\x21\x36\xca\x11\x56\x0b\x46\x76\xab\x59\x86\x14\x6e\xcb\x39\x12\x5b\x59\x87\x45\x42\x08\x49\x9a\x96\xcd\x9c\x65\x1d\x56\xba\x95\x32\xfc\x3b\x73\x5c\xc9\xce\xed\x2f\xb6\xc3\x55\x77\xeb\xd3\x99\x28\xad\x43\x33\x56\x02\x0f\x77\xc8\x78\x6e\x53\x0a\xb4\x34\x21\xc0\x34\xbf\x34\xaa\xd4\x96\xc2\xa7\x34\xfd\x9c\x00\x18\xb4\xaa\x34\x19\x06\x8a\x54\x39\xda\xf4\x35\xa4\xda\xbb\x65\x1d\x4a\xb7\x56\xa2\x2c\x30\x13\x8c\x17\xe1\x26\x53\x72\xc1\x97\x05\xd3\x36\x88\xaf\xd1\xcc\x83\xe8\x12\x9d\xbf\x16\xdc\x86\xff\x57\xe6\xb2\x55\xfa\xf9\x79\x93\x28\x73\xad\xb8\x74\x7b\xcd\x46\xa2\xca\x77\x6c\xfd\xff\x20\xc5\x6b\xf4\x5a\x5b\x82\x99\x41\xe6\x30\x28\xdd\xef\x9f\x75\xca\xb0\x25\xd6\xd0\x3f\x56\x5a\xdf\x67\x82\x59\x8b\x07\x22\xf0\x97\x12\xfd\x8e\xcb\x9c\xcb\xe5\xe1\xf9\x9e\x73\x99\x27\x3e\xe9\x63\x5c\x78\xe6\x4d\x78\x4f\x18\x4e\x00\x1e\x17\xd8\x21\x65\x65\xcb\xf9\x17\xcc\x5c\xa8\xac\xbd\x6d\xf3\x4f\x35\x0b\xd3\xda\x3e\xc0\x75\x8e\x5a\xa8\xaa\xc0\x17\xf4\xe9\x8f\x4d\x59\x8d\x19\x0d\x69\x8f\xbc\xef\xb9\xcf\x79\x75\xcd\x0b\xee\x28\x1c\x27\x00\xd6\x19\xe6\x70\x59\x79\x2e\x00\x57\x69\xa4\x30\x56\x42\x70\xb9\xfc\xa0\x73\xe6\x30\xd0\x4d\x93\x12\x59\x01\x0a\xf6\xed\x83\x64\x6b\xc6\x05\x9b\x0b\xa4\x70\xe2\xd5\xa1\xc0\xcc\x29\x13\x79\x0a\x5f\x35\xd7\x6c\x8e\xc2\x6e\x84\x98\xd6\x4f\x84\xe1\xb0\xd0\x62\x6b\xa2\x19\xbf\xff\x44\x4b\xd3\x73\xba\x00\x36\xd1\xfb\x4f\x1b\xae\x0c\x77\xd5\x99\x2f\xf6\x41\x00\x33\x8d\x20\x11\x3f\x27\x48\x66\xb8\xe3\x19\x13\x69\xcd\x6f\x5b\xb9\x1f\xbc\x2c\xf1\x01\x4a\x25\xd0\x84\xc2\x6c\x78\x0c\x40\xe0\x16\x2b\x0a\xe9\x59\x6d\xaf\x97\xe7\x4a\xda\xa1\x14\x55\xda\xe0\x02\x50\xda\x4b\x2b\x43\x21\xed\x7f\xe3\xd6\xd9\x74\x8f\x92\xe0\xb9\x2f\xde\x8e\x4f\xba\x91\xe8\x30\xf4\x5e\xa6\xa4\x33\x4a\x10\x2d\x98\xc4\x17\xe8\x05\xc0\xc5\x02\x33\x47\x21\x1d\xa8\x49\xb6\xc2\xbc\x14\xf8\x12\xc3\x05\xf3\x2d\xf7\x77\x59\xf4\x61\x30\x2e\xd1\x6c\x11\x24\xcf\xf5\x41\xfc\x78\xc1\x96\x48\xe1\xe8\x6e\xf2\x71\x32\xed\xdf\xcc\xce\xfb\x17\xbd\x0f\xd7\xd3\xd9\xb8\x7f\x79\x35\x99\x8e\x3f\xde\x1f\x19\x26\xb3\x15\x9a\xee\x7e\x45\x74\x7d\xdc\x39\xee\xfc\xf4\xa6\xad\x70\x54\x0a\x31\x52\x82\x67\x15\x85\xab\xc5\x40\xb9\x91\x41\x8b\xdb\x84\x7b\x7f\x8b\x82\xc9\xfc\x21\xdd\xe4\x39\x47\x09\x58\xc7\x8c\x6b\x9c\x09\x89\x3b\xa9\x41\xea\xa2\xcb\xba\x91\x5a\xff\x3a\x5f\xac\x92\x5b\x8e\xb8\x5d\x6e\x7c\xed\xd9\xa6\xed\x08\x55\x94\x20\x91\xa9\x81\x7c\xe1\xf9\x47\xcc\xad\x68\xcb\xc0\x96\x03\xe5\xfa\xb1\xb2\xd1\xf0\x7c\x36\xe8\xdd\xf4\x27\xa3\xde\x59\xbf\xa1\x6c\xcd\x44\x89\x17\x46\x15\xb4\x95\xdb\x05\x47\x91\xd7\xa3\xfb\x11\x3d\xda\xde\xf4\x78\x67\x3b\xc1\x92\x66\x54\x2f\x08\x28\xd2\x6f\x98\x6e\x5b\x7b\x54\x30\x35\xbe\xbb\x53\xb8\xbd\x2c\x1f\xe6\xf1\x24\xd2\xc3\xdc\x78\x72\x22\xfb\xf5\x24\xa5\x72\xcd\x9e\x6f\x6e\xd8\x9d\x56\xe1\x96\xe4\xb8\x60\xa5\x70\x24\x5c\x53\x48\x9d\x29\x31\x4d\x9a\x75\x08\x75\x9d\x7a\x81\x86\xa5\x18\x7b\xbd\x4d\x6f\x54\x8e\x14\x7e\x67\xdc\x5d\x28\x73\xc1\x8d\x75\x67\x4a\xda\xb2\x40\x93\x98\xf8\xd4\xd9\x14\xed\x39\x0a\x74\x18\x22\xaf\x57\xe4\x06\xb2\x64\xe7\xd9\xf8\xe4\xe6\xd9\x16\xe8\x0f\x96\xce\x46\xb0\x51\xab\x14\xfe\x20\x01\x90\xbb\x3a\x37\x61\x82\xf8\x0a\xb8\x61\x3a\xa5\x9f\x6a\xea\xdd\x36\x73\xe1\x3e\xa5\xe9\xa6\x73\x47\xbd\xe9\xfb\xd9\xc5\x70\x3c\x1b\x0c\x07\xb3\xeb\xab\xc9\xb4\x7f\x3e\x1b\x0c\xcf\xfb\x93\xf4\xf5\x83\x8c\xf7\xce\xa6\xf4\x53\x7a\x74\xb7\x91\xbb\x1e\x9e\xf5\xae\x67\x93\xe9\x70\xdc\xbb\xec\x07\x2d\xf7\x47\xe1\xa1\xe3\xbf\xfb\xfa\x1f\xcf\xf7\x61\x7d\x39\xff\xb8\xa8\x9d\xfd\xef\x7f\xba\x73\x2e\xbb\x76\x15\x4e\x5f\x57\x5c\x20\x2c\xd1\x29\xed\x2c\xa4\x05\xb5\x54\xd3\x14\x94\x8e\xed\x9b\xab\x87\x39\xc0\x2c\xc2\x2b\xa5\x1d\x70\xd9\xaa\x45\xfd\xbf\xd6\x91\xcd\xad\x12\xa5\x0b\x38\xfc\xfa\x6a\x38\x9a\xf6\xc6\x97\x2d\x86\xb7\x6f\x5b\x47\xdb\x16\xb7\xfc\x3b\x5e\xc9\x77\x95\x43\x7b\x88\x74\xd1\x96\x5e\x2b\xe1\x2b\xe7\x39\x49\xb4\x2c\xab\xe3\x93\xb1\xdb\x8a\xdb\x9c\x1b\x20\x05\x1c\x9f\x9e\x9e\x02\xd1\xf0\xea\xae\x19\x48\x04\x35\x5b\x15\x2a\x87\xd3\xe3\xe3\xdd\xdb\x6e\xa7\x13\xf6\x3c\x33\xb9\xfa\x2a\xff\x85\xfa\x49\xa8\x4d\x01\xc4\x2c\xf6\x00\xbc\x42\xa1\xd1\x8c\x54\xde\xa9\x58\x21\xb6\x28\xee\x74\xb1\x27\xc5\x46\x1f\xa9\x7c\xef\x8b\x2a\xf6\x76\xd4\x46\x74\xcd\xd4\x7c\x36\xfd\x78\x05\xef\x08\xc1\x8b\xd6\x6e\xc1\x8d\x51\x06\x73\x22\xf8\xdc\x30\x53\x91\x79\x69\xab\xb9\xfa\x46\x4f\x3a\x3f\xbf\xe9\x9c\x1c\xb8\x77\xff\x0c\x00\x00\xff\xff\x7c\x3e\x44\xe7\xec\x0e\x00\x00") +var _localStorageYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x56\x5f\x6f\xdb\xb6\x16\x7f\xd7\xa7\x38\x57\xb7\x79\xb8\x17\xa5\x9d\x6c\xc3\x32\xb0\xd8\x83\x9b\x38\x69\x80\xc4\x36\x6c\x77\x43\x51\x14\x06\x2d\x1d\xdb\x6c\x28\x92\x20\x29\xb7\x6a\x96\xef\x3e\x90\x94\x1d\xc9\x71\x13\x07\xdb\xde\xa6\x17\x81\x87\xe7\xef\xef\xfc\x23\xd3\xfc\x37\x34\x96\x2b\x49\x61\x7d\x92\xdc\x72\x99\x53\x98\xa0\x59\xf3\x0c\x7b\x59\xa6\x4a\xe9\x92\x02\x1d\xcb\x99\x63\x34\x01\x90\xac\x40\x0a\x42\x65\x4c\x10\xcd\xdc\x8a\x68\xa3\xd6\xdc\xcb\xa3\x21\x36\xca\x11\x56\x0b\x46\x76\xab\x59\x86\x14\x6e\xcb\x39\x12\x5b\x59\x87\x45\x42\x08\x49\x9a\x96\xcd\x9c\x65\x1d\x56\xba\x95\x32\xfc\x1b\x73\x5c\xc9\xce\xed\x2f\xb6\xc3\x55\x77\xeb\xd3\x99\x28\xad\x43\x33\x56\x02\x0f\x77\xc8\x78\x6e\x53\x0a\xb4\x34\x21\xc0\x34\xbf\x34\xaa\xd4\x96\xc2\xc7\x34\xfd\x94\x00\x18\xb4\xaa\x34\x19\x06\x8a\x54\x39\xda\xf4\x35\xa4\xda\xbb\x65\x1d\x4a\xb7\x56\xa2\x2c\x30\x13\x8c\x17\xe1\x26\x53\x72\xc1\x97\x05\xd3\x36\x88\xaf\xd1\xcc\x83\xe8\x12\x9d\xbf\x16\xdc\x86\xff\x17\xe6\xb2\x55\xfa\xe9\x79\x93\x28\x73\xad\xb8\x74\x7b\xcd\x46\xa2\xca\x77\x6c\xfd\xff\x20\xc5\x6b\xf4\x5a\x5b\x82\x99\x41\xe6\x30\x28\xdd\xef\x9f\x75\xca\xb0\x25\xd6\xd0\x3f\x56\x5a\xdf\x67\x82\x59\x8b\x07\x22\xf0\x97\x12\xfd\x96\xcb\x9c\xcb\xe5\xe1\xf9\x9e\x73\x99\x27\x3e\xe9\x63\x5c\x78\xe6\x4d\x78\x4f\x18\x4e\x00\x1e\x17\xd8\x21\x65\x65\xcb\xf9\x67\xcc\x5c\xa8\xac\xbd\x6d\xf3\x4f\x35\x0b\xd3\xda\x3e\xc0\x75\x8e\x5a\xa8\xaa\xc0\x17\xf4\xe9\xf7\x4d\x59\x8d\x19\x0d\x69\x8f\xbc\xef\xb8\xcf\x79\x75\xcd\x0b\xee\x28\x1c\x27\x00\xd6\x19\xe6\x70\x59\x79\x2e\x00\x57\x69\xa4\x30\x56\x42\x70\xb9\x7c\xaf\x73\xe6\x30\xd0\x4d\x93\x12\x59\x01\x0a\xf6\xf5\xbd\x64\x6b\xc6\x05\x9b\x0b\xa4\x70\xe2\xd5\xa1\xc0\xcc\x29\x13\x79\x0a\x5f\x35\xd7\x6c\x8e\xc2\x6e\x84\x98\xd6\x4f\x84\xe1\xb0\xd0\x62\x6b\xa2\x19\xbf\xff\x44\x4b\xd3\x73\xba\x00\x36\xd1\xfb\x4f\x1b\xae\x0c\x77\xd5\x99\x2f\xf6\x41\x00\x33\x8d\x20\x11\x3f\x27\x48\x66\xb8\xe3\x19\x13\x69\xcd\x6f\x5b\xb9\x1f\xbc\x2c\xf1\x01\x4a\x25\xd0\x84\xc2\x6c\x78\x0c\x40\xe0\x16\x2b\x0a\xe9\x59\x6d\xaf\x97\xe7\x4a\xda\xa1\x14\x55\xda\xe0\x02\x50\xda\x4b\x2b\x43\x21\xed\x7f\xe5\xd6\xd9\x74\x8f\x92\xe0\xb9\x2f\xde\x8e\x4f\xba\x91\xe8\x30\xf4\x5e\xa6\xa4\x33\x4a\x10\x2d\x98\xc4\x17\xe8\x05\xc0\xc5\x02\x33\x47\x21\x1d\xa8\x49\xb6\xc2\xbc\x14\xf8\x12\xc3\x05\xf3\x2d\xf7\x77\x59\xf4\x61\x30\x2e\xd1\x6c\x11\x24\xcf\xf5\x41\xfc\x78\xc1\x96\x48\xe1\xe8\x6e\xf2\x61\x32\xed\xdf\xcc\xce\xfb\x17\xbd\xf7\xd7\xd3\xd9\xb8\x7f\x79\x35\x99\x8e\x3f\xdc\x1f\x19\x26\xb3\x15\x9a\xee\x7e\x45\x74\x7d\xdc\x39\xee\xfc\xf0\x53\x5b\xe1\xa8\x14\x62\xa4\x04\xcf\x2a\x0a\x57\x8b\x81\x72\x23\x83\x16\xb7\x09\xf7\xfe\x16\x05\x93\xf9\x43\xba\xc9\x73\x8e\x12\xb0\x8e\x19\xd7\x38\x13\x12\x77\x52\x83\xd4\x45\x97\x75\x23\xb5\xfe\x75\x3e\x5b\x25\xb7\x1c\x71\xbb\xdc\xf8\xda\xb3\x4d\xdb\x11\xaa\x28\x41\x22\x53\x03\xf9\xc2\xf3\x8f\x98\x5b\xd1\x96\x81\x2d\x07\xca\xf5\x63\x65\xa3\xe1\xf9\x6c\xd0\xbb\xe9\x4f\x46\xbd\xb3\x7e\x43\xd9\x9a\x89\x12\x2f\x8c\x2a\x68\x2b\xb7\x0b\x8e\x22\xaf\x47\xf7\x23\x7a\xb4\xbd\xe9\xf1\xce\x76\x82\x25\xcd\xa8\x5e\x10\x50\xa4\xdf\x30\xdd\xb6\xf6\xa8\x60\x6a\x7c\x77\xa7\x70\x7b\x59\x3e\xcc\xe3\x49\xa4\x87\xb9\xf1\xe4\x44\xf6\xeb\x49\x4a\xe5\x9a\x3d\xdf\xdc\xb0\x3b\xad\xc2\x2d\xc9\x71\xc1\x4a\xe1\x48\xb8\xa6\x90\x3a\x53\x62\x9a\x34\xeb\x10\xea\x3a\xf5\x02\x0d\x4b\x31\xf6\x7a\x9b\xde\xa8\x1c\x29\xfc\xce\xb8\xbb\x50\xe6\x82\x1b\xeb\xce\x94\xb4\x65\x81\x26\x31\xf1\xa9\xb3\x29\xda\x73\x14\xe8\x30\x44\x5e\xaf\xc8\x0d\x64\xc9\xce\xb3\xf1\xc9\xcd\xb3\x2d\xd0\xef\x2c\x9d\x8d\x60\xa3\x56\x29\xfc\x41\x02\x20\x77\x75\x6e\xc2\x04\xf1\x15\x70\xc3\x74\x4a\x3f\xd6\xd4\xbb\x6d\xe6\xc2\x7d\x4a\xd3\x4d\xe7\x8e\x7a\xd3\x77\xb3\x8b\xe1\x78\x36\x18\x0e\x66\xd7\x57\x93\x69\xff\x7c\x36\x18\x9e\xf7\x27\xe9\xeb\x07\x19\xef\x9d\x4d\xe9\xc7\xf4\xe8\x6e\x23\x77\x3d\x3c\xeb\x5d\xcf\x26\xd3\xe1\xb8\x77\xd9\x0f\x5a\xee\x8f\xc2\x43\xc7\x7f\xf7\xf5\x3f\x9e\xef\xc3\xfa\x72\xfe\x71\x51\x3b\xfb\xdf\xff\x74\xe7\x5c\x76\xed\x2a\x9c\xbe\xac\xb8\x40\x58\xa2\x53\xda\x59\x48\x0b\x6a\xa9\xa6\x29\x28\x1d\xdb\x37\x57\x0f\x73\x80\x59\x84\x57\x4a\x3b\xe0\xb2\x55\x8b\xfa\x7f\xad\x23\x9b\x5b\x25\x4a\x17\x70\xf8\xf5\xd5\x70\x34\xed\x8d\x2f\x5b\x0c\x6f\xde\xb4\x8e\xb6\x2d\x6e\xf9\x37\xbc\x92\x6f\x2b\x87\xf6\x10\xe9\xa2\x2d\xbd\x56\xc2\x57\xce\x73\x92\x68\x59\x56\xc7\x27\x63\xb7\x15\xb7\x39\x37\x40\x0a\x38\x3e\x3d\x3d\x05\xa2\xe1\xd5\x5d\x33\x90\x08\x6a\xb6\x2a\x54\x0e\xa7\xc7\xc7\xbb\xb7\xdd\x4e\x27\xec\x79\x66\x72\xf5\x45\xfe\x0b\xf5\x93\x50\x9b\x02\x88\x59\xec\x01\x78\x85\x42\xa3\x19\xa9\xbc\x53\xb1\x42\x6c\x51\xdc\xe9\x62\x4f\x8a\x8d\x3e\x52\xf9\xde\x17\x55\xec\xed\xa8\x8d\xe8\x9a\xa9\xf9\x6c\xfa\xfe\x0a\xde\x11\x82\x17\xad\xdd\x82\x1b\xa3\x0c\xe6\x44\xf0\xb9\x61\xa6\x22\xf3\xd2\x56\x73\xf5\x95\x9e\x74\x7e\xfc\xb9\x73\x72\xe0\xde\xfd\x33\x00\x00\xff\xff\xe1\x31\x46\x7e\xec\x0e\x00\x00") func localStorageYamlBytes() ([]byte, error) { return bindataRead( @@ -311,7 +311,7 @@ func rolebindingsYaml() (*asset, error) { return a, nil } -var _traefikYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x91\x5f\x6f\xdb\x3a\x0c\xc5\xdf\xfd\x29\x08\x03\x79\xba\x90\xdd\xe4\xa9\xd7\x6f\xb9\xa9\x7b\x57\x6c\xeb\x8a\x38\xdd\xd0\xa7\x80\x91\x99\x58\x88\x2c\x09\x14\x1d\x2c\xeb\xfa\xdd\x07\x25\xe9\x3f\xa0\xc0\x86\x61\x7b\x13\x44\xf2\x77\xc8\x73\x94\x52\x19\x06\xf3\x99\x38\x1a\xef\x2a\xe8\xc8\xf6\x85\x46\x11\x4b\x85\xf1\xe5\x6e\x9c\x6d\x8d\x6b\x2b\x78\x47\xb6\x9f\x75\xc8\x92\xf5\x24\xd8\xa2\x60\x95\x01\x38\xec\xa9\x02\x61\xa4\xb5\xd9\x2a\xcd\xed\xe9\x2f\x06\xd4\x54\xc1\x76\x58\x91\x8a\xfb\x28\xd4\x67\x31\x90\x4e\x23\x3a\x41\x2a\xe8\x44\x42\xac\xca\x72\x74\xff\xfe\xf6\xbf\x7a\x7e\x5d\x2f\xea\x66\x39\xbd\xb9\x7a\x18\x95\x51\x50\x8c\x2e\x0f\x8d\xb1\x7c\x01\x57\x93\x71\x31\x29\xc6\xff\x0c\xe1\xf0\x38\x2b\x64\xf3\x2d\xfb\x83\x07\xfc\xbd\xe5\xdf\x5a\x1c\x20\x92\x24\x28\xc0\xc6\xfa\x15\xda\xe2\x28\x76\x41\x6b\x1c\xac\xcc\x69\x63\xa2\xf0\xbe\x82\x7c\x74\xdf\xdc\x35\x8b\xfa\xe3\xf2\xa2\xbe\x9c\xde\x7e\x58\x2c\xe7\xf5\xff\x57\xcd\x62\x7e\xb7\x9c\x4f\xbf\x3c\x8c\xf2\x0c\x60\x87\x76\xa0\x38\xf3\x4e\xc8\x49\x05\xdf\xd5\x81\x1b\x7c\x3b\x75\xce\xa7\x95\xbc\x8b\x47\x2d\x80\xc0\xbe\x27\xe9\x68\x88\xc9\xa0\xe0\xd3\x45\xf9\xf9\xd9\xf9\x24\x7f\xb3\x21\x6a\xc6\x40\x15\xe4\xc2\x03\x1d\x5b\x02\xfb\x9d\x69\x89\x9f\x90\xc9\x2b\x76\x24\x14\xaf\xdc\x86\x29\x3e\x15\x00\xc2\xb0\xb2\x26\x76\xd4\x36\xc4\x3b\xa3\xe9\xb9\x02\x40\x0e\x57\x96\xda\x14\xc0\x40\x27\xb2\xf1\x6c\x64\x3f\xb3\x18\xe3\xf5\x21\x9c\xfc\x68\x8b\xd2\x76\x88\x42\xac\x34\x1b\x31\x1a\xed\x71\x15\xd3\xe3\xe6\x89\xc9\x14\x7c\x34\xe2\x0f\xae\x31\x3a\xdd\x11\x97\xbd\x61\xf6\x4c\xad\xb2\x66\xc5\xc8\x7b\x75\x0a\xe5\xf1\x5a\xc1\x4d\x05\xf9\xa4\xf8\xb7\x18\x9f\x1d\xff\xc4\x5b\xe2\x97\x9e\x29\xd8\x52\x42\xce\x4e\xd2\xd3\xb6\xf5\x2e\x7e\x72\x76\xff\x08\xf1\x21\x4d\x78\xae\x20\xaf\xbf\x9a\x28\x31\x7f\x35\xe8\x7c\x4b\x8a\xbd\xa5\xe2\xd9\xa9\xe4\xad\xf6\x4e\xd8\x5b\x15\x2c\x3a\xfa\x09\x0b\x80\xd6\x6b\xd2\x29\xac\x6b\xdf\xe8\x8e\xda\xc1\xd2\xaf\xc9\xf4\x98\x9c\xfb\x7d\x7e\x7c\x1d\x9d\x09\x97\xd8\x1b\xbb\xbf\xf1\xd6\xe8\xa4\x7b\xc3\xb4\x26\xbe\x18\xd0\x36\x82\x7a\x9b\x67\x3f\x02\x00\x00\xff\xff\x12\x80\xc2\x85\x56\x04\x00\x00") +var _traefikYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x91\x5f\x6b\xdb\x4a\x10\xc5\xdf\xf5\x29\x06\x81\x9f\x2e\x2b\xc5\x86\x0b\x41\x6f\xbe\x8e\x72\x1b\xda\xa6\xc1\x72\x5a\xf2\x64\xc6\xab\xb1\xb5\x78\xb5\xbb\xcc\x8e\x4c\xdd\x34\xdf\xbd\xac\xed\xfc\x83\x40\x4b\x69\xdf\x96\x9d\x99\xdf\x99\x39\x47\x29\x95\x61\x30\x9f\x89\xa3\xf1\xae\x82\x8e\x6c\x5f\x68\x14\xb1\x54\x18\x5f\xee\xc6\xd9\xd6\xb8\xb6\x82\x77\x64\xfb\x59\x87\x2c\x59\x4f\x82\x2d\x0a\x56\x19\x80\xc3\x9e\x2a\x10\x46\x5a\x9b\xad\xd2\xdc\x9e\xfe\x62\x40\x4d\x15\x6c\x87\x15\xa9\xb8\x8f\x42\x7d\x16\x03\xe9\x34\xa2\x13\xa4\x82\x4e\x24\xc4\xaa\x2c\x47\xf7\xef\x6f\xff\xab\xe7\xd7\xf5\xa2\x6e\x96\xd3\x9b\xab\x87\x51\x19\x05\xc5\xe8\xf2\xd0\x18\xcb\x17\x70\x35\x19\x17\x93\x62\xfc\xcf\x10\x0e\x8f\xb3\x42\x36\xdf\xb2\x3f\x78\xc0\xdf\x5b\xfe\xad\xc5\x01\x22\x49\x82\x02\x6c\xac\x5f\xa1\x2d\x8e\x62\x17\xb4\xc6\xc1\xca\x9c\x36\x26\x0a\xef\x2b\xc8\x47\xf7\xcd\x5d\xb3\xa8\x3f\x2e\x2f\xea\xcb\xe9\xed\x87\xc5\x72\x5e\xff\x7f\xd5\x2c\xe6\x77\xcb\xf9\xf4\xcb\xc3\x28\xcf\x00\x76\x68\x07\x8a\x33\xef\x84\x9c\x54\xf0\x5d\x1d\xb8\xc1\xb7\x53\xe7\x7c\x5a\xc9\xbb\x78\xd4\x02\x08\xec\x7b\x92\x8e\x86\x98\x0c\x0a\x3e\x5d\x94\x9f\x9f\x9d\x4f\xf2\x37\x1b\xa2\x66\x0c\x54\x41\x2e\x3c\xd0\xb1\x25\xb0\xdf\x99\x96\xf8\x09\x99\xbc\x62\x47\x42\xf1\xca\x6d\x98\xe2\x53\x01\x20\x0c\x2b\x6b\x62\x47\x6d\x43\xbc\x33\x9a\x9e\x2b\x00\xe4\x70\x65\xa9\x4d\x01\x0c\x74\x22\x1b\xcf\x46\xf6\x33\x8b\x31\x5e\x1f\xc2\xc9\x8f\xb6\x28\x6d\x87\x28\xc4\x4a\xb3\x11\xa3\xd1\x1e\x57\x31\x3d\x6e\x9e\x98\x4c\xc1\x47\x23\xfe\xe0\x1a\xa3\xd3\x1d\x71\xd9\x1b\x66\xcf\xd4\x2a\x6b\x56\x8c\xbc\x57\xa7\x50\x1e\xaf\x15\xdc\x54\x90\x4f\x8a\xf1\x59\xf1\xef\xf1\x4f\xbc\x25\x7e\xe9\x99\x82\x2d\x25\xe4\xec\x24\x3d\x6d\x5b\xef\xe2\x27\x67\xf7\x8f\x10\x1f\xd2\x84\xe7\x0a\xf2\xfa\xab\x89\x12\xf3\x57\x83\xce\xb7\xa4\xd8\x5b\x2a\x9e\x9d\x4a\xde\x6a\xef\x84\xbd\x55\xc1\xa2\xa3\x9f\xb0\x00\x68\xbd\x26\x9d\xc2\xba\xf6\x8d\xee\xa8\x1d\x2c\xfd\x9a\x4c\x8f\xc9\xb9\xdf\xe7\xc7\xd7\xd1\x99\x70\x89\xbd\xb1\xfb\x1b\x6f\x8d\x4e\xba\x37\x4c\x6b\xe2\x8b\x01\x6d\x23\xa8\xb7\x79\xf6\x23\x00\x00\xff\xff\xb9\x01\x23\x5a\x56\x04\x00\x00") func traefikYamlBytes() ([]byte, error) { return bindataRead( diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 07dc1bcaccbd..fc3894b37fd3 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -1,23 +1,18 @@ package etcd import ( - "archive/zip" "bytes" "context" "crypto/tls" - "encoding/base64" "encoding/json" "fmt" - "io" "io/fs" - "math/rand" "net" "net/http" "net/url" "os" "path/filepath" "regexp" - "runtime" "sort" "strconv" "strings" @@ -33,7 +28,6 @@ import ( "github.com/k3s-io/k3s/pkg/version" "github.com/k3s-io/kine/pkg/client" endpoint2 "github.com/k3s-io/kine/pkg/endpoint" - "github.com/minio/minio-go/v7" cp "github.com/otiai10/copy" "github.com/pkg/errors" certutil "github.com/rancher/dynamiclistener/cert" @@ -47,12 +41,9 @@ import ( "go.etcd.io/etcd/etcdutl/v3/snapshot" "go.uber.org/zap" "golang.org/x/sync/semaphore" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" ) const ( @@ -71,9 +62,7 @@ const ( defaultKeepAliveTime = 30 * time.Second defaultKeepAliveTimeout = 10 * time.Second - maxBackupRetention = 5 - maxConcurrentSnapshots = 1 - compressedExtension = ".zip" + maxBackupRetention = 5 ) var ( @@ -81,22 +70,6 @@ var ( // AddressKey will contain the value of api addresses list AddressKey = version.Program + "/apiaddresses" - snapshotExtraMetadataConfigMapName = version.Program + "-etcd-snapshot-extra-metadata" - snapshotConfigMapName = version.Program + "-etcd-snapshots" - - // snapshotDataBackoff will retry at increasing steps for up to ~30 seconds. - // If the ConfigMap update fails, the list won't be reconciled again until next time - // the server starts, so we should be fairly persistent in retrying. - snapshotDataBackoff = wait.Backoff{ - Steps: 9, - Duration: 10 * time.Millisecond, - Factor: 3.0, - Jitter: 0.1, - } - - // cronLogger wraps logrus's Printf output as cron-compatible logger - cronLogger = cron.VerbosePrintfLogger(logrus.StandardLogger()) - NodeNameAnnotation = "etcd." + version.Program + ".cattle.io/node-name" NodeAddressAnnotation = "etcd." + version.Program + ".cattle.io/node-address" @@ -603,6 +576,7 @@ func (e *ETCD) Register(handler http.Handler) (http.Handler, error) { e.config.Runtime.LeaderElectedClusterControllerStarts[version.Program+"-etcd"] = func(ctx context.Context) { registerEndpointsHandlers(ctx, e) registerMemberHandlers(ctx, e) + registerSnapshotHandlers(ctx, e) } } @@ -881,6 +855,14 @@ func (e *ETCD) listenMetricsURLs(reset bool) string { return metricsURLs } +// listenClientHTTPURLs returns a list of URLs to bind to for http client connections. +// This should no longer be used, but we must set it in order to free the listen URLs +// for dedicated use by GRPC. +// Ref: https://github.com/etcd-io/etcd/issues/15402 +func (e *ETCD) listenClientHTTPURLs() string { + return fmt.Sprintf("https://%s:2382", e.config.Loopback(true)) +} + // cluster calls the executor to start etcd running with the provided configuration. func (e *ETCD) cluster(ctx context.Context, reset bool, options executor.InitialOptions) error { ctx, e.cancel = context.WithCancel(ctx) @@ -911,6 +893,7 @@ func (e *ETCD) cluster(ctx context.Context, reset bool, options executor.Initial Logger: "zap", LogOutputs: []string{"stderr"}, ExperimentalInitialCorruptCheck: true, + ListenClientHTTPURLs: e.listenClientHTTPURLs(), }, e.config.ExtraEtcdArgs) } @@ -949,10 +932,16 @@ func (e *ETCD) StartEmbeddedTemporary(ctx context.Context) error { endpoints := getEndpoints(e.config) clientURL := endpoints[0] + // peer URL is usually 1 more than client peerURL, err := addPort(endpoints[0], 1) if err != nil { return err } + // client http URL is usually 3 more than client, after peer and metrics + clientHTTPURL, err := addPort(endpoints[0], 3) + if err != nil { + return err + } embedded := executor.Embedded{} ctx, e.cancel = context.WithCancel(ctx) @@ -962,6 +951,7 @@ func (e *ETCD) StartEmbeddedTemporary(ctx context.Context) error { ForceNewCluster: true, AdvertiseClientURLs: clientURL, ListenClientURLs: clientURL, + ListenClientHTTPURLs: clientHTTPURL, ListenPeerURLs: peerURL, Logger: "zap", HeartbeatInterval: 500, @@ -1236,803 +1226,6 @@ members: return clientURLs, memberList, nil } -// snapshotDir ensures that the snapshot directory exists, and then returns its path. -func snapshotDir(config *config.Control, create bool) (string, error) { - if config.EtcdSnapshotDir == "" { - // we have to create the snapshot dir if we are using - // the default snapshot dir if it doesn't exist - defaultSnapshotDir := filepath.Join(config.DataDir, "db", "snapshots") - s, err := os.Stat(defaultSnapshotDir) - if err != nil { - if create && os.IsNotExist(err) { - if err := os.MkdirAll(defaultSnapshotDir, 0700); err != nil { - return "", err - } - return defaultSnapshotDir, nil - } - return "", err - } - if s.IsDir() { - return defaultSnapshotDir, nil - } - } - return config.EtcdSnapshotDir, nil -} - -// preSnapshotSetup checks to see if the necessary components are in place -// to perform an Etcd snapshot. This is necessary primarily for on-demand -// snapshots since they're performed before normal Etcd setup is completed. -func (e *ETCD) preSnapshotSetup(ctx context.Context) error { - if e.snapshotSem == nil { - e.snapshotSem = semaphore.NewWeighted(maxConcurrentSnapshots) - } - return nil -} - -// compressSnapshot compresses the given snapshot and provides the -// caller with the path to the file. -func (e *ETCD) compressSnapshot(snapshotDir, snapshotName, snapshotPath string) (string, error) { - logrus.Info("Compressing etcd snapshot file: " + snapshotName) - - zippedSnapshotName := snapshotName + compressedExtension - zipPath := filepath.Join(snapshotDir, zippedSnapshotName) - - zf, err := os.Create(zipPath) - if err != nil { - return "", err - } - defer zf.Close() - - zipWriter := zip.NewWriter(zf) - defer zipWriter.Close() - - uncompressedPath := filepath.Join(snapshotDir, snapshotName) - fileToZip, err := os.Open(uncompressedPath) - if err != nil { - os.Remove(zipPath) - return "", err - } - defer fileToZip.Close() - - info, err := fileToZip.Stat() - if err != nil { - os.Remove(zipPath) - return "", err - } - - header, err := zip.FileInfoHeader(info) - if err != nil { - os.Remove(zipPath) - return "", err - } - - header.Name = snapshotName - header.Method = zip.Deflate - header.Modified = time.Now() - - writer, err := zipWriter.CreateHeader(header) - if err != nil { - os.Remove(zipPath) - return "", err - } - _, err = io.Copy(writer, fileToZip) - - return zipPath, err -} - -// decompressSnapshot decompresses the given snapshot and provides the caller -// with the full path to the uncompressed snapshot. -func (e *ETCD) decompressSnapshot(snapshotDir, snapshotFile string) (string, error) { - logrus.Info("Decompressing etcd snapshot file: " + snapshotFile) - - r, err := zip.OpenReader(snapshotFile) - if err != nil { - return "", err - } - defer r.Close() - - var decompressed *os.File - for _, sf := range r.File { - decompressed, err = os.OpenFile(strings.Replace(sf.Name, compressedExtension, "", -1), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, sf.Mode()) - if err != nil { - return "", err - } - defer decompressed.Close() - - ss, err := sf.Open() - if err != nil { - return "", err - } - defer ss.Close() - - if _, err := io.Copy(decompressed, ss); err != nil { - os.Remove("") - return "", err - } - } - - return decompressed.Name(), nil -} - -// Snapshot attempts to save a new snapshot to the configured directory, and then clean up any old and failed -// snapshots in excess of the retention limits. This method is used in the internal cron snapshot -// system as well as used to do on-demand snapshots. -func (e *ETCD) Snapshot(ctx context.Context) error { - if err := e.preSnapshotSetup(ctx); err != nil { - return err - } - if !e.snapshotSem.TryAcquire(maxConcurrentSnapshots) { - return fmt.Errorf("%d snapshots already in progress", maxConcurrentSnapshots) - } - defer e.snapshotSem.Release(maxConcurrentSnapshots) - - // make sure the core.Factory is initialized before attempting to add snapshot metadata - var extraMetadata string - if e.config.Runtime.Core == nil { - logrus.Debugf("Cannot retrieve extra metadata from %s ConfigMap: runtime core not ready", snapshotExtraMetadataConfigMapName) - } else { - logrus.Debugf("Attempting to retrieve extra metadata from %s ConfigMap", snapshotExtraMetadataConfigMapName) - if snapshotExtraMetadataConfigMap, err := e.config.Runtime.Core.Core().V1().ConfigMap().Get(metav1.NamespaceSystem, snapshotExtraMetadataConfigMapName, metav1.GetOptions{}); err != nil { - logrus.Debugf("Error encountered attempting to retrieve extra metadata from %s ConfigMap, error: %v", snapshotExtraMetadataConfigMapName, err) - } else { - if m, err := json.Marshal(snapshotExtraMetadataConfigMap.Data); err != nil { - logrus.Debugf("Error attempting to marshal extra metadata contained in %s ConfigMap, error: %v", snapshotExtraMetadataConfigMapName, err) - } else { - logrus.Debugf("Setting extra metadata from %s ConfigMap", snapshotExtraMetadataConfigMapName) - logrus.Tracef("Marshalled extra metadata in %s ConfigMap was: %s", snapshotExtraMetadataConfigMapName, string(m)) - extraMetadata = base64.StdEncoding.EncodeToString(m) - } - } - } - - endpoints := getEndpoints(e.config) - var client *clientv3.Client - var err error - - // Use the internal client if possible, or create a new one - // if run from the CLI. - if e.client != nil { - client = e.client - } else { - client, err = getClient(ctx, e.config, endpoints...) - if err != nil { - return err - } - defer client.Close() - } - - status, err := client.Status(ctx, endpoints[0]) - if err != nil { - return errors.Wrap(err, "failed to check etcd status for snapshot") - } - - if status.IsLearner { - logrus.Warnf("Unable to take snapshot: not supported for learner") - return nil - } - - snapshotDir, err := snapshotDir(e.config, true) - if err != nil { - return errors.Wrap(err, "failed to get the snapshot dir") - } - - cfg, err := getClientConfig(ctx, e.config) - if err != nil { - return errors.Wrap(err, "failed to get config for etcd snapshot") - } - - nodeName := os.Getenv("NODE_NAME") - now := time.Now() - snapshotName := fmt.Sprintf("%s-%s-%d", e.config.EtcdSnapshotName, nodeName, now.Unix()) - snapshotPath := filepath.Join(snapshotDir, snapshotName) - - logrus.Infof("Saving etcd snapshot to %s", snapshotPath) - - var sf *snapshotFile - - lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) - if err != nil { - return err - } - - if err := snapshot.NewV3(lg).Save(ctx, *cfg, snapshotPath); err != nil { - sf = &snapshotFile{ - Name: snapshotName, - Location: "", - Metadata: extraMetadata, - NodeName: nodeName, - CreatedAt: &metav1.Time{ - Time: now, - }, - Status: failedSnapshotStatus, - Message: base64.StdEncoding.EncodeToString([]byte(err.Error())), - Size: 0, - Compressed: e.config.EtcdSnapshotCompress, - } - logrus.Errorf("Failed to take etcd snapshot: %v", err) - if err := e.addSnapshotData(*sf); err != nil { - return errors.Wrap(err, "failed to save local snapshot failure data to configmap") - } - } - - if e.config.EtcdSnapshotCompress { - zipPath, err := e.compressSnapshot(snapshotDir, snapshotName, snapshotPath) - if err != nil { - return err - } - if err := os.Remove(snapshotPath); err != nil { - return err - } - snapshotPath = zipPath - logrus.Info("Compressed snapshot: " + snapshotPath) - } - - // If the snapshot attempt was successful, sf will be nil as we did not set it. - if sf == nil { - f, err := os.Stat(snapshotPath) - if err != nil { - return errors.Wrap(err, "unable to retrieve snapshot information from local snapshot") - } - sf = &snapshotFile{ - Name: f.Name(), - Metadata: extraMetadata, - Location: "file://" + snapshotPath, - NodeName: nodeName, - CreatedAt: &metav1.Time{ - Time: f.ModTime(), - }, - Status: successfulSnapshotStatus, - Size: f.Size(), - Compressed: e.config.EtcdSnapshotCompress, - } - - if err := e.addSnapshotData(*sf); err != nil { - return errors.Wrap(err, "failed to save local snapshot data to configmap") - } - if err := snapshotRetention(e.config.EtcdSnapshotRetention, e.config.EtcdSnapshotName, snapshotDir); err != nil { - return errors.Wrap(err, "failed to apply local snapshot retention policy") - } - - if e.config.EtcdS3 { - logrus.Infof("Saving etcd snapshot %s to S3", snapshotName) - // Set sf to nil so that we can attempt to now upload the snapshot to S3 if needed - sf = nil - if err := e.initS3IfNil(ctx); err != nil { - logrus.Warnf("Unable to initialize S3 client: %v", err) - sf = &snapshotFile{ - Name: filepath.Base(snapshotPath), - Metadata: extraMetadata, - NodeName: "s3", - CreatedAt: &metav1.Time{ - Time: now, - }, - Message: base64.StdEncoding.EncodeToString([]byte(err.Error())), - Size: 0, - Status: failedSnapshotStatus, - S3: &s3Config{ - Endpoint: e.config.EtcdS3Endpoint, - EndpointCA: e.config.EtcdS3EndpointCA, - SkipSSLVerify: e.config.EtcdS3SkipSSLVerify, - Bucket: e.config.EtcdS3BucketName, - Region: e.config.EtcdS3Region, - Folder: e.config.EtcdS3Folder, - Insecure: e.config.EtcdS3Insecure, - }, - } - } - // sf should be nil if we were able to successfully initialize the S3 client. - if sf == nil { - sf, err = e.s3.upload(ctx, snapshotPath, extraMetadata, now) - if err != nil { - return err - } - logrus.Infof("S3 upload complete for %s", snapshotName) - if err := e.s3.snapshotRetention(ctx); err != nil { - return errors.Wrap(err, "failed to apply s3 snapshot retention policy") - } - } - if err := e.addSnapshotData(*sf); err != nil { - return errors.Wrap(err, "failed to save snapshot data to configmap") - } - } - } - - return e.ReconcileSnapshotData(ctx) -} - -type s3Config struct { - Endpoint string `json:"endpoint,omitempty"` - EndpointCA string `json:"endpointCA,omitempty"` - SkipSSLVerify bool `json:"skipSSLVerify,omitempty"` - Bucket string `json:"bucket,omitempty"` - Region string `json:"region,omitempty"` - Folder string `json:"folder,omitempty"` - Insecure bool `json:"insecure,omitempty"` -} - -type snapshotStatus string - -const ( - successfulSnapshotStatus snapshotStatus = "successful" - failedSnapshotStatus snapshotStatus = "failed" -) - -// snapshotFile represents a single snapshot and it's -// metadata. -type snapshotFile struct { - Name string `json:"name"` - // Location contains the full path of the snapshot. For - // local paths, the location will be prefixed with "file://". - Location string `json:"location,omitempty"` - Metadata string `json:"metadata,omitempty"` - Message string `json:"message,omitempty"` - NodeName string `json:"nodeName,omitempty"` - CreatedAt *metav1.Time `json:"createdAt,omitempty"` - Size int64 `json:"size,omitempty"` - Status snapshotStatus `json:"status,omitempty"` - S3 *s3Config `json:"s3Config,omitempty"` - Compressed bool `json:"compressed"` -} - -// listLocalSnapshots provides a list of the currently stored -// snapshots on disk along with their relevant -// metadata. -func (e *ETCD) listLocalSnapshots() (map[string]snapshotFile, error) { - snapshots := make(map[string]snapshotFile) - snapshotDir, err := snapshotDir(e.config, true) - if err != nil { - return snapshots, errors.Wrap(err, "failed to get the snapshot dir") - } - - dirEntries, err := os.ReadDir(snapshotDir) - if err != nil { - return nil, err - } - - nodeName := os.Getenv("NODE_NAME") - - for _, de := range dirEntries { - file, err := de.Info() - if err != nil { - return nil, err - } - sf := snapshotFile{ - Name: file.Name(), - Location: "file://" + filepath.Join(snapshotDir, file.Name()), - NodeName: nodeName, - CreatedAt: &metav1.Time{ - Time: file.ModTime(), - }, - Size: file.Size(), - Status: successfulSnapshotStatus, - } - sfKey := generateSnapshotConfigMapKey(sf) - snapshots[sfKey] = sf - } - - return snapshots, nil -} - -// listS3Snapshots provides a list of currently stored -// snapshots in S3 along with their relevant -// metadata. -func (e *ETCD) listS3Snapshots(ctx context.Context) (map[string]snapshotFile, error) { - snapshots := make(map[string]snapshotFile) - - if e.config.EtcdS3 { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - if err := e.initS3IfNil(ctx); err != nil { - return nil, err - } - - var loo minio.ListObjectsOptions - if e.config.EtcdS3Folder != "" { - loo = minio.ListObjectsOptions{ - Prefix: e.config.EtcdS3Folder, - Recursive: true, - } - } - - objects := e.s3.client.ListObjects(ctx, e.config.EtcdS3BucketName, loo) - - for obj := range objects { - if obj.Err != nil { - return nil, obj.Err - } - if obj.Size == 0 { - continue - } - - ca, err := time.Parse(time.RFC3339, obj.LastModified.Format(time.RFC3339)) - if err != nil { - return nil, err - } - - sf := snapshotFile{ - Name: filepath.Base(obj.Key), - NodeName: "s3", - CreatedAt: &metav1.Time{ - Time: ca, - }, - Size: obj.Size, - S3: &s3Config{ - Endpoint: e.config.EtcdS3Endpoint, - EndpointCA: e.config.EtcdS3EndpointCA, - SkipSSLVerify: e.config.EtcdS3SkipSSLVerify, - Bucket: e.config.EtcdS3BucketName, - Region: e.config.EtcdS3Region, - Folder: e.config.EtcdS3Folder, - Insecure: e.config.EtcdS3Insecure, - }, - Status: successfulSnapshotStatus, - } - sfKey := generateSnapshotConfigMapKey(sf) - snapshots[sfKey] = sf - } - } - return snapshots, nil -} - -// initS3IfNil initializes the S3 client -// if it hasn't yet been initialized. -func (e *ETCD) initS3IfNil(ctx context.Context) error { - if e.s3 == nil { - s3, err := NewS3(ctx, e.config) - if err != nil { - return err - } - e.s3 = s3 - } - - return nil -} - -// PruneSnapshots performs a retention run with the given -// retention duration and removes expired snapshots. -func (e *ETCD) PruneSnapshots(ctx context.Context) error { - snapshotDir, err := snapshotDir(e.config, false) - if err != nil { - return errors.Wrap(err, "failed to get the snapshot dir") - } - if err := snapshotRetention(e.config.EtcdSnapshotRetention, e.config.EtcdSnapshotName, snapshotDir); err != nil { - logrus.Errorf("Error applying snapshot retention policy: %v", err) - } - - if e.config.EtcdS3 { - if err := e.initS3IfNil(ctx); err != nil { - logrus.Warnf("Unable to initialize S3 client during prune: %v", err) - } else { - if err := e.s3.snapshotRetention(ctx); err != nil { - logrus.Errorf("Error applying S3 snapshot retention policy: %v", err) - } - } - } - - return e.ReconcileSnapshotData(ctx) -} - -// ListSnapshots is an exported wrapper method that wraps an -// unexported method of the same name. -func (e *ETCD) ListSnapshots(ctx context.Context) (map[string]snapshotFile, error) { - if e.config.EtcdS3 { - return e.listS3Snapshots(ctx) - } - return e.listLocalSnapshots() -} - -// deleteSnapshots removes the given snapshots from -// either local storage or S3. -func (e *ETCD) DeleteSnapshots(ctx context.Context, snapshots []string) error { - snapshotDir, err := snapshotDir(e.config, false) - if err != nil { - return errors.Wrap(err, "failed to get the snapshot dir") - } - - if e.config.EtcdS3 { - logrus.Info("Removing the given etcd snapshot(s) from S3") - logrus.Debugf("Removing the given etcd snapshot(s) from S3: %v", snapshots) - - if e.initS3IfNil(ctx); err != nil { - return err - } - - objectsCh := make(chan minio.ObjectInfo) - - ctx, cancel := context.WithTimeout(ctx, e.config.EtcdS3Timeout) - defer cancel() - - go func() { - defer close(objectsCh) - - opts := minio.ListObjectsOptions{ - Recursive: true, - } - - for obj := range e.s3.client.ListObjects(ctx, e.config.EtcdS3BucketName, opts) { - if obj.Err != nil { - logrus.Error(obj.Err) - return - } - - // iterate through the given snapshots and only - // add them to the channel for remove if they're - // actually found from the bucket listing. - for _, snapshot := range snapshots { - if snapshot == obj.Key { - objectsCh <- obj - } - } - } - }() - - err = func() error { - for { - select { - case <-ctx.Done(): - logrus.Errorf("Unable to delete snapshot: %v", ctx.Err()) - return e.ReconcileSnapshotData(ctx) - case <-time.After(time.Millisecond * 100): - continue - case err, ok := <-e.s3.client.RemoveObjects(ctx, e.config.EtcdS3BucketName, objectsCh, minio.RemoveObjectsOptions{}): - if err.Err != nil { - logrus.Errorf("Unable to delete snapshot: %v", err.Err) - } - if !ok { - return e.ReconcileSnapshotData(ctx) - } - } - } - }() - if err != nil { - return err - } - } - - logrus.Info("Removing the given locally stored etcd snapshot(s)") - logrus.Debugf("Attempting to remove the given locally stored etcd snapshot(s): %v", snapshots) - - for _, s := range snapshots { - // check if the given snapshot exists. If it does, - // remove it, otherwise continue. - sf := filepath.Join(snapshotDir, s) - if _, err := os.Stat(sf); os.IsNotExist(err) { - logrus.Infof("Snapshot %s, does not exist", s) - continue - } - if err := os.Remove(sf); err != nil { - return err - } - logrus.Debug("Removed snapshot ", s) - } - - return e.ReconcileSnapshotData(ctx) -} - -// AddSnapshotData adds the given snapshot file information to the snapshot configmap, using the existing extra metadata -// available at the time. -func (e *ETCD) addSnapshotData(sf snapshotFile) error { - return retry.OnError(snapshotDataBackoff, func(err error) bool { - return apierrors.IsConflict(err) || apierrors.IsAlreadyExists(err) - }, func() error { - // make sure the core.Factory is initialized. There can - // be a race between this core code startup. - for e.config.Runtime.Core == nil { - runtime.Gosched() - } - snapshotConfigMap, getErr := e.config.Runtime.Core.Core().V1().ConfigMap().Get(metav1.NamespaceSystem, snapshotConfigMapName, metav1.GetOptions{}) - - sfKey := generateSnapshotConfigMapKey(sf) - marshalledSnapshotFile, err := json.Marshal(sf) - if err != nil { - return err - } - if apierrors.IsNotFound(getErr) { - cm := v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: snapshotConfigMapName, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{sfKey: string(marshalledSnapshotFile)}, - } - _, err := e.config.Runtime.Core.Core().V1().ConfigMap().Create(&cm) - return err - } - - if snapshotConfigMap.Data == nil { - snapshotConfigMap.Data = make(map[string]string) - } - - snapshotConfigMap.Data[sfKey] = string(marshalledSnapshotFile) - - _, err = e.config.Runtime.Core.Core().V1().ConfigMap().Update(snapshotConfigMap) - return err - }) -} - -func generateSnapshotConfigMapKey(sf snapshotFile) string { - name := invalidKeyChars.ReplaceAllString(sf.Name, "_") - if sf.NodeName == "s3" { - return "s3-" + name - } - return "local-" + name -} - -// ReconcileSnapshotData reconciles snapshot data in the snapshot ConfigMap. -// It will reconcile snapshot data from disk locally always, and if S3 is enabled, will attempt to list S3 snapshots -// and reconcile snapshots from S3. Notably, -func (e *ETCD) ReconcileSnapshotData(ctx context.Context) error { - logrus.Infof("Reconciling etcd snapshot data in %s ConfigMap", snapshotConfigMapName) - defer logrus.Infof("Reconciliation of snapshot data in %s ConfigMap complete", snapshotConfigMapName) - return retry.OnError(retry.DefaultBackoff, func(err error) bool { - return apierrors.IsConflict(err) || apierrors.IsAlreadyExists(err) - }, func() error { - // make sure the core.Factory is initialize. There can - // be a race between this core code startup. - for e.config.Runtime.Core == nil { - runtime.Gosched() - } - - logrus.Debug("core.Factory is initialized") - - snapshotConfigMap, getErr := e.config.Runtime.Core.Core().V1().ConfigMap().Get(metav1.NamespaceSystem, snapshotConfigMapName, metav1.GetOptions{}) - if apierrors.IsNotFound(getErr) { - // Can't reconcile what doesn't exist. - return errors.New("No snapshot configmap found") - } - - logrus.Debugf("Attempting to reconcile etcd snapshot data for configmap generation %d", snapshotConfigMap.Generation) - - // if the snapshot config map data is nil, no need to reconcile. - if snapshotConfigMap.Data == nil { - return nil - } - - snapshotFiles, err := e.listLocalSnapshots() - if err != nil { - return err - } - - // s3ListSuccessful is set to true if we are successful at listing snapshots from S3 to eliminate accidental - // clobbering of S3 snapshots in the configmap due to misconfigured S3 credentials/details - s3ListSuccessful := false - - if e.config.EtcdS3 { - if s3Snapshots, err := e.listS3Snapshots(ctx); err != nil { - logrus.Errorf("error retrieving S3 snapshots for reconciliation: %v", err) - } else { - for k, v := range s3Snapshots { - snapshotFiles[k] = v - } - s3ListSuccessful = true - } - } - - nodeName := os.Getenv("NODE_NAME") - - // deletedSnapshots is a map[string]string where key is the configmap key and the value is the marshalled snapshot file - // it will be populated below with snapshots that are either from S3 or on the local node. Notably, deletedSnapshots will - // not contain snapshots that are in the "failed" status - deletedSnapshots := make(map[string]string) - // failedSnapshots is a slice of unmarshaled snapshot files sourced from the configmap - // These are stored unmarshaled so we can sort based on name. - var failedSnapshots []snapshotFile - var failedS3Snapshots []snapshotFile - - // remove entries for this node and s3 (if S3 is enabled) only - for k, v := range snapshotConfigMap.Data { - var sf snapshotFile - if err := json.Unmarshal([]byte(v), &sf); err != nil { - return err - } - if (sf.NodeName == nodeName || (sf.NodeName == "s3" && s3ListSuccessful)) && sf.Status != failedSnapshotStatus { - // Only delete the snapshot if the snapshot was not failed - // sf.Status != FailedSnapshotStatus is intentional, as it is possible we are reconciling snapshots stored from older versions that did not set status - deletedSnapshots[generateSnapshotConfigMapKey(sf)] = v // store a copy of the snapshot - delete(snapshotConfigMap.Data, k) - } else if sf.Status == failedSnapshotStatus && sf.NodeName == nodeName && e.config.EtcdSnapshotRetention >= 1 { - // Handle locally failed snapshots. - failedSnapshots = append(failedSnapshots, sf) - delete(snapshotConfigMap.Data, k) - } else if sf.Status == failedSnapshotStatus && e.config.EtcdS3 && sf.NodeName == "s3" && strings.HasPrefix(sf.Name, e.config.EtcdSnapshotName+"-"+nodeName) && e.config.EtcdSnapshotRetention >= 1 { - // If we're operating against S3, we can clean up failed S3 snapshots that failed on this node. - failedS3Snapshots = append(failedS3Snapshots, sf) - delete(snapshotConfigMap.Data, k) - } - } - - // Apply the failed snapshot retention policy to locally failed snapshots - if len(failedSnapshots) > 0 && e.config.EtcdSnapshotRetention >= 1 { - sort.Slice(failedSnapshots, func(i, j int) bool { - return failedSnapshots[i].Name > failedSnapshots[j].Name - }) - - var keepCount int - if e.config.EtcdSnapshotRetention >= len(failedSnapshots) { - keepCount = len(failedSnapshots) - } else { - keepCount = e.config.EtcdSnapshotRetention - } - for _, dfs := range failedSnapshots[:keepCount] { - sfKey := generateSnapshotConfigMapKey(dfs) - marshalledSnapshot, err := json.Marshal(dfs) - if err != nil { - logrus.Errorf("unable to marshal snapshot to store in configmap %v", err) - } else { - snapshotConfigMap.Data[sfKey] = string(marshalledSnapshot) - } - } - } - - // Apply the failed snapshot retention policy to the S3 snapshots - if len(failedS3Snapshots) > 0 && e.config.EtcdSnapshotRetention >= 1 { - sort.Slice(failedS3Snapshots, func(i, j int) bool { - return failedS3Snapshots[i].Name > failedS3Snapshots[j].Name - }) - - var keepCount int - if e.config.EtcdSnapshotRetention >= len(failedS3Snapshots) { - keepCount = len(failedS3Snapshots) - } else { - keepCount = e.config.EtcdSnapshotRetention - } - for _, dfs := range failedS3Snapshots[:keepCount] { - sfKey := generateSnapshotConfigMapKey(dfs) - marshalledSnapshot, err := json.Marshal(dfs) - if err != nil { - logrus.Errorf("unable to marshal snapshot to store in configmap %v", err) - } else { - snapshotConfigMap.Data[sfKey] = string(marshalledSnapshot) - } - } - } - - // save the local entries to the ConfigMap if they are still on disk or in S3. - for _, snapshot := range snapshotFiles { - var sf snapshotFile - sfKey := generateSnapshotConfigMapKey(snapshot) - if v, ok := deletedSnapshots[sfKey]; ok { - // use the snapshot file we have from the existing configmap, and unmarshal it so we can manipulate it - if err := json.Unmarshal([]byte(v), &sf); err != nil { - logrus.Errorf("error unmarshaling snapshot file: %v", err) - // use the snapshot with info we sourced from disk/S3 (will be missing metadata, but something is better than nothing) - sf = snapshot - } - } else { - sf = snapshot - } - - sf.Status = successfulSnapshotStatus // if the snapshot is on disk or in S3, it was successful. - - marshalledSnapshot, err := json.Marshal(sf) - if err != nil { - logrus.Warnf("unable to marshal snapshot metadata %s to store in configmap, received error: %v", sf.Name, err) - } else { - snapshotConfigMap.Data[sfKey] = string(marshalledSnapshot) - } - } - - logrus.Debugf("Updating snapshot ConfigMap (%s) with %d entries", snapshotConfigMapName, len(snapshotConfigMap.Data)) - _, err = e.config.Runtime.Core.Core().V1().ConfigMap().Update(snapshotConfigMap) - return err - }) -} - -// setSnapshotFunction schedules snapshots at the configured interval. -func (e *ETCD) setSnapshotFunction(ctx context.Context) { - skipJob := cron.SkipIfStillRunning(cronLogger) - e.cron.AddJob(e.config.EtcdSnapshotCron, skipJob(cron.FuncJob(func() { - // Add a small amount of jitter to the actual snapshot execution. On clusters with multiple servers, - // having all the nodes take a snapshot at the exact same time can lead to excessive retry thrashing - // when updating the snapshot list configmap. - time.Sleep(time.Duration(rand.Float64() * float64(snapshotJitterMax))) - if err := e.Snapshot(ctx); err != nil { - logrus.Error(err) - } - }))) -} - // Restore performs a restore of the ETCD datastore from // the given snapshot path. This operation exists upon // completion. @@ -2086,49 +1279,6 @@ func (e *ETCD) Restore(ctx context.Context) error { }) } -// snapshotRetention iterates through the snapshots and removes the oldest -// leaving the desired number of snapshots. -func snapshotRetention(retention int, snapshotPrefix string, snapshotDir string) error { - if retention < 1 { - return nil - } - - logrus.Infof("Applying local snapshot retention policy: retention: %d, snapshotPrefix: %s, directory: %s", retention, snapshotPrefix, snapshotDir) - - var snapshotFiles []os.FileInfo - if err := filepath.Walk(snapshotDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if strings.HasPrefix(info.Name(), snapshotPrefix) { - snapshotFiles = append(snapshotFiles, info) - } - return nil - }); err != nil { - return err - } - if len(snapshotFiles) <= retention { - return nil - } - sort.Slice(snapshotFiles, func(firstSnapshot, secondSnapshot int) bool { - // it takes the name from the snapshot file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date - firstSnapshotName, secondSnapshotName := strings.Split(snapshotFiles[firstSnapshot].Name(), "-"), strings.Split(snapshotFiles[secondSnapshot].Name(), "-") - firstSnapshotDate, secondSnapshotDate := firstSnapshotName[len(firstSnapshotName)-1], secondSnapshotName[len(secondSnapshotName)-1] - return firstSnapshotDate < secondSnapshotDate - }) - - delCount := len(snapshotFiles) - retention - for _, df := range snapshotFiles[:delCount] { - snapshotPath := filepath.Join(snapshotDir, df.Name()) - logrus.Infof("Removing local snapshot %s", snapshotPath) - if err := os.Remove(snapshotPath); err != nil { - return err - } - } - - return nil -} - // backupDirWithRetention will move the dir to a backup dir // and will keep only maxBackupRetention of dirs. func backupDirWithRetention(dir string, maxBackupRetention int) (string, error) { diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index fe15f8f1f129..3409337d0bb2 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -7,26 +7,42 @@ import ( "encoding/base64" "encoding/pem" "fmt" - "io" + "io/ioutil" "net/http" + "net/textproto" "os" + "path" "path/filepath" + "runtime" "sort" + "strconv" "strings" "time" "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/util" + "github.com/k3s-io/k3s/pkg/version" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/pkg/errors" "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +var ( + clusterIDKey = textproto.CanonicalMIMEHeaderKey(version.Program + "-cluster-id") + tokenHashKey = textproto.CanonicalMIMEHeaderKey(version.Program + "-token-hash") + nodeNameKey = textproto.CanonicalMIMEHeaderKey(version.Program + "-node-name") +) + // S3 maintains state for S3 functionality. type S3 struct { - config *config.Control - client *minio.Client + config *config.Control + client *minio.Client + clusterID string + tokenHash string + nodeName string } // newS3 creates a new value of type s3 pointer with a @@ -80,146 +96,177 @@ func NewS3(ctx context.Context, config *config.Control) (*S3, error) { return nil, err } if !exists { - return nil, fmt.Errorf("bucket: %s does not exist", config.EtcdS3BucketName) + return nil, fmt.Errorf("bucket %s does not exist", config.EtcdS3BucketName) } logrus.Infof("S3 bucket %s exists", config.EtcdS3BucketName) + for config.Runtime.Core == nil { + runtime.Gosched() + } + + // cluster id hack: see https://groups.google.com/forum/#!msg/kubernetes-sig-architecture/mVGobfD4TpY/nkdbkX1iBwAJ + var clusterID string + if ns, err := config.Runtime.Core.Core().V1().Namespace().Get(metav1.NamespaceSystem, metav1.GetOptions{}); err != nil { + logrus.Warnf("Failed to set cluster ID: %v", err) + } else { + clusterID = string(ns.UID) + } + + tokenHash, err := util.GetTokenHash(config) + if err != nil { + return nil, errors.Wrap(err, "failed to get server token hash for etcd snapshot") + } + return &S3{ - config: config, - client: c, + config: config, + client: c, + clusterID: clusterID, + tokenHash: tokenHash, + nodeName: os.Getenv("NODE_NAME"), }, nil } // upload uploads the given snapshot to the configured S3 // compatible backend. -func (s *S3) upload(ctx context.Context, snapshot, extraMetadata string, now time.Time) (*snapshotFile, error) { - logrus.Infof("Uploading snapshot %s to S3", snapshot) +func (s *S3) upload(ctx context.Context, snapshot string, extraMetadata *v1.ConfigMap, now time.Time) (*snapshotFile, error) { + logrus.Infof("Uploading snapshot to s3://%s/%s", s.config.EtcdS3BucketName, snapshot) basename := filepath.Base(snapshot) - var snapshotFileName string - var sf snapshotFile - if s.config.EtcdS3Folder != "" { - snapshotFileName = filepath.Join(s.config.EtcdS3Folder, basename) - } else { - snapshotFileName = basename + metadata := filepath.Join(filepath.Dir(snapshot), "..", metadataDir, basename) + snapshotKey := path.Join(s.config.EtcdS3Folder, basename) + metadataKey := path.Join(s.config.EtcdS3Folder, metadataDir, basename) + + sf := &snapshotFile{ + Name: basename, + Location: fmt.Sprintf("s3://%s/%s", s.config.EtcdS3BucketName, snapshotKey), + NodeName: "s3", + CreatedAt: &metav1.Time{ + Time: now, + }, + S3: &s3Config{ + Endpoint: s.config.EtcdS3Endpoint, + EndpointCA: s.config.EtcdS3EndpointCA, + SkipSSLVerify: s.config.EtcdS3SkipSSLVerify, + Bucket: s.config.EtcdS3BucketName, + Region: s.config.EtcdS3Region, + Folder: s.config.EtcdS3Folder, + Insecure: s.config.EtcdS3Insecure, + }, + Compressed: strings.HasSuffix(snapshot, compressedExtension), + metadataSource: extraMetadata, + nodeSource: s.nodeName, } - toCtx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) - defer cancel() - opts := minio.PutObjectOptions{NumThreads: 2} - if strings.HasSuffix(snapshot, compressedExtension) { - opts.ContentType = "application/zip" + uploadInfo, err := s.uploadSnapshot(ctx, snapshotKey, snapshot) + if err != nil { + sf.Status = failedSnapshotStatus + sf.Message = base64.StdEncoding.EncodeToString([]byte(err.Error())) } else { - opts.ContentType = "application/octet-stream" + sf.Status = successfulSnapshotStatus + sf.Size = uploadInfo.Size + sf.tokenHash = s.tokenHash } - uploadInfo, err := s.client.FPutObject(toCtx, s.config.EtcdS3BucketName, snapshotFileName, snapshot, opts) - if err != nil { - sf = snapshotFile{ - Name: filepath.Base(uploadInfo.Key), - Metadata: extraMetadata, - NodeName: "s3", - CreatedAt: &metav1.Time{ - Time: now, - }, - Message: base64.StdEncoding.EncodeToString([]byte(err.Error())), - Size: 0, - Status: failedSnapshotStatus, - S3: &s3Config{ - Endpoint: s.config.EtcdS3Endpoint, - EndpointCA: s.config.EtcdS3EndpointCA, - SkipSSLVerify: s.config.EtcdS3SkipSSLVerify, - Bucket: s.config.EtcdS3BucketName, - Region: s.config.EtcdS3Region, - Folder: s.config.EtcdS3Folder, - Insecure: s.config.EtcdS3Insecure, - }, - } - logrus.Errorf("Error received during snapshot upload to S3: %s", err) + if _, err := s.uploadSnapshotMetadata(ctx, metadataKey, metadata); err != nil { + logrus.Warnf("Failed to upload snapshot metadata to S3: %v", err) } else { - ca, err := time.Parse(time.RFC3339, uploadInfo.LastModified.Format(time.RFC3339)) - if err != nil { - return nil, err - } - - sf = snapshotFile{ - Name: filepath.Base(uploadInfo.Key), - Metadata: extraMetadata, - NodeName: "s3", - CreatedAt: &metav1.Time{ - Time: ca, - }, - Size: uploadInfo.Size, - Status: successfulSnapshotStatus, - S3: &s3Config{ - Endpoint: s.config.EtcdS3Endpoint, - EndpointCA: s.config.EtcdS3EndpointCA, - SkipSSLVerify: s.config.EtcdS3SkipSSLVerify, - Bucket: s.config.EtcdS3BucketName, - Region: s.config.EtcdS3Region, - Folder: s.config.EtcdS3Folder, - Insecure: s.config.EtcdS3Insecure, - }, - } + logrus.Infof("Uploaded snapshot metadata s3://%s/%s", s.config.EtcdS3BucketName, metadata) } - return &sf, nil + return sf, err } -// download downloads the given snapshot from the configured S3 -// compatible backend. -func (s *S3) Download(ctx context.Context) error { - var remotePath string - if s.config.EtcdS3Folder != "" { - remotePath = filepath.Join(s.config.EtcdS3Folder, s.config.ClusterResetRestorePath) +// uploadSnapshot uploads the snapshot file to S3 using the minio API. +func (s *S3) uploadSnapshot(ctx context.Context, key, path string) (info minio.UploadInfo, err error) { + opts := minio.PutObjectOptions{ + NumThreads: 2, + UserMetadata: map[string]string{ + clusterIDKey: s.clusterID, + nodeNameKey: s.nodeName, + tokenHashKey: s.tokenHash, + }, + } + if strings.HasSuffix(key, compressedExtension) { + opts.ContentType = "application/zip" } else { - remotePath = s.config.ClusterResetRestorePath + opts.ContentType = "application/octet-stream" } - - logrus.Debugf("retrieving snapshot: %s", remotePath) - toCtx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) + ctx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) defer cancel() - r, err := s.client.GetObject(toCtx, s.config.EtcdS3BucketName, remotePath, minio.GetObjectOptions{}) - if err != nil { - return nil + return s.client.FPutObject(ctx, s.config.EtcdS3BucketName, key, path, opts) +} + +// uploadSnapshotMetadata marshals and uploads the snapshot metadata to S3 using the minio API. +// The upload is silently skipped if no extra metadata is provided. +func (s *S3) uploadSnapshotMetadata(ctx context.Context, key, path string) (info minio.UploadInfo, err error) { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return minio.UploadInfo{}, nil + } + return minio.UploadInfo{}, err + } + + opts := minio.PutObjectOptions{ + NumThreads: 2, + ContentType: "application/json", + UserMetadata: map[string]string{ + clusterIDKey: s.clusterID, + nodeNameKey: s.nodeName, + }, } - defer r.Close() + ctx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) + defer cancel() + return s.client.FPutObject(ctx, s.config.EtcdS3BucketName, key, path, opts) +} +// Download downloads the given snapshot from the configured S3 +// compatible backend. +func (s *S3) Download(ctx context.Context) error { + snapshotKey := path.Join(s.config.EtcdS3Folder, s.config.ClusterResetRestorePath) + metadataKey := path.Join(s.config.EtcdS3Folder, metadataDir, s.config.ClusterResetRestorePath) snapshotDir, err := snapshotDir(s.config, true) if err != nil { return errors.Wrap(err, "failed to get the snapshot dir") } + snapshotFile := filepath.Join(snapshotDir, s.config.ClusterResetRestorePath) + metadataFile := filepath.Join(snapshotDir, "..", metadataDir, s.config.ClusterResetRestorePath) - fullSnapshotPath := filepath.Join(snapshotDir, s.config.ClusterResetRestorePath) - sf, err := os.Create(fullSnapshotPath) - if err != nil { + logrus.Debugf("Downloading snapshot from s3://%s/%s", s.config.EtcdS3BucketName, snapshotKey) + if err := s.downloadSnapshot(ctx, snapshotKey, snapshotFile); err != nil { return err } - defer sf.Close() - - stat, err := r.Stat() - if err != nil { + if err := s.downloadSnapshotMetadata(ctx, metadataKey, metadataFile); err != nil { return err } - if _, err := io.CopyN(sf, r, stat.Size); err != nil { - return err - } + s.config.ClusterResetRestorePath = snapshotFile + return nil +} - s.config.ClusterResetRestorePath = fullSnapshotPath +// downloadSnapshot downloads the snapshot file from S3 using the minio API. +func (s *S3) downloadSnapshot(ctx context.Context, key, file string) error { + ctx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) + defer cancel() + defer os.Chmod(file, 0600) + return s.client.FGetObject(ctx, s.config.EtcdS3BucketName, key, file, minio.GetObjectOptions{}) +} - return os.Chmod(fullSnapshotPath, 0600) +// downloadSnapshotMetadata downloads the snapshot metadata file from S3 using the minio API. +// No error is returned if the metadata file does not exist, as it is optional. +func (s *S3) downloadSnapshotMetadata(ctx context.Context, key, file string) error { + logrus.Debugf("Downloading snapshot metadata from s3://%s/%s", s.config.EtcdS3BucketName, key) + ctx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) + defer cancel() + defer os.Chmod(file, 0600) + err := s.client.FGetObject(ctx, s.config.EtcdS3BucketName, key, file, minio.GetObjectOptions{}) + if resp := minio.ToErrorResponse(err); resp.StatusCode == http.StatusNotFound { + return nil + } + return err } // snapshotPrefix returns the prefix used in the // naming of the snapshots. func (s *S3) snapshotPrefix() string { - fullSnapshotPrefix := s.config.EtcdSnapshotName - var prefix string - if s.config.EtcdS3Folder != "" { - prefix = filepath.Join(s.config.EtcdS3Folder, fullSnapshotPrefix) - } else { - prefix = fullSnapshotPrefix - } - return prefix + return path.Join(s.config.EtcdS3Folder, s.config.EtcdSnapshotName) } // snapshotRetention prunes snapshots in the configured S3 compatible backend for this specific node. @@ -227,21 +274,27 @@ func (s *S3) snapshotRetention(ctx context.Context) error { if s.config.EtcdSnapshotRetention < 1 { return nil } - logrus.Infof("Applying snapshot retention policy to snapshots stored in S3: retention: %d, snapshotPrefix: %s", s.config.EtcdSnapshotRetention, s.snapshotPrefix()) + logrus.Infof("Applying snapshot retention=%d to snapshots stored in s3://%s/%s", s.config.EtcdSnapshotRetention, s.config.EtcdS3BucketName, s.snapshotPrefix()) var snapshotFiles []minio.ObjectInfo toCtx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) defer cancel() - loo := minio.ListObjectsOptions{ - Recursive: true, + opts := minio.ListObjectsOptions{ Prefix: s.snapshotPrefix(), + Recursive: true, } - for info := range s.client.ListObjects(toCtx, s.config.EtcdS3BucketName, loo) { + for info := range s.client.ListObjects(toCtx, s.config.EtcdS3BucketName, opts) { if info.Err != nil { return info.Err } + + // skip metadata + if path.Base(path.Dir(info.Key)) == metadataDir { + continue + } + snapshotFiles = append(snapshotFiles, info) } @@ -249,17 +302,21 @@ func (s *S3) snapshotRetention(ctx context.Context) error { return nil } - sort.Slice(snapshotFiles, func(firstSnapshot, secondSnapshot int) bool { - // it takes the key from the snapshot file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date - firstSnapshotName, secondSnapshotName := strings.Split(snapshotFiles[firstSnapshot].Key, "-"), strings.Split(snapshotFiles[secondSnapshot].Key, "-") - firstSnapshotDate, secondSnapshotDate := firstSnapshotName[len(firstSnapshotName)-1], secondSnapshotName[len(secondSnapshotName)-1] - return firstSnapshotDate < secondSnapshotDate + // sort newest-first so we can prune entries past the retention count + sort.Slice(snapshotFiles, func(i, j int) bool { + return snapshotFiles[j].LastModified.Before(snapshotFiles[i].LastModified) }) - delCount := len(snapshotFiles) - s.config.EtcdSnapshotRetention - for _, df := range snapshotFiles[:delCount] { - logrus.Infof("Removing S3 snapshot: %s", df.Key) - if err := s.client.RemoveObject(ctx, s.config.EtcdS3BucketName, df.Key, minio.RemoveObjectOptions{}); err != nil { + for _, df := range snapshotFiles[s.config.EtcdSnapshotRetention:] { + logrus.Infof("Removing S3 snapshot: s3://%s/%s", s.config.EtcdS3BucketName, df.Key) + if err := s.client.RemoveObject(toCtx, s.config.EtcdS3BucketName, df.Key, minio.RemoveObjectOptions{}); err != nil { + return err + } + metadataKey := path.Join(path.Dir(df.Key), metadataDir, path.Base(df.Key)) + if err := s.client.RemoveObject(toCtx, s.config.EtcdS3BucketName, metadataKey, minio.RemoveObjectOptions{}); err != nil { + if isNotExist(err) { + return nil + } return err } } @@ -267,6 +324,112 @@ func (s *S3) snapshotRetention(ctx context.Context) error { return nil } +func (s *S3) deleteSnapshot(ctx context.Context, key string) error { + ctx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) + defer cancel() + + key = path.Join(s.config.EtcdS3Folder, key) + err := s.client.RemoveObject(ctx, s.config.EtcdS3BucketName, key, minio.RemoveObjectOptions{}) + if err == nil || isNotExist(err) { + metadataKey := path.Join(path.Dir(key), metadataDir, path.Base(key)) + if merr := s.client.RemoveObject(ctx, s.config.EtcdS3BucketName, metadataKey, minio.RemoveObjectOptions{}); merr != nil && !isNotExist(merr) { + err = merr + } + } + + return err +} + +// listSnapshots provides a list of currently stored +// snapshots in S3 along with their relevant +// metadata. +func (s *S3) listSnapshots(ctx context.Context) (map[string]snapshotFile, error) { + snapshots := map[string]snapshotFile{} + metadatas := []string{} + ctx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) + defer cancel() + + opts := minio.ListObjectsOptions{ + Prefix: s.config.EtcdS3Folder, + Recursive: true, + WithMetadata: true, + } + + objects := s.client.ListObjects(ctx, s.config.EtcdS3BucketName, opts) + + for obj := range objects { + if obj.Err != nil { + return nil, obj.Err + } + if obj.Size == 0 { + continue + } + + if o, err := s.client.StatObject(ctx, s.config.EtcdS3BucketName, obj.Key, minio.StatObjectOptions{}); err != nil { + logrus.Warnf("Failed to get object metadata: %v", err) + } else { + obj = o + } + + filename := path.Base(obj.Key) + if path.Base(path.Dir(obj.Key)) == metadataDir { + metadatas = append(metadatas, obj.Key) + continue + } + + basename, compressed := strings.CutSuffix(filename, compressedExtension) + ts, err := strconv.ParseInt(basename[strings.LastIndexByte(basename, '-')+1:], 10, 64) + if err != nil { + ts = obj.LastModified.Unix() + } + + sf := snapshotFile{ + Name: filename, + Location: fmt.Sprintf("s3://%s/%s", s.config.EtcdS3BucketName, obj.Key), + NodeName: "s3", + CreatedAt: &metav1.Time{ + Time: time.Unix(ts, 0), + }, + Size: obj.Size, + S3: &s3Config{ + Endpoint: s.config.EtcdS3Endpoint, + EndpointCA: s.config.EtcdS3EndpointCA, + SkipSSLVerify: s.config.EtcdS3SkipSSLVerify, + Bucket: s.config.EtcdS3BucketName, + Region: s.config.EtcdS3Region, + Folder: s.config.EtcdS3Folder, + Insecure: s.config.EtcdS3Insecure, + }, + Status: successfulSnapshotStatus, + Compressed: compressed, + nodeSource: obj.UserMetadata[nodeNameKey], + tokenHash: obj.UserMetadata[tokenHashKey], + } + sfKey := generateSnapshotConfigMapKey(sf) + snapshots[sfKey] = sf + } + + for _, metadataKey := range metadatas { + filename := path.Base(metadataKey) + sfKey := generateSnapshotConfigMapKey(snapshotFile{Name: filename, NodeName: "s3"}) + if sf, ok := snapshots[sfKey]; ok { + logrus.Debugf("Loading snapshot metadata from s3://%s/%s", s.config.EtcdS3BucketName, metadataKey) + if obj, err := s.client.GetObject(ctx, s.config.EtcdS3BucketName, metadataKey, minio.GetObjectOptions{}); err != nil { + logrus.Warnf("Failed to get snapshot metadata: %v", err) + } else { + if m, err := ioutil.ReadAll(obj); err != nil { + logrus.Warnf("Failed to read snapshot metadata: %v", err) + } else { + sf.Metadata = base64.StdEncoding.EncodeToString(m) + snapshots[sfKey] = sf + } + } + } + } + + return snapshots, nil +} + func readS3EndpointCA(endpointCA string) ([]byte, error) { ca, err := base64.StdEncoding.DecodeString(endpointCA) if err != nil { diff --git a/pkg/etcd/snapshot.go b/pkg/etcd/snapshot.go new file mode 100644 index 000000000000..d11a7fb5b0b9 --- /dev/null +++ b/pkg/etcd/snapshot.go @@ -0,0 +1,1117 @@ +package etcd + +import ( + "archive/zip" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "time" + + apisv1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1" + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/util" + "github.com/k3s-io/k3s/pkg/version" + "github.com/minio/minio-go/v7" + "github.com/pkg/errors" + "github.com/robfig/cron/v3" + "github.com/sirupsen/logrus" + "go.etcd.io/etcd/client/pkg/v3/logutil" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/etcdutl/v3/snapshot" + "go.uber.org/zap" + "golang.org/x/sync/semaphore" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "k8s.io/utils/pointer" +) + +const ( + maxConcurrentSnapshots = 1 + compressedExtension = ".zip" + metadataDir = ".metadata" + errorTTL = 24 * time.Hour +) + +var ( + snapshotExtraMetadataConfigMapName = version.Program + "-etcd-snapshot-extra-metadata" + labelStorageNode = "etcd." + version.Program + ".cattle.io/snapshot-storage-node" + annotationLocalReconciled = "etcd." + version.Program + ".cattle.io/local-snapshots-timestamp" + annotationS3Reconciled = "etcd." + version.Program + ".cattle.io/s3-snapshots-timestamp" + annotationTokenHash = "etcd." + version.Program + ".cattle.io/snapshot-token-hash" + + // snapshotDataBackoff will retry at increasing steps for up to ~30 seconds. + // If the ConfigMap update fails, the list won't be reconciled again until next time + // the server starts, so we should be fairly persistent in retrying. + snapshotDataBackoff = wait.Backoff{ + Steps: 9, + Duration: 10 * time.Millisecond, + Factor: 3.0, + Jitter: 0.1, + } + + // cronLogger wraps logrus's Printf output as cron-compatible logger + cronLogger = cron.VerbosePrintfLogger(logrus.StandardLogger()) +) + +// snapshotDir ensures that the snapshot directory exists, and then returns its path. +func snapshotDir(config *config.Control, create bool) (string, error) { + if config.EtcdSnapshotDir == "" { + // we have to create the snapshot dir if we are using + // the default snapshot dir if it doesn't exist + defaultSnapshotDir := filepath.Join(config.DataDir, "db", "snapshots") + s, err := os.Stat(defaultSnapshotDir) + if err != nil { + if create && os.IsNotExist(err) { + if err := os.MkdirAll(defaultSnapshotDir, 0700); err != nil { + return "", err + } + return defaultSnapshotDir, nil + } + return "", err + } + if s.IsDir() { + return defaultSnapshotDir, nil + } + } + return config.EtcdSnapshotDir, nil +} + +// preSnapshotSetup checks to see if the necessary components are in place +// to perform an Etcd snapshot. This is necessary primarily for on-demand +// snapshots since they're performed before normal Etcd setup is completed. +func (e *ETCD) preSnapshotSetup(ctx context.Context) error { + if e.snapshotSem == nil { + e.snapshotSem = semaphore.NewWeighted(maxConcurrentSnapshots) + } + return nil +} + +// compressSnapshot compresses the given snapshot and provides the +// caller with the path to the file. +func (e *ETCD) compressSnapshot(snapshotDir, snapshotName, snapshotPath string, now time.Time) (string, error) { + logrus.Info("Compressing etcd snapshot file: " + snapshotName) + + zippedSnapshotName := snapshotName + compressedExtension + zipPath := filepath.Join(snapshotDir, zippedSnapshotName) + + zf, err := os.Create(zipPath) + if err != nil { + return "", err + } + defer zf.Close() + + zipWriter := zip.NewWriter(zf) + defer zipWriter.Close() + + uncompressedPath := filepath.Join(snapshotDir, snapshotName) + fileToZip, err := os.Open(uncompressedPath) + if err != nil { + os.Remove(zipPath) + return "", err + } + defer fileToZip.Close() + + info, err := fileToZip.Stat() + if err != nil { + os.Remove(zipPath) + return "", err + } + + header, err := zip.FileInfoHeader(info) + if err != nil { + os.Remove(zipPath) + return "", err + } + + header.Name = snapshotName + header.Method = zip.Deflate + header.Modified = now + + writer, err := zipWriter.CreateHeader(header) + if err != nil { + os.Remove(zipPath) + return "", err + } + _, err = io.Copy(writer, fileToZip) + + return zipPath, err +} + +// decompressSnapshot decompresses the given snapshot and provides the caller +// with the full path to the uncompressed snapshot. +func (e *ETCD) decompressSnapshot(snapshotDir, snapshotFile string) (string, error) { + logrus.Info("Decompressing etcd snapshot file: " + snapshotFile) + + r, err := zip.OpenReader(snapshotFile) + if err != nil { + return "", err + } + defer r.Close() + + var decompressed *os.File + for _, sf := range r.File { + decompressed, err = os.OpenFile(strings.Replace(sf.Name, compressedExtension, "", -1), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, sf.Mode()) + if err != nil { + return "", err + } + defer decompressed.Close() + + ss, err := sf.Open() + if err != nil { + return "", err + } + defer ss.Close() + + if _, err := io.Copy(decompressed, ss); err != nil { + os.Remove(decompressed.Name()) + return "", err + } + } + + return decompressed.Name(), nil +} + +// Snapshot attempts to save a new snapshot to the configured directory, and then clean up any old and failed +// snapshots in excess of the retention limits. This method is used in the internal cron snapshot +// system as well as used to do on-demand snapshots. +func (e *ETCD) Snapshot(ctx context.Context) error { + if err := e.preSnapshotSetup(ctx); err != nil { + return err + } + if !e.snapshotSem.TryAcquire(maxConcurrentSnapshots) { + return fmt.Errorf("%d snapshots already in progress", maxConcurrentSnapshots) + } + defer e.snapshotSem.Release(maxConcurrentSnapshots) + + // make sure the core.Factory is initialized before attempting to add snapshot metadata + var extraMetadata *v1.ConfigMap + if e.config.Runtime.Core == nil { + logrus.Debugf("Cannot retrieve extra metadata from %s ConfigMap: runtime core not ready", snapshotExtraMetadataConfigMapName) + } else { + logrus.Debugf("Attempting to retrieve extra metadata from %s ConfigMap", snapshotExtraMetadataConfigMapName) + if snapshotExtraMetadataConfigMap, err := e.config.Runtime.Core.Core().V1().ConfigMap().Get(metav1.NamespaceSystem, snapshotExtraMetadataConfigMapName, metav1.GetOptions{}); err != nil { + logrus.Debugf("Error encountered attempting to retrieve extra metadata from %s ConfigMap, error: %v", snapshotExtraMetadataConfigMapName, err) + } else { + logrus.Debugf("Setting extra metadata from %s ConfigMap", snapshotExtraMetadataConfigMapName) + extraMetadata = snapshotExtraMetadataConfigMap + } + } + + endpoints := getEndpoints(e.config) + var client *clientv3.Client + var err error + + // Use the internal client if possible, or create a new one + // if run from the CLI. + if e.client != nil { + client = e.client + } else { + client, err = getClient(ctx, e.config, endpoints...) + if err != nil { + return err + } + defer client.Close() + } + + status, err := client.Status(ctx, endpoints[0]) + if err != nil { + return errors.Wrap(err, "failed to check etcd status for snapshot") + } + + if status.IsLearner { + logrus.Warnf("Unable to take snapshot: not supported for learner") + return nil + } + + snapshotDir, err := snapshotDir(e.config, true) + if err != nil { + return errors.Wrap(err, "failed to get the snapshot dir") + } + + cfg, err := getClientConfig(ctx, e.config) + if err != nil { + return errors.Wrap(err, "failed to get config for etcd snapshot") + } + + tokenHash, err := util.GetTokenHash(e.config) + if err != nil { + return errors.Wrap(err, "failed to get server token hash for etcd snapshot") + } + + nodeName := os.Getenv("NODE_NAME") + now := time.Now().Round(time.Second) + snapshotName := fmt.Sprintf("%s-%s-%d", e.config.EtcdSnapshotName, nodeName, now.Unix()) + snapshotPath := filepath.Join(snapshotDir, snapshotName) + + logrus.Infof("Saving etcd snapshot to %s", snapshotPath) + + var sf *snapshotFile + + lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) + if err != nil { + return err + } + + if err := snapshot.NewV3(lg).Save(ctx, *cfg, snapshotPath); err != nil { + sf = &snapshotFile{ + Name: snapshotName, + Location: "", + NodeName: nodeName, + CreatedAt: &metav1.Time{ + Time: now, + }, + Status: failedSnapshotStatus, + Message: base64.StdEncoding.EncodeToString([]byte(err.Error())), + Size: 0, + metadataSource: extraMetadata, + } + logrus.Errorf("Failed to take etcd snapshot: %v", err) + if err := e.addSnapshotData(*sf); err != nil { + return errors.Wrap(err, "failed to sync ETCDSnapshotFile") + } + } + + // If the snapshot attempt was successful, sf will be nil as we did not set it to store the error message. + if sf == nil { + if e.config.EtcdSnapshotCompress { + zipPath, err := e.compressSnapshot(snapshotDir, snapshotName, snapshotPath, now) + if err != nil { + return errors.Wrap(err, "failed to compress snapshot") + } + if err := os.Remove(snapshotPath); err != nil { + return errors.Wrap(err, "failed to remove uncompressed snapshot") + } + snapshotPath = zipPath + logrus.Info("Compressed snapshot: " + snapshotPath) + } + + f, err := os.Stat(snapshotPath) + if err != nil { + return errors.Wrap(err, "unable to retrieve snapshot information from local snapshot") + } + sf = &snapshotFile{ + Name: f.Name(), + Location: "file://" + snapshotPath, + NodeName: nodeName, + CreatedAt: &metav1.Time{ + Time: now, + }, + Status: successfulSnapshotStatus, + Size: f.Size(), + Compressed: e.config.EtcdSnapshotCompress, + metadataSource: extraMetadata, + tokenHash: tokenHash, + } + + if err := saveSnapshotMetadata(snapshotPath, extraMetadata); err != nil { + return errors.Wrap(err, "failed to save local snapshot metadata") + } + + if err := e.addSnapshotData(*sf); err != nil { + return errors.Wrap(err, "failed to sync ETCDSnapshotFile") + } + + if err := snapshotRetention(e.config.EtcdSnapshotRetention, e.config.EtcdSnapshotName, snapshotDir); err != nil { + return errors.Wrap(err, "failed to apply local snapshot retention policy") + } + + if e.config.EtcdS3 { + if err := e.initS3IfNil(ctx); err != nil { + logrus.Warnf("Unable to initialize S3 client: %v", err) + sf = &snapshotFile{ + Name: filepath.Base(snapshotPath), + NodeName: "s3", + CreatedAt: &metav1.Time{ + Time: now, + }, + Message: base64.StdEncoding.EncodeToString([]byte(err.Error())), + Size: 0, + Status: failedSnapshotStatus, + S3: &s3Config{ + Endpoint: e.config.EtcdS3Endpoint, + EndpointCA: e.config.EtcdS3EndpointCA, + SkipSSLVerify: e.config.EtcdS3SkipSSLVerify, + Bucket: e.config.EtcdS3BucketName, + Region: e.config.EtcdS3Region, + Folder: e.config.EtcdS3Folder, + Insecure: e.config.EtcdS3Insecure, + }, + metadataSource: extraMetadata, + } + } else { + logrus.Infof("Saving etcd snapshot %s to S3", snapshotName) + // upload will return a snapshotFile even on error - if there was an + // error, it will be reflected in the status and message. + sf, err = e.s3.upload(ctx, snapshotPath, extraMetadata, now) + if err != nil { + logrus.Errorf("Error received during snapshot upload to S3: %s", err) + } else { + logrus.Infof("S3 upload complete for %s", snapshotName) + } + } + if err := e.addSnapshotData(*sf); err != nil { + return errors.Wrap(err, "failed to sync ETCDSnapshotFile") + } + if err := e.s3.snapshotRetention(ctx); err != nil { + logrus.Errorf("Failed to apply s3 snapshot retention policy: %v", err) + } + + } + } + + return e.ReconcileSnapshotData(ctx) +} + +type s3Config struct { + Endpoint string `json:"endpoint,omitempty"` + EndpointCA string `json:"endpointCA,omitempty"` + SkipSSLVerify bool `json:"skipSSLVerify,omitempty"` + Bucket string `json:"bucket,omitempty"` + Region string `json:"region,omitempty"` + Folder string `json:"folder,omitempty"` + Insecure bool `json:"insecure,omitempty"` +} + +type snapshotStatus string + +const ( + successfulSnapshotStatus snapshotStatus = "successful" + failedSnapshotStatus snapshotStatus = "failed" +) + +// snapshotFile represents a single snapshot and it's +// metadata. +type snapshotFile struct { + Name string `json:"name"` + // Location contains the full path of the snapshot. For + // local paths, the location will be prefixed with "file://". + Location string `json:"location,omitempty"` + Metadata string `json:"metadata,omitempty"` + Message string `json:"message,omitempty"` + NodeName string `json:"nodeName,omitempty"` + CreatedAt *metav1.Time `json:"createdAt,omitempty"` + Size int64 `json:"size,omitempty"` + Status snapshotStatus `json:"status,omitempty"` + S3 *s3Config `json:"s3Config,omitempty"` + Compressed bool `json:"compressed"` + + // these fields are used for the internal representation of the snapshot + // to populate other fields before serialization to the legacy configmap. + metadataSource *v1.ConfigMap `json:"-"` + nodeSource string `json:"-"` + tokenHash string `json:"-"` +} + +// listLocalSnapshots provides a list of the currently stored +// snapshots on disk along with their relevant +// metadata. +func (e *ETCD) listLocalSnapshots() (map[string]snapshotFile, error) { + nodeName := os.Getenv("NODE_NAME") + snapshots := make(map[string]snapshotFile) + snapshotDir, err := snapshotDir(e.config, true) + if err != nil { + return snapshots, errors.Wrap(err, "failed to get the snapshot dir") + } + + if err := filepath.Walk(snapshotDir, func(path string, file os.FileInfo, err error) error { + if file.IsDir() || err != nil { + return err + } + + basename, compressed := strings.CutSuffix(file.Name(), compressedExtension) + ts, err := strconv.ParseInt(basename[strings.LastIndexByte(basename, '-')+1:], 10, 64) + if err != nil { + ts = file.ModTime().Unix() + } + + // try to read metadata from disk; don't warn if it is missing as it will not exist + // for snapshot files from old releases or if there was no metadata provided. + var metadata string + metadataFile := filepath.Join(filepath.Dir(path), "..", metadataDir, file.Name()) + if m, err := os.ReadFile(metadataFile); err == nil { + logrus.Debugf("Loading snapshot metadata from %s", metadataFile) + metadata = base64.StdEncoding.EncodeToString(m) + } + + sf := snapshotFile{ + Name: file.Name(), + Location: "file://" + filepath.Join(snapshotDir, file.Name()), + NodeName: nodeName, + Metadata: metadata, + CreatedAt: &metav1.Time{ + Time: time.Unix(ts, 0), + }, + Size: file.Size(), + Status: successfulSnapshotStatus, + Compressed: compressed, + } + sfKey := generateSnapshotConfigMapKey(sf) + snapshots[sfKey] = sf + return nil + }); err != nil { + return nil, err + } + + return snapshots, nil +} + +// initS3IfNil initializes the S3 client +// if it hasn't yet been initialized. +func (e *ETCD) initS3IfNil(ctx context.Context) error { + if e.s3 == nil { + s3, err := NewS3(ctx, e.config) + if err != nil { + return err + } + e.s3 = s3 + } + + return nil +} + +// PruneSnapshots performs a retention run with the given +// retention duration and removes expired snapshots. +func (e *ETCD) PruneSnapshots(ctx context.Context) error { + snapshotDir, err := snapshotDir(e.config, false) + if err != nil { + return errors.Wrap(err, "failed to get the snapshot dir") + } + if err := snapshotRetention(e.config.EtcdSnapshotRetention, e.config.EtcdSnapshotName, snapshotDir); err != nil { + logrus.Errorf("Error applying snapshot retention policy: %v", err) + } + + if e.config.EtcdS3 { + if err := e.initS3IfNil(ctx); err != nil { + logrus.Warnf("Unable to initialize S3 client: %v", err) + } else { + if err := e.s3.snapshotRetention(ctx); err != nil { + logrus.Errorf("Error applying S3 snapshot retention policy: %v", err) + } + } + } + return e.ReconcileSnapshotData(ctx) +} + +// ListSnapshots is an exported wrapper method that wraps an +// unexported method of the same name. +func (e *ETCD) ListSnapshots(ctx context.Context) (map[string]snapshotFile, error) { + snapshotFiles := map[string]snapshotFile{} + if e.config.EtcdS3 { + if err := e.initS3IfNil(ctx); err != nil { + logrus.Warnf("Unable to initialize S3 client: %v", err) + return nil, err + } + sfs, err := e.s3.listSnapshots(ctx) + if err != nil { + return nil, err + } + snapshotFiles = sfs + } + + sfs, err := e.listLocalSnapshots() + if err != nil { + return nil, err + } + for k, sf := range sfs { + snapshotFiles[k] = sf + } + + return snapshotFiles, err +} + +// DeleteSnapshots removes the given snapshots from local storage and S3. +func (e *ETCD) DeleteSnapshots(ctx context.Context, snapshots []string) error { + snapshotDir, err := snapshotDir(e.config, false) + if err != nil { + return errors.Wrap(err, "failed to get the snapshot dir") + } + if e.config.EtcdS3 { + if err := e.initS3IfNil(ctx); err != nil { + return err + } + } + + for _, s := range snapshots { + if err := e.deleteSnapshot(filepath.Join(snapshotDir, s)); err != nil { + if isNotExist(err) { + logrus.Infof("Snapshot %s not found locally", s) + } else { + logrus.Errorf("Failed to delete local snapshot %s: %v", s, err) + } + } else { + logrus.Infof("Snapshot %s deleted locally", s) + } + + if e.config.EtcdS3 { + if err := e.s3.deleteSnapshot(ctx, s); err != nil { + if isNotExist(err) { + logrus.Infof("Snapshot %s not found in S3", s) + } else { + logrus.Errorf("Failed to delete S3 snapshot %s: %v", s, err) + } + } else { + logrus.Infof("Snapshot %s deleted from S3", s) + } + } + } + + return e.ReconcileSnapshotData(ctx) +} + +func (e *ETCD) deleteSnapshot(snapshotPath string) error { + dir := filepath.Join(filepath.Dir(snapshotPath), "..", metadataDir) + filename := filepath.Base(snapshotPath) + metadataPath := filepath.Join(dir, filename) + + err := os.Remove(snapshotPath) + if err == nil || os.IsNotExist(err) { + if merr := os.Remove(metadataPath); err != nil && !isNotExist(err) { + err = merr + } + } + + return err +} + +func marshalSnapshotFile(sf snapshotFile) ([]byte, error) { + if sf.metadataSource != nil { + if m, err := json.Marshal(sf.metadataSource.Data); err != nil { + logrus.Debugf("Error attempting to marshal extra metadata contained in %s ConfigMap, error: %v", snapshotExtraMetadataConfigMapName, err) + } else { + sf.Metadata = base64.StdEncoding.EncodeToString(m) + } + } + return json.Marshal(sf) +} + +// addSnapshotData syncs an internal snapshotFile representation to an ETCDSnapshotFile resource +// of the same name. Resources will be created or updated as necessary. +func (e *ETCD) addSnapshotData(sf snapshotFile) error { + // make sure the K3s factory is initialized. + for e.config.Runtime.K3s == nil { + runtime.Gosched() + } + + snapshots := e.config.Runtime.K3s.K3s().V1().ETCDSnapshotFile() + esfName := generateSnapshotName(sf) + + var esf *apisv1.ETCDSnapshotFile + return retry.OnError(snapshotDataBackoff, func(err error) bool { + return apierrors.IsConflict(err) || apierrors.IsAlreadyExists(err) + }, func() (err error) { + // Get current object or create new one + esf, err = snapshots.Get(esfName, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + return err + } + esf = &apisv1.ETCDSnapshotFile{ + ObjectMeta: metav1.ObjectMeta{ + Name: esfName, + }, + } + } + + // mutate object + existing := esf.DeepCopyObject() + sf.toETCDSnapshotFile(esf) + + // create or update as necessary + if esf.CreationTimestamp.IsZero() { + var created *apisv1.ETCDSnapshotFile + created, err = snapshots.Create(esf) + if err == nil { + // Only emit an event for the snapshot when creating the resource + e.emitEvent(created) + } + } else if !equality.Semantic.DeepEqual(existing, esf) { + _, err = snapshots.Update(esf) + } + return err + }) +} + +// generateSnapshotConfigMapKey generates a derived name for the snapshot that is safe for use +// as a configmap key. +func generateSnapshotConfigMapKey(sf snapshotFile) string { + name := invalidKeyChars.ReplaceAllString(sf.Name, "_") + if sf.NodeName == "s3" { + return "s3-" + name + } + return "local-" + name +} + +// generateSnapshotName generates a derived name for the snapshot that is safe for use +// as a resource name. +func generateSnapshotName(sf snapshotFile) string { + name := strings.ToLower(sf.Name) + nodename := sf.nodeSource + if nodename == "" { + nodename = sf.NodeName + } + // Include a digest of the hostname and location to ensure unique resource + // names. Snapshots should already include the hostname, but this ensures we + // don't accidentally hide records if a snapshot with the same name somehow + // exists on multiple nodes. + digest := sha256.Sum256([]byte(nodename + sf.Location)) + // If the lowercase filename isn't usable as a resource name, and short enough that we can include a prefix and suffix, + // generate a safe name derived from the hostname and timestamp. + if errs := validation.IsDNS1123Subdomain(name); len(errs) != 0 || len(name)+13 > validation.DNS1123SubdomainMaxLength { + nodename, _, _ := strings.Cut(nodename, ".") + name = fmt.Sprintf("etcd-snapshot-%s-%d", nodename, sf.CreatedAt.Unix()) + if sf.Compressed { + name += compressedExtension + } + } + if sf.NodeName == "s3" { + return "s3-" + name + "-" + hex.EncodeToString(digest[0:])[0:6] + } + return "local-" + name + "-" + hex.EncodeToString(digest[0:])[0:6] +} + +// generateETCDSnapshotFileConfigMapKey generates a key that the corresponding +// snapshotFile would be stored under in the legacy configmap +func generateETCDSnapshotFileConfigMapKey(esf apisv1.ETCDSnapshotFile) string { + name := invalidKeyChars.ReplaceAllString(esf.Spec.SnapshotName, "_") + if esf.Spec.S3 != nil { + return "s3-" + name + } + return "local-" + name +} + +func (e *ETCD) emitEvent(esf *apisv1.ETCDSnapshotFile) { + switch { + case e.config.Runtime.Event == nil: + case !esf.DeletionTimestamp.IsZero(): + e.config.Runtime.Event.Eventf(esf, v1.EventTypeNormal, "ETCDSnapshotDeleted", "Snapshot %s deleted", esf.Spec.SnapshotName) + case esf.Status.Error != nil: + message := fmt.Sprintf("Failed to save snapshot %s on %s", esf.Spec.SnapshotName, esf.Spec.NodeName) + if esf.Status.Error.Message != nil { + message += ": " + *esf.Status.Error.Message + } + e.config.Runtime.Event.Event(esf, v1.EventTypeWarning, "ETCDSnapshotFailed", message) + default: + e.config.Runtime.Event.Eventf(esf, v1.EventTypeNormal, "ETCDSnapshotCreated", "Snapshot %s saved on %s", esf.Spec.SnapshotName, esf.Spec.NodeName) + } +} + +// ReconcileSnapshotData reconciles snapshot data in the ETCDSnapshotFile resources. +// It will reconcile snapshot data from disk locally always, and if S3 is enabled, will attempt to list S3 snapshots +// and reconcile snapshots from S3. +func (e *ETCD) ReconcileSnapshotData(ctx context.Context) error { + // make sure the core.Factory is initialized. There can + // be a race between this core code startup. + for e.config.Runtime.Core == nil { + runtime.Gosched() + } + + logrus.Infof("Reconciling ETCDSnapshotFile resources") + defer logrus.Infof("Reconciliation of ETCDSnapshotFile resources complete") + + // Get snapshots from local filesystem + snapshotFiles, err := e.listLocalSnapshots() + if err != nil { + return err + } + + nodeNames := []string{os.Getenv("NODE_NAME")} + + // Get snapshots from S3 + if e.config.EtcdS3 { + if err := e.initS3IfNil(ctx); err != nil { + return err + } + + if s3Snapshots, err := e.s3.listSnapshots(ctx); err != nil { + logrus.Errorf("Error retrieving S3 snapshots for reconciliation: %v", err) + } else { + for k, v := range s3Snapshots { + snapshotFiles[k] = v + } + nodeNames = append(nodeNames, "s3") + } + } + + // Try to load metadata from the legacy configmap, in case any local or s3 snapshots + // were created by an old release that does not write the metadata alongside the snapshot file. + snapshotConfigMap, err := e.config.Runtime.Core.Core().V1().ConfigMap().Get(metav1.NamespaceSystem, snapshotConfigMapName, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + + if snapshotConfigMap != nil { + for sfKey, sf := range snapshotFiles { + logrus.Debugf("Found snapshotFile for %s with key %s", sf.Name, sfKey) + // if the configmap has data for this snapshot, and local metadata is empty, + // deserialize the value from the configmap and attempt to load it. + if cmSnapshotValue := snapshotConfigMap.Data[sfKey]; cmSnapshotValue != "" && sf.Metadata == "" && sf.metadataSource == nil { + sfTemp := &snapshotFile{} + if err := json.Unmarshal([]byte(cmSnapshotValue), sfTemp); err != nil { + logrus.Warnf("Failed to unmarshal configmap data for snapshot %s: %v", sfKey, err) + continue + } + sf.Metadata = sfTemp.Metadata + snapshotFiles[sfKey] = sf + } + } + } + + labelSelector := &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: labelStorageNode, + Operator: metav1.LabelSelectorOpIn, + Values: nodeNames, + }}, + } + + selector, err := metav1.LabelSelectorAsSelector(labelSelector) + if err != nil { + return err + } + + // List all snapshots matching the selector + snapshots := e.config.Runtime.K3s.K3s().V1().ETCDSnapshotFile() + esfList, err := snapshots.List(metav1.ListOptions{LabelSelector: selector.String()}) + if err != nil { + return err + } + + // If a snapshot from Kubernetes was found on disk/s3, it is in sync and we can remove it from the map to sync. + // If a snapshot from Kubernetes was not found on disk/s3, is is gone and can be removed from Kubernetes. + // The one exception to the last rule is failed snapshots - these must be retained for a period of time. + for _, esf := range esfList.Items { + sfKey := generateETCDSnapshotFileConfigMapKey(esf) + logrus.Debugf("Found ETCDSnapshotFile for %s with key %s", esf.Spec.SnapshotName, sfKey) + if sf, ok := snapshotFiles[sfKey]; ok && generateSnapshotName(sf) == esf.Name { + // exists in both and names match, don't need to sync + delete(snapshotFiles, sfKey) + } else { + // doesn't exist on disk - if it's an error that hasn't expired yet, leave it, otherwise remove it + if esf.Status.Error != nil && esf.Status.Error.Time != nil { + expires := esf.Status.Error.Time.Add(errorTTL) + if time.Now().Before(expires) { + continue + } + } + if ok { + logrus.Debugf("Name of ETCDSnapshotFile for snapshotFile with key %s does not match: %s vs %s", sfKey, generateSnapshotName(sf), esf.Name) + } else { + logrus.Debugf("Key %s not found in snapshotFile list", sfKey) + } + logrus.Infof("Deleting ETCDSnapshotFile for %s", esf.Spec.SnapshotName) + if err := snapshots.Delete(esf.Name, &metav1.DeleteOptions{}); err != nil { + logrus.Errorf("Failed to delete ETCDSnapshotFile: %v", err) + } + } + } + + // Any snapshots remaining in the map from disk/s3 were not found in Kubernetes and need to be created + for _, sf := range snapshotFiles { + logrus.Infof("Creating ETCDSnapshotFile for %s", sf.Name) + if err := e.addSnapshotData(sf); err != nil { + logrus.Errorf("Failed to create ETCDSnapshotFile: %v", err) + } + } + + // List all snapshots in Kubernetes not stored on S3 or a current etcd node. + // These snapshots are local to a node that no longer runs etcd and cannot be restored. + // If the node rejoins later and has local snapshots, it will reconcile them itself. + labelSelector.MatchExpressions[0].Operator = metav1.LabelSelectorOpNotIn + labelSelector.MatchExpressions[0].Values = []string{"s3"} + + // Get a list of all etcd nodes currently in the cluster and add them to the selector + nodes := e.config.Runtime.Core.Core().V1().Node() + etcdSelector := labels.Set{util.ETCDRoleLabelKey: "true"} + nodeList, err := nodes.List(metav1.ListOptions{LabelSelector: etcdSelector.String()}) + if err != nil { + return err + } + + for _, node := range nodeList.Items { + labelSelector.MatchExpressions[0].Values = append(labelSelector.MatchExpressions[0].Values, node.Name) + } + + selector, err = metav1.LabelSelectorAsSelector(labelSelector) + if err != nil { + return err + } + + // List and remove all snapshots stored on nodes that do not match the selector + esfList, err = snapshots.List(metav1.ListOptions{LabelSelector: selector.String()}) + if err != nil { + return err + } + + for _, esf := range esfList.Items { + if err := snapshots.Delete(esf.Name, &metav1.DeleteOptions{}); err != nil { + logrus.Errorf("Failed to delete ETCDSnapshotFile for non-etcd node %s: %v", esf.Spec.NodeName, err) + } + } + + // Update our Node object to note the timestamp of the snapshot storages that have been reconciled + now := time.Now().Round(time.Second).Format(time.RFC3339) + patch := []map[string]string{ + { + "op": "add", + "value": now, + "path": "/metadata/annotations/" + strings.ReplaceAll(annotationLocalReconciled, "/", "~1"), + }, + } + if e.config.EtcdS3 { + patch = append(patch, map[string]string{ + "op": "add", + "value": now, + "path": "/metadata/annotations/" + strings.ReplaceAll(annotationS3Reconciled, "/", "~1"), + }) + } + b, err := json.Marshal(patch) + if err != nil { + return err + } + _, err = nodes.Patch(nodeNames[0], types.JSONPatchType, b) + return err +} + +// setSnapshotFunction schedules snapshots at the configured interval. +func (e *ETCD) setSnapshotFunction(ctx context.Context) { + skipJob := cron.SkipIfStillRunning(cronLogger) + e.cron.AddJob(e.config.EtcdSnapshotCron, skipJob(cron.FuncJob(func() { + // Add a small amount of jitter to the actual snapshot execution. On clusters with multiple servers, + // having all the nodes take a snapshot at the exact same time can lead to excessive retry thrashing + // when updating the snapshot list configmap. + time.Sleep(time.Duration(rand.Float64() * float64(snapshotJitterMax))) + if err := e.Snapshot(ctx); err != nil { + logrus.Errorf("Failed to take scheduled snapshot: %v", err) + } + }))) +} + +// snapshotRetention iterates through the snapshots and removes the oldest +// leaving the desired number of snapshots. +func snapshotRetention(retention int, snapshotPrefix string, snapshotDir string) error { + if retention < 1 { + return nil + } + + logrus.Infof("Applying snapshot retention=%d to local snapshots with prefix %s in %s", retention, snapshotPrefix, snapshotDir) + + var snapshotFiles []snapshotFile + if err := filepath.Walk(snapshotDir, func(path string, info os.FileInfo, err error) error { + if info.IsDir() || err != nil { + return err + } + if strings.HasPrefix(info.Name(), snapshotPrefix) { + basename, compressed := strings.CutSuffix(info.Name(), compressedExtension) + ts, err := strconv.ParseInt(basename[strings.LastIndexByte(basename, '-')+1:], 10, 64) + if err != nil { + ts = info.ModTime().Unix() + } + snapshotFiles = append(snapshotFiles, snapshotFile{Name: info.Name(), CreatedAt: &metav1.Time{Time: time.Unix(ts, 0)}, Compressed: compressed}) + } + return nil + }); err != nil { + return err + } + if len(snapshotFiles) <= retention { + return nil + } + + // sort newest-first so we can prune entries past the retention count + sort.Slice(snapshotFiles, func(i, j int) bool { + return snapshotFiles[j].CreatedAt.Before(snapshotFiles[i].CreatedAt) + }) + + for _, df := range snapshotFiles[retention:] { + snapshotPath := filepath.Join(snapshotDir, df.Name) + metadataPath := filepath.Join(snapshotDir, "..", metadataDir, df.Name) + logrus.Infof("Removing local snapshot %s", snapshotPath) + if err := os.Remove(snapshotPath); err != nil { + return err + } + if err := os.Remove(metadataPath); err != nil && !os.IsNotExist(err) { + return err + } + } + + return nil +} + +func isNotExist(err error) bool { + if resp := minio.ToErrorResponse(err); resp.StatusCode == http.StatusNotFound || os.IsNotExist(err) { + return true + } + return false +} + +// saveSnapshotMetadata writes extra metadata to disk. +// The upload is silently skipped if no extra metadata is provided. +func saveSnapshotMetadata(snapshotPath string, extraMetadata *v1.ConfigMap) error { + if extraMetadata == nil || len(extraMetadata.Data) == 0 { + return nil + } + + dir := filepath.Join(filepath.Dir(snapshotPath), "..", metadataDir) + filename := filepath.Base(snapshotPath) + metadataPath := filepath.Join(dir, filename) + logrus.Infof("Saving snapshot metadata to %s", metadataPath) + m, err := json.Marshal(extraMetadata.Data) + if err != nil { + return err + } + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + return os.WriteFile(metadataPath, m, 0700) +} + +func (sf *snapshotFile) fromETCDSnapshotFile(esf *apisv1.ETCDSnapshotFile) { + if esf == nil { + panic("cannot convert from nil ETCDSnapshotFile") + } + + sf.Name = esf.Spec.SnapshotName + sf.Location = esf.Spec.Location + sf.CreatedAt = esf.Status.CreationTime + sf.nodeSource = esf.Spec.NodeName + sf.Compressed = strings.HasSuffix(esf.Spec.SnapshotName, compressedExtension) + + if esf.Status.ReadyToUse != nil && *esf.Status.ReadyToUse { + sf.Status = successfulSnapshotStatus + } else { + sf.Status = failedSnapshotStatus + } + + if esf.Status.Size != nil { + sf.Size = esf.Status.Size.Value() + } + + if esf.Status.Error != nil { + if esf.Status.Error.Time != nil { + sf.CreatedAt = esf.Status.Error.Time + } + message := "etcd snapshot failed" + if esf.Status.Error.Message != nil { + message = *esf.Status.Error.Message + } + sf.Message = base64.StdEncoding.EncodeToString([]byte(message)) + } + + if len(esf.Spec.Metadata) > 0 { + if b, err := json.Marshal(esf.Spec.Metadata); err != nil { + logrus.Warnf("Failed to marshal metadata for %s: %v", esf.Name, err) + } else { + sf.Metadata = base64.StdEncoding.EncodeToString(b) + } + } + + if tokenHash := esf.Annotations[annotationTokenHash]; tokenHash != "" { + sf.tokenHash = tokenHash + } + + if esf.Spec.S3 == nil { + sf.NodeName = esf.Spec.NodeName + } else { + sf.NodeName = "s3" + sf.S3 = &s3Config{ + Endpoint: esf.Spec.S3.Endpoint, + EndpointCA: esf.Spec.S3.EndpointCA, + SkipSSLVerify: esf.Spec.S3.SkipSSLVerify, + Bucket: esf.Spec.S3.Bucket, + Region: esf.Spec.S3.Region, + Folder: esf.Spec.S3.Prefix, + Insecure: esf.Spec.S3.Insecure, + } + } +} + +func (sf *snapshotFile) toETCDSnapshotFile(esf *apisv1.ETCDSnapshotFile) { + if esf == nil { + panic("cannot convert to nil ETCDSnapshotFile") + } + esf.Spec.SnapshotName = sf.Name + esf.Spec.Location = sf.Location + esf.Status.CreationTime = sf.CreatedAt + esf.Status.ReadyToUse = pointer.Bool(sf.Status == successfulSnapshotStatus) + esf.Status.Size = resource.NewQuantity(sf.Size, resource.DecimalSI) + + if sf.nodeSource != "" { + esf.Spec.NodeName = sf.nodeSource + } else { + esf.Spec.NodeName = sf.NodeName + } + + if sf.Message != "" { + var message string + b, err := base64.StdEncoding.DecodeString(sf.Message) + if err != nil { + logrus.Warnf("Failed to decode error message for %s: %v", sf.Name, err) + message = "etcd snapshot failed" + } else { + message = string(b) + } + esf.Status.Error = &apisv1.ETCDSnapshotError{ + Time: sf.CreatedAt, + Message: &message, + } + } + + if sf.metadataSource != nil { + esf.Spec.Metadata = sf.metadataSource.Data + } else if sf.Metadata != "" { + metadata, err := base64.StdEncoding.DecodeString(sf.Metadata) + if err != nil { + logrus.Warnf("Failed to decode metadata for %s: %v", sf.Name, err) + } else { + if err := json.Unmarshal(metadata, &esf.Spec.Metadata); err != nil { + logrus.Warnf("Failed to unmarshal metadata for %s: %v", sf.Name, err) + } + } + } + + if esf.ObjectMeta.Labels == nil { + esf.ObjectMeta.Labels = map[string]string{} + } + + if esf.ObjectMeta.Annotations == nil { + esf.ObjectMeta.Annotations = map[string]string{} + } + + if sf.tokenHash != "" { + esf.ObjectMeta.Annotations[annotationTokenHash] = sf.tokenHash + } + + if sf.S3 == nil { + esf.ObjectMeta.Labels[labelStorageNode] = esf.Spec.NodeName + } else { + esf.ObjectMeta.Labels[labelStorageNode] = "s3" + esf.Spec.S3 = &apisv1.ETCDSnapshotS3{ + Endpoint: sf.S3.Endpoint, + EndpointCA: sf.S3.EndpointCA, + SkipSSLVerify: sf.S3.SkipSSLVerify, + Bucket: sf.S3.Bucket, + Region: sf.S3.Region, + Prefix: sf.S3.Folder, + Insecure: sf.S3.Insecure, + } + } +} diff --git a/pkg/etcd/snapshot_controller.go b/pkg/etcd/snapshot_controller.go new file mode 100644 index 000000000000..7da376741b40 --- /dev/null +++ b/pkg/etcd/snapshot_controller.go @@ -0,0 +1,312 @@ +package etcd + +import ( + "context" + "sort" + "strconv" + "strings" + "time" + + apisv1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1" + controllersv1 "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io/v1" + "github.com/k3s-io/k3s/pkg/util" + "github.com/k3s-io/k3s/pkg/version" + "github.com/pkg/errors" + controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + + "github.com/sirupsen/logrus" +) + +const ( + pruneStepSize = 4 + reconcileKey = "_reconcile_" + reconcileInterval = 600 * time.Minute +) + +var ( + snapshotConfigMapName = version.Program + "-etcd-snapshots" +) + +type etcdSnapshotHandler struct { + ctx context.Context + etcd *ETCD + snapshots controllersv1.ETCDSnapshotFileController + configmaps controllerv1.ConfigMapController +} + +func registerSnapshotHandlers(ctx context.Context, etcd *ETCD) { + snapshots := etcd.config.Runtime.K3s.K3s().V1().ETCDSnapshotFile() + e := &etcdSnapshotHandler{ + ctx: ctx, + etcd: etcd, + snapshots: snapshots, + configmaps: etcd.config.Runtime.Core.Core().V1().ConfigMap(), + } + + logrus.Infof("Starting managed etcd snapshot ConfigMap controller") + snapshots.OnChange(ctx, "managed-etcd-snapshots-controller", e.sync) + snapshots.OnRemove(ctx, "managed-etcd-snapshots-controller", e.onRemove) + go wait.JitterUntil(func() { snapshots.Enqueue(reconcileKey) }, reconcileInterval, 0.04, false, ctx.Done()) +} + +func (e *etcdSnapshotHandler) sync(key string, esf *apisv1.ETCDSnapshotFile) (*apisv1.ETCDSnapshotFile, error) { + if key == reconcileKey { + return nil, e.reconcile() + } + if esf == nil || !esf.DeletionTimestamp.IsZero() { + return nil, nil + } + + sf := snapshotFile{} + sf.fromETCDSnapshotFile(esf) + sfKey := generateSnapshotConfigMapKey(sf) + m, err := marshalSnapshotFile(sf) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal snapshot ConfigMap data") + } + marshalledSnapshot := string(m) + + snapshotConfigMap, err := e.configmaps.Get(metav1.NamespaceSystem, snapshotConfigMapName, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + return nil, errors.Wrap(err, "failed to get snapshot ConfigMap") + } + snapshotConfigMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: snapshotConfigMapName, + Namespace: metav1.NamespaceSystem, + }, + } + } + + if snapshotConfigMap.Data[sfKey] != marshalledSnapshot { + if snapshotConfigMap.Data == nil { + snapshotConfigMap.Data = map[string]string{} + } + snapshotConfigMap.Data[sfKey] = marshalledSnapshot + + // Try to create or update the ConfigMap. If it is too large, prune old entries + // until it fits, or until it cannot be pruned any further. + pruneCount := pruneStepSize + err = retry.OnError(snapshotDataBackoff, isTooLargeError, func() (err error) { + if snapshotConfigMap.CreationTimestamp.IsZero() { + _, err = e.configmaps.Create(snapshotConfigMap) + } else { + _, err = e.configmaps.Update(snapshotConfigMap) + } + + if isTooLargeError(err) { + logrus.Warnf("Snapshot ConfigMap is too large, attempting to elide %d of %d entries to reduce size", pruneCount, len(snapshotConfigMap.Data)) + if perr := pruneConfigMap(snapshotConfigMap, pruneCount); perr != nil { + err = perr + } + // if the entry we're trying to add just got pruned, give up on adding it, + // as it is always going to get pushed off due to being too old to keep. + if _, ok := snapshotConfigMap.Data[sfKey]; !ok { + logrus.Warnf("Snapshot %s has been elided from ConfigMap to reduce size; not requeuing", key) + return nil + } + + pruneCount += pruneStepSize + } + return err + }) + } + + if err != nil { + err = errors.Wrap(err, "failed to sync snapshot to ConfigMap") + } + + return nil, err +} + +func (e *etcdSnapshotHandler) onRemove(key string, esf *apisv1.ETCDSnapshotFile) (*apisv1.ETCDSnapshotFile, error) { + if esf == nil { + return nil, nil + } + snapshotConfigMap, err := e.configmaps.Get(metav1.NamespaceSystem, snapshotConfigMapName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, errors.Wrap(err, "failed to get snapshot ConfigMap") + } + + sfKey := generateETCDSnapshotFileConfigMapKey(*esf) + if _, ok := snapshotConfigMap.Data[sfKey]; ok { + delete(snapshotConfigMap.Data, sfKey) + if _, err := e.configmaps.Update(snapshotConfigMap); err != nil { + return nil, errors.Wrap(err, "failed to remove snapshot from ConfigMap") + } + } + e.etcd.emitEvent(esf) + return nil, nil +} + +func (e *etcdSnapshotHandler) reconcile() error { + logrus.Infof("Reconciling snapshot ConfigMap data") + + snapshotConfigMap, err := e.configmaps.Get(metav1.NamespaceSystem, snapshotConfigMapName, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + return errors.Wrap(err, "failed to get snapshot ConfigMap") + } + snapshotConfigMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: snapshotConfigMapName, + Namespace: metav1.NamespaceSystem, + }, + } + } + + // Get a list of all etcd nodes currently in the cluster. + // We will use this list to prune local entries for any node that does not exist. + nodes := e.etcd.config.Runtime.Core.Core().V1().Node() + etcdSelector := labels.Set{util.ETCDRoleLabelKey: "true"} + nodeList, err := nodes.List(metav1.ListOptions{LabelSelector: etcdSelector.String()}) + if err != nil { + return err + } + + // Once a node has set the reconcile annotation, it is considered to have + // migrated to using ETCDSnapshotFile resources, and any old configmap + // entries for it can be pruned. Until the annotation is set, we will leave + // its entries alone. + syncedNodes := map[string]bool{} + for _, node := range nodeList.Items { + if _, ok := node.Annotations[annotationLocalReconciled]; ok { + syncedNodes[node.Name] = true + } + if _, ok := node.Annotations[annotationS3Reconciled]; ok { + syncedNodes["s3"] = true + } + } + + if len(syncedNodes) == 0 { + return errors.New("no nodes have reconciled ETCDSnapshotFile resources") + } + + // Get a list of existing snapshots + snapshotList, err := e.snapshots.List(metav1.ListOptions{}) + if err != nil { + return err + } + + snapshots := map[string]*apisv1.ETCDSnapshotFile{} + for i := range snapshotList.Items { + esf := &snapshotList.Items[i] + if esf.DeletionTimestamp.IsZero() { + sfKey := generateETCDSnapshotFileConfigMapKey(*esf) + snapshots[sfKey] = esf + } + } + + // Make a copy of the configmap for change detection + existing := snapshotConfigMap.DeepCopyObject() + + // Delete any keys missing from synced storages, or associated with missing nodes + for key := range snapshotConfigMap.Data { + if strings.HasPrefix(key, "s3-") { + // If a node has syncd s3 and the key is missing then delete it + if syncedNodes["s3"] && snapshots[key] == nil { + delete(snapshotConfigMap.Data, key) + } + } else if s, ok := strings.CutPrefix(key, "local-"); ok { + // If a matching node has synced and the key is missing then delete it + // If a matching node does not exist, delete the key + // A node is considered to match the snapshot if the snapshot name matches the node name + // after trimming the leading local- prefix and trailing timestamp and extension. + s, _ = strings.CutSuffix(s, ".zip") + s = strings.TrimRight(s, "-012345678") + var matchingNode bool + for _, node := range nodeList.Items { + if strings.HasSuffix(s, node.Name) { + if syncedNodes[node.Name] && snapshots[key] == nil { + delete(snapshotConfigMap.Data, key) + } + matchingNode = true + break + } + } + if !matchingNode { + delete(snapshotConfigMap.Data, key) + } + } + } + + // Ensure keys for existing snapshots + for sfKey, esf := range snapshots { + sf := snapshotFile{} + sf.fromETCDSnapshotFile(esf) + m, err := marshalSnapshotFile(sf) + if err != nil { + logrus.Warnf("Failed to marshal snapshot ConfigMap data for %s", sfKey) + continue + } + marshalledSnapshot := string(m) + snapshotConfigMap.Data[sfKey] = marshalledSnapshot + } + + // If the configmap didn't change, don't bother updating it + if equality.Semantic.DeepEqual(existing, snapshotConfigMap) { + return nil + } + + // Try to create or update the ConfigMap. If it is too large, prune old entries + // until it fits, or until it cannot be pruned any further. + pruneCount := pruneStepSize + return retry.OnError(snapshotDataBackoff, isTooLargeError, func() (err error) { + if snapshotConfigMap.CreationTimestamp.IsZero() { + _, err = e.configmaps.Create(snapshotConfigMap) + } else { + _, err = e.configmaps.Update(snapshotConfigMap) + } + + if isTooLargeError(err) { + logrus.Warnf("Snapshot ConfigMap is too large, attempting to elide %d of %d entries to reduce size", pruneCount, len(snapshotConfigMap.Data)) + if perr := pruneConfigMap(snapshotConfigMap, pruneCount); perr != nil { + err = perr + } + pruneCount += pruneStepSize + } + return err + }) +} + +// pruneConfigMap drops the oldest entries from the configMap. +// Note that the actual snapshot files are not removed, just the entries that track them in the configmap. +func pruneConfigMap(snapshotConfigMap *v1.ConfigMap, pruneCount int) error { + if pruneCount >= len(snapshotConfigMap.Data) { + return errors.New("unable to reduce snapshot ConfigMap size by eliding old snapshots") + } + + var snapshotFiles []snapshotFile + retention := len(snapshotConfigMap.Data) - pruneCount + for name := range snapshotConfigMap.Data { + basename, compressed := strings.CutSuffix(name, compressedExtension) + ts, _ := strconv.ParseInt(basename[strings.LastIndexByte(basename, '-')+1:], 10, 64) + snapshotFiles = append(snapshotFiles, snapshotFile{Name: name, CreatedAt: &metav1.Time{Time: time.Unix(ts, 0)}, Compressed: compressed}) + } + + // sort newest-first so we can prune entries past the retention count + sort.Slice(snapshotFiles, func(i, j int) bool { + return snapshotFiles[j].CreatedAt.Before(snapshotFiles[i].CreatedAt) + }) + + for _, snapshotFile := range snapshotFiles[retention:] { + delete(snapshotConfigMap.Data, snapshotFile.Name) + } + return nil +} + +func isTooLargeError(err error) bool { + // There are no helpers for unpacking field validation errors, so we just check for "Too long" in the error string. + return apierrors.IsRequestEntityTooLargeError(err) || (apierrors.IsInvalid(err) && strings.Contains(err.Error(), "Too long")) +} diff --git a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/etcdsnapshotfile.go b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/etcdsnapshotfile.go new file mode 100644 index 000000000000..148cd2af8340 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/etcdsnapshotfile.go @@ -0,0 +1,184 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by main. DO NOT EDIT. + +package v1 + +import ( + "context" + "time" + + v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1" + scheme "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// ETCDSnapshotFilesGetter has a method to return a ETCDSnapshotFileInterface. +// A group's client should implement this interface. +type ETCDSnapshotFilesGetter interface { + ETCDSnapshotFiles() ETCDSnapshotFileInterface +} + +// ETCDSnapshotFileInterface has methods to work with ETCDSnapshotFile resources. +type ETCDSnapshotFileInterface interface { + Create(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.CreateOptions) (*v1.ETCDSnapshotFile, error) + Update(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (*v1.ETCDSnapshotFile, error) + UpdateStatus(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (*v1.ETCDSnapshotFile, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.ETCDSnapshotFile, error) + List(ctx context.Context, opts metav1.ListOptions) (*v1.ETCDSnapshotFileList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.ETCDSnapshotFile, err error) + ETCDSnapshotFileExpansion +} + +// eTCDSnapshotFiles implements ETCDSnapshotFileInterface +type eTCDSnapshotFiles struct { + client rest.Interface +} + +// newETCDSnapshotFiles returns a ETCDSnapshotFiles +func newETCDSnapshotFiles(c *K3sV1Client) *eTCDSnapshotFiles { + return &eTCDSnapshotFiles{ + client: c.RESTClient(), + } +} + +// Get takes name of the eTCDSnapshotFile, and returns the corresponding eTCDSnapshotFile object, and an error if there is any. +func (c *eTCDSnapshotFiles) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.ETCDSnapshotFile, err error) { + result = &v1.ETCDSnapshotFile{} + err = c.client.Get(). + Resource("etcdsnapshotfiles"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of ETCDSnapshotFiles that match those selectors. +func (c *eTCDSnapshotFiles) List(ctx context.Context, opts metav1.ListOptions) (result *v1.ETCDSnapshotFileList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1.ETCDSnapshotFileList{} + err = c.client.Get(). + Resource("etcdsnapshotfiles"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested eTCDSnapshotFiles. +func (c *eTCDSnapshotFiles) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("etcdsnapshotfiles"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a eTCDSnapshotFile and creates it. Returns the server's representation of the eTCDSnapshotFile, and an error, if there is any. +func (c *eTCDSnapshotFiles) Create(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.CreateOptions) (result *v1.ETCDSnapshotFile, err error) { + result = &v1.ETCDSnapshotFile{} + err = c.client.Post(). + Resource("etcdsnapshotfiles"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(eTCDSnapshotFile). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a eTCDSnapshotFile and updates it. Returns the server's representation of the eTCDSnapshotFile, and an error, if there is any. +func (c *eTCDSnapshotFiles) Update(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (result *v1.ETCDSnapshotFile, err error) { + result = &v1.ETCDSnapshotFile{} + err = c.client.Put(). + Resource("etcdsnapshotfiles"). + Name(eTCDSnapshotFile.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(eTCDSnapshotFile). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *eTCDSnapshotFiles) UpdateStatus(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (result *v1.ETCDSnapshotFile, err error) { + result = &v1.ETCDSnapshotFile{} + err = c.client.Put(). + Resource("etcdsnapshotfiles"). + Name(eTCDSnapshotFile.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(eTCDSnapshotFile). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the eTCDSnapshotFile and deletes it. Returns an error if one occurs. +func (c *eTCDSnapshotFiles) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { + return c.client.Delete(). + Resource("etcdsnapshotfiles"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *eTCDSnapshotFiles) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Resource("etcdsnapshotfiles"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched eTCDSnapshotFile. +func (c *eTCDSnapshotFiles) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.ETCDSnapshotFile, err error) { + result = &v1.ETCDSnapshotFile{} + err = c.client.Patch(pt). + Resource("etcdsnapshotfiles"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_etcdsnapshotfile.go b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_etcdsnapshotfile.go new file mode 100644 index 000000000000..b4ad567c34d4 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_etcdsnapshotfile.go @@ -0,0 +1,132 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by main. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeETCDSnapshotFiles implements ETCDSnapshotFileInterface +type FakeETCDSnapshotFiles struct { + Fake *FakeK3sV1 +} + +var etcdsnapshotfilesResource = v1.SchemeGroupVersion.WithResource("etcdsnapshotfiles") + +var etcdsnapshotfilesKind = v1.SchemeGroupVersion.WithKind("ETCDSnapshotFile") + +// Get takes name of the eTCDSnapshotFile, and returns the corresponding eTCDSnapshotFile object, and an error if there is any. +func (c *FakeETCDSnapshotFiles) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.ETCDSnapshotFile, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(etcdsnapshotfilesResource, name), &v1.ETCDSnapshotFile{}) + if obj == nil { + return nil, err + } + return obj.(*v1.ETCDSnapshotFile), err +} + +// List takes label and field selectors, and returns the list of ETCDSnapshotFiles that match those selectors. +func (c *FakeETCDSnapshotFiles) List(ctx context.Context, opts metav1.ListOptions) (result *v1.ETCDSnapshotFileList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(etcdsnapshotfilesResource, etcdsnapshotfilesKind, opts), &v1.ETCDSnapshotFileList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1.ETCDSnapshotFileList{ListMeta: obj.(*v1.ETCDSnapshotFileList).ListMeta} + for _, item := range obj.(*v1.ETCDSnapshotFileList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested eTCDSnapshotFiles. +func (c *FakeETCDSnapshotFiles) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(etcdsnapshotfilesResource, opts)) +} + +// Create takes the representation of a eTCDSnapshotFile and creates it. Returns the server's representation of the eTCDSnapshotFile, and an error, if there is any. +func (c *FakeETCDSnapshotFiles) Create(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.CreateOptions) (result *v1.ETCDSnapshotFile, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(etcdsnapshotfilesResource, eTCDSnapshotFile), &v1.ETCDSnapshotFile{}) + if obj == nil { + return nil, err + } + return obj.(*v1.ETCDSnapshotFile), err +} + +// Update takes the representation of a eTCDSnapshotFile and updates it. Returns the server's representation of the eTCDSnapshotFile, and an error, if there is any. +func (c *FakeETCDSnapshotFiles) Update(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (result *v1.ETCDSnapshotFile, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(etcdsnapshotfilesResource, eTCDSnapshotFile), &v1.ETCDSnapshotFile{}) + if obj == nil { + return nil, err + } + return obj.(*v1.ETCDSnapshotFile), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeETCDSnapshotFiles) UpdateStatus(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (*v1.ETCDSnapshotFile, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(etcdsnapshotfilesResource, "status", eTCDSnapshotFile), &v1.ETCDSnapshotFile{}) + if obj == nil { + return nil, err + } + return obj.(*v1.ETCDSnapshotFile), err +} + +// Delete takes name of the eTCDSnapshotFile and deletes it. Returns an error if one occurs. +func (c *FakeETCDSnapshotFiles) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteActionWithOptions(etcdsnapshotfilesResource, name, opts), &v1.ETCDSnapshotFile{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeETCDSnapshotFiles) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(etcdsnapshotfilesResource, listOpts) + + _, err := c.Fake.Invokes(action, &v1.ETCDSnapshotFileList{}) + return err +} + +// Patch applies the patch and returns the patched eTCDSnapshotFile. +func (c *FakeETCDSnapshotFiles) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.ETCDSnapshotFile, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(etcdsnapshotfilesResource, name, pt, data, subresources...), &v1.ETCDSnapshotFile{}) + if obj == nil { + return nil, err + } + return obj.(*v1.ETCDSnapshotFile), err +} diff --git a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_k3s.cattle.io_client.go b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_k3s.cattle.io_client.go index 562baa963898..7167f94bf941 100644 --- a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_k3s.cattle.io_client.go +++ b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_k3s.cattle.io_client.go @@ -32,6 +32,10 @@ func (c *FakeK3sV1) Addons(namespace string) v1.AddonInterface { return &FakeAddons{c, namespace} } +func (c *FakeK3sV1) ETCDSnapshotFiles() v1.ETCDSnapshotFileInterface { + return &FakeETCDSnapshotFiles{c} +} + // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *FakeK3sV1) RESTClient() rest.Interface { diff --git a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/generated_expansion.go index 1b681d3f1fe3..d152245a2913 100644 --- a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/generated_expansion.go @@ -19,3 +19,5 @@ limitations under the License. package v1 type AddonExpansion interface{} + +type ETCDSnapshotFileExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/k3s.cattle.io_client.go b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/k3s.cattle.io_client.go index a1e0d1fbafa6..77bd599332e5 100644 --- a/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/k3s.cattle.io_client.go +++ b/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/k3s.cattle.io_client.go @@ -29,6 +29,7 @@ import ( type K3sV1Interface interface { RESTClient() rest.Interface AddonsGetter + ETCDSnapshotFilesGetter } // K3sV1Client is used to interact with features provided by the k3s.cattle.io group. @@ -40,6 +41,10 @@ func (c *K3sV1Client) Addons(namespace string) AddonInterface { return newAddons(c, namespace) } +func (c *K3sV1Client) ETCDSnapshotFiles() ETCDSnapshotFileInterface { + return newETCDSnapshotFiles(c) +} + // NewForConfig creates a new K3sV1Client for the given config. // NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), // where httpClient was generated with rest.HTTPClientFor(c). diff --git a/pkg/generated/controllers/k3s.cattle.io/v1/etcdsnapshotfile.go b/pkg/generated/controllers/k3s.cattle.io/v1/etcdsnapshotfile.go new file mode 100644 index 000000000000..ad9a1cdf3052 --- /dev/null +++ b/pkg/generated/controllers/k3s.cattle.io/v1/etcdsnapshotfile.go @@ -0,0 +1,258 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by main. DO NOT EDIT. + +package v1 + +import ( + "context" + "time" + + v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1" + "github.com/rancher/wrangler/pkg/apply" + "github.com/rancher/wrangler/pkg/condition" + "github.com/rancher/wrangler/pkg/generic" + "github.com/rancher/wrangler/pkg/kv" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" +) + +// ETCDSnapshotFileController interface for managing ETCDSnapshotFile resources. +type ETCDSnapshotFileController interface { + generic.ControllerMeta + ETCDSnapshotFileClient + + // OnChange runs the given handler when the controller detects a resource was changed. + OnChange(ctx context.Context, name string, sync ETCDSnapshotFileHandler) + + // OnRemove runs the given handler when the controller detects a resource was changed. + OnRemove(ctx context.Context, name string, sync ETCDSnapshotFileHandler) + + // Enqueue adds the resource with the given name to the worker queue of the controller. + Enqueue(name string) + + // EnqueueAfter runs Enqueue after the provided duration. + EnqueueAfter(name string, duration time.Duration) + + // Cache returns a cache for the resource type T. + Cache() ETCDSnapshotFileCache +} + +// ETCDSnapshotFileClient interface for managing ETCDSnapshotFile resources in Kubernetes. +type ETCDSnapshotFileClient interface { + // Create creates a new object and return the newly created Object or an error. + Create(*v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) + + // Update updates the object and return the newly updated Object or an error. + Update(*v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) + // UpdateStatus updates the Status field of a the object and return the newly updated Object or an error. + // Will always return an error if the object does not have a status field. + UpdateStatus(*v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) + + // Delete deletes the Object in the given name. + Delete(name string, options *metav1.DeleteOptions) error + + // Get will attempt to retrieve the resource with the specified name. + Get(name string, options metav1.GetOptions) (*v1.ETCDSnapshotFile, error) + + // List will attempt to find multiple resources. + List(opts metav1.ListOptions) (*v1.ETCDSnapshotFileList, error) + + // Watch will start watching resources. + Watch(opts metav1.ListOptions) (watch.Interface, error) + + // Patch will patch the resource with the matching name. + Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.ETCDSnapshotFile, err error) +} + +// ETCDSnapshotFileCache interface for retrieving ETCDSnapshotFile resources in memory. +type ETCDSnapshotFileCache interface { + // Get returns the resources with the specified name from the cache. + Get(name string) (*v1.ETCDSnapshotFile, error) + + // List will attempt to find resources from the Cache. + List(selector labels.Selector) ([]*v1.ETCDSnapshotFile, error) + + // AddIndexer adds a new Indexer to the cache with the provided name. + // If you call this after you already have data in the store, the results are undefined. + AddIndexer(indexName string, indexer ETCDSnapshotFileIndexer) + + // GetByIndex returns the stored objects whose set of indexed values + // for the named index includes the given indexed value. + GetByIndex(indexName, key string) ([]*v1.ETCDSnapshotFile, error) +} + +// ETCDSnapshotFileHandler is function for performing any potential modifications to a ETCDSnapshotFile resource. +type ETCDSnapshotFileHandler func(string, *v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) + +// ETCDSnapshotFileIndexer computes a set of indexed values for the provided object. +type ETCDSnapshotFileIndexer func(obj *v1.ETCDSnapshotFile) ([]string, error) + +// ETCDSnapshotFileGenericController wraps wrangler/pkg/generic.NonNamespacedController so that the function definitions adhere to ETCDSnapshotFileController interface. +type ETCDSnapshotFileGenericController struct { + generic.NonNamespacedControllerInterface[*v1.ETCDSnapshotFile, *v1.ETCDSnapshotFileList] +} + +// OnChange runs the given resource handler when the controller detects a resource was changed. +func (c *ETCDSnapshotFileGenericController) OnChange(ctx context.Context, name string, sync ETCDSnapshotFileHandler) { + c.NonNamespacedControllerInterface.OnChange(ctx, name, generic.ObjectHandler[*v1.ETCDSnapshotFile](sync)) +} + +// OnRemove runs the given object handler when the controller detects a resource was changed. +func (c *ETCDSnapshotFileGenericController) OnRemove(ctx context.Context, name string, sync ETCDSnapshotFileHandler) { + c.NonNamespacedControllerInterface.OnRemove(ctx, name, generic.ObjectHandler[*v1.ETCDSnapshotFile](sync)) +} + +// Cache returns a cache of resources in memory. +func (c *ETCDSnapshotFileGenericController) Cache() ETCDSnapshotFileCache { + return &ETCDSnapshotFileGenericCache{ + c.NonNamespacedControllerInterface.Cache(), + } +} + +// ETCDSnapshotFileGenericCache wraps wrangler/pkg/generic.NonNamespacedCache so the function definitions adhere to ETCDSnapshotFileCache interface. +type ETCDSnapshotFileGenericCache struct { + generic.NonNamespacedCacheInterface[*v1.ETCDSnapshotFile] +} + +// AddIndexer adds a new Indexer to the cache with the provided name. +// If you call this after you already have data in the store, the results are undefined. +func (c ETCDSnapshotFileGenericCache) AddIndexer(indexName string, indexer ETCDSnapshotFileIndexer) { + c.NonNamespacedCacheInterface.AddIndexer(indexName, generic.Indexer[*v1.ETCDSnapshotFile](indexer)) +} + +type ETCDSnapshotFileStatusHandler func(obj *v1.ETCDSnapshotFile, status v1.ETCDSnapshotStatus) (v1.ETCDSnapshotStatus, error) + +type ETCDSnapshotFileGeneratingHandler func(obj *v1.ETCDSnapshotFile, status v1.ETCDSnapshotStatus) ([]runtime.Object, v1.ETCDSnapshotStatus, error) + +func FromETCDSnapshotFileHandlerToHandler(sync ETCDSnapshotFileHandler) generic.Handler { + return generic.FromObjectHandlerToHandler(generic.ObjectHandler[*v1.ETCDSnapshotFile](sync)) +} + +func RegisterETCDSnapshotFileStatusHandler(ctx context.Context, controller ETCDSnapshotFileController, condition condition.Cond, name string, handler ETCDSnapshotFileStatusHandler) { + statusHandler := &eTCDSnapshotFileStatusHandler{ + client: controller, + condition: condition, + handler: handler, + } + controller.AddGenericHandler(ctx, name, FromETCDSnapshotFileHandlerToHandler(statusHandler.sync)) +} + +func RegisterETCDSnapshotFileGeneratingHandler(ctx context.Context, controller ETCDSnapshotFileController, apply apply.Apply, + condition condition.Cond, name string, handler ETCDSnapshotFileGeneratingHandler, opts *generic.GeneratingHandlerOptions) { + statusHandler := &eTCDSnapshotFileGeneratingHandler{ + ETCDSnapshotFileGeneratingHandler: handler, + apply: apply, + name: name, + gvk: controller.GroupVersionKind(), + } + if opts != nil { + statusHandler.opts = *opts + } + controller.OnChange(ctx, name, statusHandler.Remove) + RegisterETCDSnapshotFileStatusHandler(ctx, controller, condition, name, statusHandler.Handle) +} + +type eTCDSnapshotFileStatusHandler struct { + client ETCDSnapshotFileClient + condition condition.Cond + handler ETCDSnapshotFileStatusHandler +} + +func (a *eTCDSnapshotFileStatusHandler) sync(key string, obj *v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) { + if obj == nil { + return obj, nil + } + + origStatus := obj.Status.DeepCopy() + obj = obj.DeepCopy() + newStatus, err := a.handler(obj, obj.Status) + if err != nil { + // Revert to old status on error + newStatus = *origStatus.DeepCopy() + } + + if a.condition != "" { + if errors.IsConflict(err) { + a.condition.SetError(&newStatus, "", nil) + } else { + a.condition.SetError(&newStatus, "", err) + } + } + if !equality.Semantic.DeepEqual(origStatus, &newStatus) { + if a.condition != "" { + // Since status has changed, update the lastUpdatedTime + a.condition.LastUpdated(&newStatus, time.Now().UTC().Format(time.RFC3339)) + } + + var newErr error + obj.Status = newStatus + newObj, newErr := a.client.UpdateStatus(obj) + if err == nil { + err = newErr + } + if newErr == nil { + obj = newObj + } + } + return obj, err +} + +type eTCDSnapshotFileGeneratingHandler struct { + ETCDSnapshotFileGeneratingHandler + apply apply.Apply + opts generic.GeneratingHandlerOptions + gvk schema.GroupVersionKind + name string +} + +func (a *eTCDSnapshotFileGeneratingHandler) Remove(key string, obj *v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) { + if obj != nil { + return obj, nil + } + + obj = &v1.ETCDSnapshotFile{} + obj.Namespace, obj.Name = kv.RSplit(key, "/") + obj.SetGroupVersionKind(a.gvk) + + return nil, generic.ConfigureApplyForObject(a.apply, obj, &a.opts). + WithOwner(obj). + WithSetID(a.name). + ApplyObjects() +} + +func (a *eTCDSnapshotFileGeneratingHandler) Handle(obj *v1.ETCDSnapshotFile, status v1.ETCDSnapshotStatus) (v1.ETCDSnapshotStatus, error) { + if !obj.DeletionTimestamp.IsZero() { + return status, nil + } + + objs, newStatus, err := a.ETCDSnapshotFileGeneratingHandler(obj, status) + if err != nil { + return newStatus, err + } + + return newStatus, generic.ConfigureApplyForObject(a.apply, obj, &a.opts). + WithOwner(obj). + WithSetID(a.name). + ApplyObjects(objs...) +} diff --git a/pkg/generated/controllers/k3s.cattle.io/v1/interface.go b/pkg/generated/controllers/k3s.cattle.io/v1/interface.go index 12b3029d6b71..ba85bd1e850d 100644 --- a/pkg/generated/controllers/k3s.cattle.io/v1/interface.go +++ b/pkg/generated/controllers/k3s.cattle.io/v1/interface.go @@ -32,6 +32,7 @@ func init() { type Interface interface { Addon() AddonController + ETCDSnapshotFile() ETCDSnapshotFileController } func New(controllerFactory controller.SharedControllerFactory) Interface { @@ -49,3 +50,9 @@ func (v *version) Addon() AddonController { generic.NewController[*v1.Addon, *v1.AddonList](schema.GroupVersionKind{Group: "k3s.cattle.io", Version: "v1", Kind: "Addon"}, "addons", true, v.controllerFactory), } } + +func (v *version) ETCDSnapshotFile() ETCDSnapshotFileController { + return &ETCDSnapshotFileGenericController{ + generic.NewNonNamespacedController[*v1.ETCDSnapshotFile, *v1.ETCDSnapshotFileList](schema.GroupVersionKind{Group: "k3s.cattle.io", Version: "v1", Kind: "ETCDSnapshotFile"}, "etcdsnapshotfiles", v.controllerFactory), + } +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 2734f6ed5196..7ddc7c23fa18 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -114,6 +114,7 @@ func runControllers(ctx context.Context, config *Config) error { controlConfig.Runtime.NodePasswdFile); err != nil { logrus.Warn(errors.Wrap(err, "error migrating node-password file")) } + controlConfig.Runtime.K3s = sc.K3s controlConfig.Runtime.Event = sc.Event controlConfig.Runtime.Core = sc.Core diff --git a/pkg/util/token.go b/pkg/util/token.go index a47a4eefd99d..c4d3495af2bd 100644 --- a/pkg/util/token.go +++ b/pkg/util/token.go @@ -1,8 +1,16 @@ package util import ( + "bytes" cryptorand "crypto/rand" + "crypto/sha256" "encoding/hex" + "os" + "path/filepath" + + "github.com/k3s-io/k3s/pkg/clientaccess" + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/pkg/errors" ) func Random(size int) (string, error) { @@ -13,3 +21,57 @@ func Random(size int) (string, error) { } return hex.EncodeToString(token), err } + +// ReadTokenFromFile will attempt to get the token from /token if it the file not found +// in case of fresh installation it will try to use the runtime serverToken saved in memory +// after stripping it from any additional information like the username or cahash, if the file +// found then it will still strip the token from any additional info +func ReadTokenFromFile(serverToken, certs, dataDir string) (string, error) { + tokenFile := filepath.Join(dataDir, "token") + + b, err := os.ReadFile(tokenFile) + if err != nil { + if os.IsNotExist(err) { + token, err := clientaccess.FormatToken(serverToken, certs) + if err != nil { + return token, err + } + return token, nil + } + return "", err + } + + // strip the token from any new line if its read from file + return string(bytes.TrimRight(b, "\n")), nil +} + +// NormalizeToken will normalize the token read from file or passed as a cli flag +func NormalizeToken(token string) (string, error) { + _, password, ok := clientaccess.ParseUsernamePassword(token) + if !ok { + return password, errors.New("failed to normalize server token; must be in format K10::: or ") + } + + return password, nil +} + +func GetTokenHash(config *config.Control) (string, error) { + token := config.Token + if token == "" { + tokenFromFile, err := ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) + if err != nil { + return "", err + } + token = tokenFromFile + } + normalizedToken, err := NormalizeToken(token) + if err != nil { + return "", err + } + return ShortHash(normalizedToken, 12), nil +} + +func ShortHash(s string, i int) string { + digest := sha256.Sum256([]byte(s)) + return hex.EncodeToString(digest[:])[:i] +} diff --git a/scripts/airgap/image-list.txt b/scripts/airgap/image-list.txt index f1c3b1a69bda..2c97f06eb065 100644 --- a/scripts/airgap/image-list.txt +++ b/scripts/airgap/image-list.txt @@ -2,7 +2,7 @@ docker.io/rancher/klipper-helm:v0.8.2-build20230815 docker.io/rancher/klipper-lb:v0.4.4 docker.io/rancher/local-path-provisioner:v0.0.24 docker.io/rancher/mirrored-coredns-coredns:1.10.1 -docker.io/rancher/mirrored-library-busybox:1.34.1 -docker.io/rancher/mirrored-library-traefik:2.9.10 +docker.io/rancher/mirrored-library-busybox:1.36.1 +docker.io/rancher/mirrored-library-traefik:2.10.5 docker.io/rancher/mirrored-metrics-server:v0.6.3 docker.io/rancher/mirrored-pause:3.6 diff --git a/scripts/build b/scripts/build index eaed4f93c6ce..c6a4b7c869db 100755 --- a/scripts/build +++ b/scripts/build @@ -133,7 +133,7 @@ if [ ! -x ${INSTALLBIN}/cni ]; then fi echo Building k3s -CGO_ENABLED=1 "${GO}" build $BLDFLAGS -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s ./cmd/server/main.go +CGO_ENABLED=1 "${GO}" build $BLDFLAGS -tags "$TAGS" -buildvcs=false -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s ./cmd/server ln -s k3s ./bin/containerd ln -s k3s ./bin/crictl ln -s k3s ./bin/ctr diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index 88dcfadd8ec6..10a939044d72 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -236,7 +236,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { }) It("Runs an interactive command a pod", func() { - cmd := "kubectl run busybox --rm -it --restart=Never --image=rancher/mirrored-library-busybox:1.34.1 -- uname -a" + cmd := "kubectl run busybox --rm -it --restart=Never --image=rancher/mirrored-library-busybox:1.36.1 -- uname -a" _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) }) diff --git a/tests/integration/etcdrestore/etcd_restore_int_test.go b/tests/integration/etcdrestore/etcd_restore_int_test.go index 22bb0f2b6ee5..5ea168d53237 100644 --- a/tests/integration/etcdrestore/etcd_restore_int_test.go +++ b/tests/integration/etcdrestore/etcd_restore_int_test.go @@ -41,6 +41,11 @@ var _ = Describe("etcd snapshot restore", Ordered, func() { Expect(result).To(ContainSubstring("deployment.apps/nginx-deployment created")) Expect(err).NotTo(HaveOccurred()) }) + It("make sure workload exists", func() { + res, err := testutil.K3sCmd("kubectl", "rollout", "status", "deployment", "nginx-deployment", "--watch=true", "--timeout=360s") + Expect(res).To(ContainSubstring("successfully rolled out")) + Expect(err).ToNot(HaveOccurred()) + }) It("saves an etcd snapshot", func() { Expect(testutil.K3sCmd("etcd-snapshot", "save", "-d", tmpdDataDir, "--name", "snapshot-to-restore")). To(ContainSubstring("saved")) @@ -83,15 +88,15 @@ var _ = Describe("etcd snapshot restore", Ordered, func() { return testutil.K3sDefaultDeployments() }, "360s", "5s").Should(Succeed()) }) - It("Make sure Workload 1 exists", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get", "deployment", "nginx-deployment") - }, "360s", "5s").Should(ContainSubstring("3/3")) + It("make sure workload 1 exists", func() { + res, err := testutil.K3sCmd("kubectl", "rollout", "status", "deployment", "nginx-deployment", "--watch=true", "--timeout=360s") + Expect(res).To(ContainSubstring("successfully rolled out")) + Expect(err).ToNot(HaveOccurred()) }) - It("Make sure Workload 2 does not exists", func() { + It("make sure workload 2 does not exists", func() { res, err := testutil.K3sCmd("kubectl", "get", "deployment", "nginx-deployment-post-snapshot") - Expect(err).To(HaveOccurred()) Expect(res).To(ContainSubstring("not found")) + Expect(err).To(HaveOccurred()) }) It("check if CA cert hash matches", func() { // get md5sum of the CA certs diff --git a/tests/integration/etcdrestore/testdata/temp_depl.yaml b/tests/integration/etcdrestore/testdata/temp_depl.yaml index 3649247c1bb1..8e8c564fec83 100644 --- a/tests/integration/etcdrestore/testdata/temp_depl.yaml +++ b/tests/integration/etcdrestore/testdata/temp_depl.yaml @@ -6,6 +6,9 @@ metadata: app: nginx spec: replicas: 3 + revisionHistoryLimit: 0 + strategy: + type: Recreate selector: matchLabels: app: nginx @@ -18,4 +21,4 @@ spec: - name: nginx image: nginx:1.14.2 ports: - - containerPort: 80 \ No newline at end of file + - containerPort: 80 diff --git a/tests/integration/etcdrestore/testdata/temp_depl2.yaml b/tests/integration/etcdrestore/testdata/temp_depl2.yaml index 8cea5e6f2d95..c5247a77e75d 100644 --- a/tests/integration/etcdrestore/testdata/temp_depl2.yaml +++ b/tests/integration/etcdrestore/testdata/temp_depl2.yaml @@ -6,6 +6,9 @@ metadata: app: nginx spec: replicas: 3 + revisionHistoryLimit: 0 + strategy: + type: Recreate selector: matchLabels: app: nginx @@ -18,4 +21,4 @@ spec: - name: nginx image: nginx:1.14.2 ports: - - containerPort: 80 \ No newline at end of file + - containerPort: 80 diff --git a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go index 3fe9f4152b84..1d7c9b5ea21b 100644 --- a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go +++ b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go @@ -54,11 +54,11 @@ var _ = Describe("etcd snapshots", Ordered, func() { It("deletes a snapshot", func() { lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls") Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`on-demand[^\s]+`) + reg, err := regexp.Compile(`(?m)^on-demand[^\s]+`) Expect(err).ToNot(HaveOccurred()) snapshotName := reg.FindString(lsResult) Expect(testutil.K3sCmd("etcd-snapshot", "delete", snapshotName)). - To(ContainSubstring("Removing the given locally stored etcd snapshot")) + To(ContainSubstring("Snapshot " + snapshotName + " deleted locally")) }) }) When("saving a custom name", func() { @@ -69,11 +69,11 @@ var _ = Describe("etcd snapshots", Ordered, func() { It("deletes that snapshot", func() { lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls") Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`ALIVEBEEF[^\s]+`) + reg, err := regexp.Compile(`(?m)^ALIVEBEEF[^\s]+`) Expect(err).ToNot(HaveOccurred()) snapshotName := reg.FindString(lsResult) Expect(testutil.K3sCmd("etcd-snapshot", "delete", snapshotName)). - To(ContainSubstring("Removing the given locally stored etcd snapshot")) + To(ContainSubstring("Snapshot " + snapshotName + " deleted locally")) }) }) When("using etcd snapshot prune", func() { @@ -91,7 +91,7 @@ var _ = Describe("etcd snapshots", Ordered, func() { It("lists all 3 snapshots", func() { lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls") Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`) + reg, err := regexp.Compile(`(?m):///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`) Expect(err).ToNot(HaveOccurred()) sepLines := reg.FindAllString(lsResult, -1) Expect(sepLines).To(HaveLen(3)) @@ -101,7 +101,7 @@ var _ = Describe("etcd snapshots", Ordered, func() { To(ContainSubstring("Removing local snapshot")) lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls") Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`) + reg, err := regexp.Compile(`(?m):///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`) Expect(err).ToNot(HaveOccurred()) sepLines := reg.FindAllString(lsResult, -1) Expect(sepLines).To(HaveLen(2)) @@ -109,11 +109,11 @@ var _ = Describe("etcd snapshots", Ordered, func() { It("cleans up remaining snapshots", func() { lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls") Expect(err).ToNot(HaveOccurred()) - reg, err := regexp.Compile(`PRUNE_TEST[^\s]+`) + reg, err := regexp.Compile(`(?m)^PRUNE_TEST[^\s]+`) Expect(err).ToNot(HaveOccurred()) for _, snapshotName := range reg.FindAllString(lsResult, -1) { Expect(testutil.K3sCmd("etcd-snapshot", "delete", snapshotName)). - To(ContainSubstring("Removing the given locally stored etcd snapshot")) + To(ContainSubstring("Snapshot " + snapshotName + " deleted locally")) } }) }) diff --git a/tests/integration/integration.go b/tests/integration/integration.go index 5f7f9ee74a80..2aea49de0ab1 100644 --- a/tests/integration/integration.go +++ b/tests/integration/integration.go @@ -280,6 +280,9 @@ func K3sStopServer(server *K3sServer) error { // K3sKillServer terminates the running K3s server and its children. // Equivalent to k3s-killall.sh func K3sKillServer(server *K3sServer) error { + if server == nil { + return nil + } if server.log != nil { server.log.Close() os.Remove(server.log.Name())