From 21f04cee3439ea366157baf68ae0b939c4d39078 Mon Sep 17 00:00:00 2001 From: Joe Haddad Date: Wed, 10 Oct 2018 14:14:37 -0400 Subject: [PATCH 01/24] feat: annotate stack with label and checksum --- Gopkg.lock | 1 + Gopkg.toml | 4 ++ cluster/cluster.go | 3 +- cluster/kubernetes/kubernetes.go | 64 +++++++++++++++++++++++++++++--- cluster/kubernetes/sync.go | 8 ++-- cluster/kubernetes/sync_test.go | 4 +- cluster/mock.go | 6 +-- daemon/daemon_test.go | 4 +- daemon/loop.go | 3 +- daemon/loop_test.go | 7 ++-- policy/policy.go | 13 ++++--- sync/sync.go | 44 +++++++++++++++++++++- sync/sync_test.go | 7 ++-- 13 files changed, 135 insertions(+), 33 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index f9696fc42..041107b45 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1185,6 +1185,7 @@ "github.com/google/go-cmp/cmp", "github.com/gorilla/mux", "github.com/gorilla/websocket", + "github.com/imdario/mergo", "github.com/justinbarrick/go-k8s-portforward", "github.com/ncabatoff/go-seq/seq", "github.com/opencontainers/go-digest", diff --git a/Gopkg.toml b/Gopkg.toml index 884f7acb5..2479e105b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -57,3 +57,7 @@ required = ["k8s.io/code-generator/cmd/client-gen"] [[override]] name = "github.com/BurntSushi/toml" version = "v0.3.1" + +[[constraint]] + name = "github.com/imdario/mergo" + version = "0.3.2" diff --git a/cluster/cluster.go b/cluster/cluster.go index d153ff328..01f07731e 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/weaveworks/flux" + "github.com/weaveworks/flux/policy" "github.com/weaveworks/flux/resource" "github.com/weaveworks/flux/ssh" "github.com/weaveworks/flux/policy" @@ -29,7 +30,7 @@ type Cluster interface { SomeControllers([]flux.ResourceID) ([]Controller, error) Ping() error Export() ([]byte, error) - Sync(SyncDef) error + Sync(SyncDef, map[string]policy.Update, map[string]policy.Update) error PublicSSHKey(regenerate bool) (ssh.PublicKey, error) } diff --git a/cluster/kubernetes/kubernetes.go b/cluster/kubernetes/kubernetes.go index 7f4459f12..96203af26 100644 --- a/cluster/kubernetes/kubernetes.go +++ b/cluster/kubernetes/kubernetes.go @@ -7,8 +7,11 @@ import ( k8syaml "github.com/ghodss/yaml" "github.com/go-kit/kit/log" + "github.com/imdario/mergo" "github.com/pkg/errors" + kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" fhrclient "github.com/weaveworks/flux/integrations/client/clientset/versioned" + "github.com/weaveworks/flux/policy" "gopkg.in/yaml.v2" apiv1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -37,9 +40,10 @@ type metadata struct { } type apiObject struct { - resource.Resource - Kind string `yaml:"kind"` - Metadata metadata `yaml:"metadata"` + OriginalResource resource.Resource + Payload []byte + Kind string `yaml:"kind"` + Metadata metadata `yaml:"metadata"` } // A convenience for getting an minimal object from some bytes. @@ -212,9 +216,51 @@ func (c *Cluster) AllControllers(namespace string) (res []cluster.Controller, er return allControllers, nil } +func applyMetadata(res resource.Resource, resourceLabels map[string]policy.Update, resourcePolicyUpdates map[string]policy.Update) ([]byte, error) { + id := res.ResourceID().String() + + definition := make(map[interface{}]interface{}) + if err := yaml.Unmarshal(res.Bytes(), &definition); err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("failed to parse yaml from %s", res.Source())) + } + + if update, ok := resourceLabels[id]; ok { + mixin := make(map[interface{}]interface{}) + var mixinBuffer bytes.Buffer + mixinBuffer.WriteString("metadata:\n labels:\n") + for key, value := range update.Add.ToStringMap() { + mixinBuffer.WriteString(fmt.Sprintf(" %s: %s\n", fmt.Sprintf("%s%s", kresource.PolicyPrefix, key), value)) + } + if err := yaml.Unmarshal(mixinBuffer.Bytes(), &mixin); err != nil { + return nil, errors.Wrap(err, "failed to parse yaml for mixin") + } + mergo.Merge(&definition, mixin) + } + + if update, ok := resourcePolicyUpdates[id]; ok { + mixin := make(map[interface{}]interface{}) + var mixinBuffer bytes.Buffer + mixinBuffer.WriteString("metadata:\n annotations:\n") + for key, value := range update.Add.ToStringMap() { + mixinBuffer.WriteString(fmt.Sprintf(" %s: %s\n", fmt.Sprintf("%s%s", kresource.PolicyPrefix, key), value)) + } + if err := yaml.Unmarshal(mixinBuffer.Bytes(), &mixin); err != nil { + return nil, errors.Wrap(err, "failed to parse yaml for mixin") + } + mergo.Merge(&definition, mixin) + } + + bytes, err := yaml.Marshal(definition) + if err != nil { + return nil, errors.Wrap(err, "failed to serialize yaml after applying metadata") + } + + return bytes, nil +} + // Sync performs the given actions on resources. Operations are // asynchronous, but serialised. -func (c *Cluster) Sync(spec cluster.SyncDef) error { +func (c *Cluster) Sync(spec cluster.SyncDef, resourceLabels map[string]policy.Update, resourcePolicyUpdates map[string]policy.Update) error { logger := log.With(c.logger, "method", "Sync") cs := makeChangeSet() @@ -231,9 +277,15 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { if stage.res == nil { continue } - obj, err := parseObj(stage.res.Bytes()) + + var obj *apiObject + resBytes, err := applyMetadata(stage.res, resourceLabels, resourcePolicyUpdates) + if err == nil { + obj, err = parseObj(resBytes) + } if err == nil { - obj.Resource = stage.res + obj.OriginalResource = stage.res + obj.Payload = resBytes cs.stage(stage.cmd, obj) } else { errs = append(errs, cluster.ResourceError{Resource: stage.res, Error: err}) diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index a2bd1b0f1..53f2a5373 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -127,7 +127,7 @@ func (c *Kubectl) apply(logger log.Logger, cs changeSet, errored map[flux.Resour multi = objs } else { for _, obj := range objs { - if _, ok := errored[obj.ResourceID()]; ok { + if _, ok := errored[obj.OriginalResource.ResourceID()]; ok { // Resources that errored before shall be applied separately single = append(single, obj) } else { @@ -143,9 +143,9 @@ func (c *Kubectl) apply(logger log.Logger, cs changeSet, errored map[flux.Resour } } for _, obj := range single { - r := bytes.NewReader(obj.Bytes()) + r := bytes.NewReader(obj.Payload) if err := c.doCommand(logger, r, args...); err != nil { - errs = append(errs, cluster.ResourceError{obj.Resource, err}) + errs = append(errs, cluster.ResourceError{obj.OriginalResource, err}) } } } @@ -189,7 +189,7 @@ func makeMultidoc(objs []*apiObject) *bytes.Buffer { buf := &bytes.Buffer{} for _, obj := range objs { buf.WriteString("\n---\n") - buf.Write(obj.Bytes()) + buf.Write(obj.Payload) } return buf } diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index b3c09cc44..4847ca350 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -56,7 +56,7 @@ func setup(t *testing.T) (*Cluster, *mockApplier) { func TestSyncNop(t *testing.T) { kube, mock := setup(t) - if err := kube.Sync(cluster.SyncDef{}); err != nil { + if err := kube.Sync(cluster.SyncDef{}, map[string]policy.Update{}, map[string]policy.Update{}); err != nil { t.Errorf("%#v", err) } if mock.commandRun { @@ -72,7 +72,7 @@ func TestSyncMalformed(t *testing.T) { Apply: rsc{"default:deployment/trash", []byte("garbage")}, }, }, - }) + }, map[string]policy.Update{}, map[string]policy.Update{}) if err == nil { t.Error("expected error because malformed resource def, but got nil") } diff --git a/cluster/mock.go b/cluster/mock.go index 4943bce44..9d44467e1 100644 --- a/cluster/mock.go +++ b/cluster/mock.go @@ -14,7 +14,7 @@ type Mock struct { SomeServicesFunc func([]flux.ResourceID) ([]Controller, error) PingFunc func() error ExportFunc func() ([]byte, error) - SyncFunc func(SyncDef) error + SyncFunc func(SyncDef, map[string]policy.Update, map[string]policy.Update) error PublicSSHKeyFunc func(regenerate bool) (ssh.PublicKey, error) UpdateImageFunc func(def []byte, id flux.ResourceID, container string, newImageID image.Ref) ([]byte, error) LoadManifestsFunc func(base string, paths []string) (map[string]resource.Resource, error) @@ -39,8 +39,8 @@ func (m *Mock) Export() ([]byte, error) { return m.ExportFunc() } -func (m *Mock) Sync(c SyncDef) error { - return m.SyncFunc(c) +func (m *Mock) Sync(c SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { + return m.SyncFunc(c, l, p) } func (m *Mock) PublicSSHKey(regenerate bool) (ssh.PublicKey, error) { diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go index 6cdaca4b6..62dd9f22d 100644 --- a/daemon/daemon_test.go +++ b/daemon/daemon_test.go @@ -359,7 +359,7 @@ func TestDaemon_NotifyChange(t *testing.T) { var syncCalled int var syncDef *cluster.SyncDef var syncMu sync.Mutex - mockK8s.SyncFunc = func(def cluster.SyncDef) error { + mockK8s.SyncFunc = func(def cluster.SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { syncMu.Lock() syncCalled++ syncDef = &def @@ -660,7 +660,7 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *cluster.Mock, *mockEven singleService, }, nil } - k8s.SyncFunc = func(def cluster.SyncDef) error { return nil } + k8s.SyncFunc = func(def cluster.SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { return nil } k8s.UpdatePoliciesFunc = (&kubernetes.Manifests{}).UpdatePolicies k8s.UpdateImageFunc = (&kubernetes.Manifests{}).UpdateImage } diff --git a/daemon/loop.go b/daemon/loop.go index dd0bc8823..1c15c523c 100644 --- a/daemon/loop.go +++ b/daemon/loop.go @@ -206,7 +206,8 @@ func (d *Daemon) doSync(logger log.Logger, lastKnownSyncTagRev *string, warnedAb var resourceErrors []event.ResourceError // TODO supply deletes argument from somewhere (command-line?) - if err := fluxsync.Sync(logger, d.Manifests, allResources, d.Cluster, false); err != nil { + // TODO: supply tracking argument from somewhere + if err := fluxsync.Sync(logger, d.Manifests, allResources, d.Cluster, true, false); err != nil { logger.Log("err", err) switch syncerr := err.(type) { case cluster.SyncError: diff --git a/daemon/loop_test.go b/daemon/loop_test.go index 547928a43..697ea2f76 100644 --- a/daemon/loop_test.go +++ b/daemon/loop_test.go @@ -21,6 +21,7 @@ import ( "github.com/weaveworks/flux/git" "github.com/weaveworks/flux/git/gittest" "github.com/weaveworks/flux/job" + "github.com/weaveworks/flux/policy" registryMock "github.com/weaveworks/flux/registry/mock" "github.com/weaveworks/flux/resource" ) @@ -100,7 +101,7 @@ func TestPullAndSync_InitialSync(t *testing.T) { expectedResourceIDs = append(expectedResourceIDs, id) } expectedResourceIDs.Sort() - k8s.SyncFunc = func(def cluster.SyncDef) error { + k8s.SyncFunc = func(def cluster.SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { syncCalled++ syncDef = &def return nil @@ -173,7 +174,7 @@ func TestDoSync_NoNewCommits(t *testing.T) { expectedResourceIDs = append(expectedResourceIDs, id) } expectedResourceIDs.Sort() - k8s.SyncFunc = func(def cluster.SyncDef) error { + k8s.SyncFunc = func(def cluster.SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { syncCalled++ syncDef = &def return nil @@ -271,7 +272,7 @@ func TestDoSync_WithNewCommit(t *testing.T) { expectedResourceIDs = append(expectedResourceIDs, id) } expectedResourceIDs.Sort() - k8s.SyncFunc = func(def cluster.SyncDef) error { + k8s.SyncFunc = func(def cluster.SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { syncCalled++ syncDef = &def return nil diff --git a/policy/policy.go b/policy/policy.go index ef846858a..bfdbb1df6 100644 --- a/policy/policy.go +++ b/policy/policy.go @@ -8,12 +8,13 @@ import ( ) const ( - Ignore = Policy("ignore") - Locked = Policy("locked") - LockedUser = Policy("locked_user") - LockedMsg = Policy("locked_msg") - Automated = Policy("automated") - TagAll = Policy("tag_all") + Ignore = Policy("ignore") + Locked = Policy("locked") + LockedUser = Policy("locked_user") + LockedMsg = Policy("locked_msg") + Automated = Policy("automated") + TagAll = Policy("tag_all") + StackChecksum = Policy("stack_checksum") ) // Policy is an string, denoting the current deployment policy of a service, diff --git a/sync/sync.go b/sync/sync.go index e0df168a3..18c292af0 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -1,6 +1,10 @@ package sync import ( + "crypto/sha1" + "encoding/hex" + "sort" + "github.com/go-kit/kit/log" "github.com/pkg/errors" @@ -9,9 +13,24 @@ import ( "github.com/weaveworks/flux/resource" ) +// Checksum generates a unique identifier for all apply actions in the stack +func getStackChecksum(repoResources map[string]resource.Resource) string { + checksum := sha1.New() + + sortedKeys := make([]string, 0, len(repoResources)) + for resourceID := range repoResources { + sortedKeys = append(sortedKeys, resourceID) + } + sort.Strings(sortedKeys) + for resourceIDIndex := range sortedKeys { + checksum.Write(repoResources[sortedKeys[resourceIDIndex]].Bytes()) + } + return hex.EncodeToString(checksum.Sum(nil)) +} + // Sync synchronises the cluster to the files in a directory func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resource.Resource, clus cluster.Cluster, - deletes bool) error { + deletes, tracks bool) error { // Get a map of resources defined in the cluster clusterBytes, err := clus.Export() @@ -30,6 +49,27 @@ func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resou // no-op. sync := cluster.SyncDef{} + resourceLabels := map[string]policy.Update{} + resourcePolicyUpdates := map[string]policy.Update{} + if tracks { + stackName := "default" // TODO: multiple stack support + stackChecksum := getStackChecksum(repoResources) + + logger.Log("stack", stackName, "checksum", stackChecksum) + + for id := range repoResources { + logger.Log("resource", id, "applying checksum", stackChecksum) + resourceLabels[id] = policy.Update{ + Add: policy.Set{"stack": stackName}, + } + resourcePolicyUpdates[id] = policy.Update{ + Add: policy.Set{policy.StackChecksum: stackChecksum}, + } + } + + // label flux.weave.works/stack + } + // DANGER ZONE (tamara) This works and is dangerous. At the moment will delete Flux and // other pods unless the relevant manifests are part of the user repo. Needs a lot of thought // before this cleanup cluster feature can be unleashed on the world. @@ -43,7 +83,7 @@ func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resou prepareSyncApply(logger, clusterResources, id, res, &sync) } - return clus.Sync(sync) + return clus.Sync(sync, resourceLabels, resourcePolicyUpdates) } func prepareSyncDelete(logger log.Logger, repoResources map[string]resource.Resource, id string, res resource.Resource, sync *cluster.SyncDef) { diff --git a/sync/sync_test.go b/sync/sync_test.go index e9d62dcf9..cf3d0846e 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -19,6 +19,7 @@ import ( "github.com/weaveworks/flux/cluster/kubernetes/testfiles" "github.com/weaveworks/flux/git" "github.com/weaveworks/flux/git/gittest" + "github.com/weaveworks/flux/policy" "github.com/weaveworks/flux/resource" ) @@ -40,7 +41,7 @@ func TestSync(t *testing.T) { t.Fatal(err) } - if err := Sync(log.NewNopLogger(), manifests, resources, clus, true); err != nil { + if err := Sync(log.NewNopLogger(), manifests, resources, clus, false, true); err != nil { t.Fatal(err) } checkClusterMatchesFiles(t, manifests, clus, checkout.Dir(), dirs) @@ -60,7 +61,7 @@ func TestSync(t *testing.T) { if err != nil { t.Fatal(err) } - if err := Sync(log.NewNopLogger(), manifests, resources, clus, true); err != nil { + if err := Sync(log.NewNopLogger(), manifests, resources, clus, false, true); err != nil { t.Fatal(err) } checkClusterMatchesFiles(t, manifests, clus, checkout.Dir(), dirs) @@ -200,7 +201,7 @@ type syncCluster struct { resources map[string][]byte } -func (p *syncCluster) Sync(def cluster.SyncDef) error { +func (p *syncCluster) Sync(def cluster.SyncDef, l map[string]policy.Update, pl map[string]policy.Update) error { println("=== Syncing ===") for _, action := range def.Actions { if action.Delete != nil { From 41a6d3819d56408b127346d22559a42bb09b34cf Mon Sep 17 00:00:00 2001 From: Joe Haddad Date: Thu, 11 Oct 2018 16:02:02 -0400 Subject: [PATCH 02/24] Scan cluster stack and compare hash during sync --- cluster/cluster.go | 1 + cluster/kubernetes/images_test.go | 4 +- cluster/kubernetes/kubernetes.go | 79 +++++++++++++++++++++++++++++++ cmd/fluxd/main.go | 8 +++- sync/sync.go | 43 ++++++++++++++--- 5 files changed, 125 insertions(+), 10 deletions(-) diff --git a/cluster/cluster.go b/cluster/cluster.go index 01f07731e..a91e2d294 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -30,6 +30,7 @@ type Cluster interface { SomeControllers([]flux.ResourceID) ([]Controller, error) Ping() error Export() ([]byte, error) + ExportByLabel(string, string) ([]byte, error) Sync(SyncDef, map[string]policy.Update, map[string]policy.Update) error PublicSSHKey(regenerate bool) (ssh.PublicKey, error) } diff --git a/cluster/kubernetes/images_test.go b/cluster/kubernetes/images_test.go index ba63812d8..84d8d8701 100644 --- a/cluster/kubernetes/images_test.go +++ b/cluster/kubernetes/images_test.go @@ -64,7 +64,7 @@ func TestMergeCredentials(t *testing.T) { makeServiceAccount(ns, saName, []string{secretName2}), makeImagePullSecret(ns, secretName1, "docker.io"), makeImagePullSecret(ns, secretName2, "quay.io")) - client := extendedClient{clientset, nil} + client := extendedClient{coreClient: clientset} creds := registry.ImageCreds{} @@ -97,7 +97,7 @@ func TestMergeCredentials_ImageExclusion(t *testing.T) { } clientset := fake.NewSimpleClientset() - client := extendedClient{clientset, nil} + client := extendedClient{coreClient: clientset} var includeImage = func(imageName string) bool { for _, exp := range []string{"k8s.gcr.io/*", "*test*"} { diff --git a/cluster/kubernetes/kubernetes.go b/cluster/kubernetes/kubernetes.go index 96203af26..73daf2aeb 100644 --- a/cluster/kubernetes/kubernetes.go +++ b/cluster/kubernetes/kubernetes.go @@ -3,6 +3,7 @@ package kubernetes import ( "bytes" "fmt" + "strings" "sync" k8syaml "github.com/ghodss/yaml" @@ -16,6 +17,8 @@ import ( apiv1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + k8sclientdynamic "k8s.io/client-go/dynamic" k8sclient "k8s.io/client-go/kubernetes" "github.com/weaveworks/flux" @@ -25,10 +28,12 @@ import ( ) type coreClient k8sclient.Interface +type dynamicClient k8sclientdynamic.Interface type fluxHelmClient fhrclient.Interface type extendedClient struct { coreClient + dynamicClient fluxHelmClient } @@ -114,6 +119,7 @@ type Cluster struct { // NewCluster returns a usable cluster. func NewCluster(clientset k8sclient.Interface, + dynamicClientset k8sclientdynamic.Interface, fluxHelmClientset fhrclient.Interface, applier Applier, sshKeyRing ssh.KeyRing, @@ -124,6 +130,7 @@ func NewCluster(clientset k8sclient.Interface, c := &Cluster{ client: extendedClient{ clientset, + dynamicClientset, fluxHelmClientset, }, applier: applier, @@ -374,6 +381,78 @@ func (c *Cluster) Export() ([]byte, error) { return config.Bytes(), nil } +func contains(a []string, x string) bool { + for _, n := range a { + if x == n { + return true + } + } + return false +} + +func (c *Cluster) ExportByLabel(labelName string, labelValue string) ([]byte, error) { + var config bytes.Buffer + + resources, err := c.client.coreClient.Discovery().ServerResources() + if err != nil { + return nil, err + } + for _, resource := range resources { + for _, apiResource := range resource.APIResources { + verbs := apiResource.Verbs + // skip resources that can't be listed + if !contains(verbs, "list") { + continue + } + + // get group and version + var group, version string + groupVersion := resource.GroupVersion + if strings.Contains(groupVersion, "/") { + a := strings.SplitN(groupVersion, "/", 2) + group = a[0] + version = a[1] + } else { + group = "" + version = groupVersion + } + + resourceClient := c.client.dynamicClient.Resource(schema.GroupVersionResource{ + Group: group, + Version: version, + Resource: apiResource.Name, + }) + data, err := resourceClient.List(meta_v1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelName, labelValue), + }) + if err != nil { + return nil, err + } + + for _, item := range data.Items { + apiVersion := item.GetAPIVersion() + kind := item.GetKind() + + itemDesc := fmt.Sprintf("%s:%s", apiVersion, kind) + // https://github.com/kontena/k8s-client/blob/6e9a7ba1f03c255bd6f06e8724a1c7286b22e60f/lib/k8s/stack.rb#L17-L22 + if itemDesc == "v1:ComponentStatus" || itemDesc == "v1:Endpoints" { + continue + } + + yamlBytes, err := k8syaml.Marshal(item.Object) + if err != nil { + return nil, err + } + config.WriteString("---\n") + config.Write(yamlBytes) + config.WriteString("\n") + } + } + } + + return config.Bytes(), nil +} + // kind & apiVersion must be passed separately as the object's TypeMeta is not populated func appendYAML(buffer *bytes.Buffer, apiVersion, kind string, object interface{}) error { yamlBytes, err := k8syaml.Marshal(object) diff --git a/cmd/fluxd/main.go b/cmd/fluxd/main.go index 3a3e75b78..e8f977d57 100644 --- a/cmd/fluxd/main.go +++ b/cmd/fluxd/main.go @@ -18,6 +18,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/pflag" k8sifclient "github.com/weaveworks/flux/integrations/client/clientset/versioned" + k8sclientdynamic "k8s.io/client-go/dynamic" k8sclient "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -207,6 +208,11 @@ func main() { logger.Log("err", err) os.Exit(1) } + dynamicClientset, err := k8sclientdynamic.NewForConfig(restClientConfig) + if err != nil { + logger.Log("err", err) + os.Exit(1) + } ifclientset, err := k8sifclient.NewForConfig(restClientConfig) if err != nil { @@ -261,7 +267,7 @@ func main() { logger.Log("kubectl", kubectl) kubectlApplier := kubernetes.NewKubectl(kubectl, restClientConfig) - k8sInst := kubernetes.NewCluster(clientset, ifclientset, kubectlApplier, sshKeyRing, logger, *k8sNamespaceWhitelist, *registryExcludeImage) + k8sInst := kubernetes.NewCluster(clientset, dynamicClientset, ifclientset, kubectlApplier, sshKeyRing, logger, *k8sNamespaceWhitelist, *registryExcludeImage) if err := k8sInst.Ping(); err != nil { logger.Log("ping", err) diff --git a/sync/sync.go b/sync/sync.go index 18c292af0..36a2a6e54 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -3,12 +3,14 @@ package sync import ( "crypto/sha1" "encoding/hex" + "fmt" "sort" "github.com/go-kit/kit/log" "github.com/pkg/errors" "github.com/weaveworks/flux/cluster" + kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" "github.com/weaveworks/flux/policy" "github.com/weaveworks/flux/resource" ) @@ -49,16 +51,17 @@ func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resou // no-op. sync := cluster.SyncDef{} + var stackName, stackChecksum string resourceLabels := map[string]policy.Update{} resourcePolicyUpdates := map[string]policy.Update{} if tracks { - stackName := "default" // TODO: multiple stack support - stackChecksum := getStackChecksum(repoResources) + stackName = "default" // TODO: multiple stack support + stackChecksum = getStackChecksum(repoResources) - logger.Log("stack", stackName, "checksum", stackChecksum) + fmt.Printf("[stack-tracking] stack=%s, checksum=%s\n", stackName, stackChecksum) for id := range repoResources { - logger.Log("resource", id, "applying checksum", stackChecksum) + fmt.Printf("[stack-tracking] resource=%s, applying checksum=%s\n", id, stackChecksum) resourceLabels[id] = policy.Update{ Add: policy.Set{"stack": stackName}, } @@ -66,8 +69,6 @@ func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resou Add: policy.Set{policy.StackChecksum: stackChecksum}, } } - - // label flux.weave.works/stack } // DANGER ZONE (tamara) This works and is dangerous. At the moment will delete Flux and @@ -83,7 +84,35 @@ func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resou prepareSyncApply(logger, clusterResources, id, res, &sync) } - return clus.Sync(sync, resourceLabels, resourcePolicyUpdates) + if err := clus.Sync(sync, resourceLabels, resourcePolicyUpdates); err != nil { + return err + } + if tracks { + fmt.Printf("[stack-tracking] scanning stack (%s) for orphaned resources\n", stackName) + clusterResourceBytes, err := clus.ExportByLabel(fmt.Sprintf("%s%s", kresource.PolicyPrefix, "stack"), stackName) + if err != nil { + return errors.Wrap(err, "exporting resource defs from cluster post-sync") + } + clusterResources, err = m.ParseManifests(clusterResourceBytes) + if err != nil { + return errors.Wrap(err, "parsing exported resources post-sync") + } + + for resourceID, res := range clusterResources { + if res.Policy().Has(policy.StackChecksum) { + val, _ := res.Policy().Get(policy.StackChecksum) + if val != stackChecksum { + fmt.Printf("[stack-tracking] cluster resource=%s, invalid checksum=%s\n", resourceID, val) + } else { + fmt.Printf("[stack-tracking] cluster resource ok: %s\n", resourceID) + } + } else { + fmt.Printf("warning: [stack-tracking] cluster resource=%s, missing policy=%s\n", resourceID, policy.StackChecksum) + } + } + } + + return nil } func prepareSyncDelete(logger log.Logger, repoResources map[string]resource.Resource, id string, res resource.Resource, sync *cluster.SyncDef) { From 084091592b1364184414187267ac09701117d4b8 Mon Sep 17 00:00:00 2001 From: Joe Haddad Date: Thu, 11 Oct 2018 21:28:50 -0400 Subject: [PATCH 03/24] feat: delete orphaned resources --- sync/sync.go | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/sync/sync.go b/sync/sync.go index 36a2a6e54..0aec0d2db 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -30,6 +30,22 @@ func getStackChecksum(repoResources map[string]resource.Resource) string { return hex.EncodeToString(checksum.Sum(nil)) } +func garbageCollect(orphans []string, clusterResources map[string]resource.Resource, clus cluster.Cluster, logger log.Logger) error { + garbageCollect := cluster.SyncDef{} + emptyResources := map[string]resource.Resource{"noop/noop:noop": nil} + for _, id := range orphans { + res, ok := clusterResources[id] + if !ok { + return errors.Errorf("invariant: unable to find resource %s\n", id) + } + if prepareSyncDelete(logger, emptyResources, id, res, &garbageCollect) { + // TODO: use logger + fmt.Printf("[stack-tracking] marking resource %s for deletion\n", id) + } + } + return clus.Sync(garbageCollect, map[string]policy.Update{}, map[string]policy.Update{}) +} + // Sync synchronises the cluster to the files in a directory func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resource.Resource, clus cluster.Cluster, deletes, tracks bool) error { @@ -88,6 +104,8 @@ func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resou return err } if tracks { + orphanedResources := make([]string, 0) + fmt.Printf("[stack-tracking] scanning stack (%s) for orphaned resources\n", stackName) clusterResourceBytes, err := clus.ExportByLabel(fmt.Sprintf("%s%s", kresource.PolicyPrefix, "stack"), stackName) if err != nil { @@ -103,6 +121,7 @@ func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resou val, _ := res.Policy().Get(policy.StackChecksum) if val != stackChecksum { fmt.Printf("[stack-tracking] cluster resource=%s, invalid checksum=%s\n", resourceID, val) + orphanedResources = append(orphanedResources, resourceID) } else { fmt.Printf("[stack-tracking] cluster resource ok: %s\n", resourceID) } @@ -110,24 +129,30 @@ func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resou fmt.Printf("warning: [stack-tracking] cluster resource=%s, missing policy=%s\n", resourceID, policy.StackChecksum) } } + + if len(orphanedResources) > 0 { + return garbageCollect(orphanedResources, clusterResources, clus, logger) + } } return nil } -func prepareSyncDelete(logger log.Logger, repoResources map[string]resource.Resource, id string, res resource.Resource, sync *cluster.SyncDef) { +func prepareSyncDelete(logger log.Logger, repoResources map[string]resource.Resource, id string, res resource.Resource, sync *cluster.SyncDef) bool { if len(repoResources) == 0 { - return + return false } if res.Policy().Has(policy.Ignore) { logger.Log("resource", res.ResourceID(), "ignore", "delete") - return + return false } if _, ok := repoResources[id]; !ok { sync.Actions = append(sync.Actions, cluster.SyncAction{ Delete: res, }) + return true } + return false } func prepareSyncApply(logger log.Logger, clusterResources map[string]resource.Resource, id string, res resource.Resource, sync *cluster.SyncDef) { From f76156e41265bb445f1a46646b2eadc65de4c377 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 18 Oct 2018 16:12:39 +0100 Subject: [PATCH 04/24] Move garbage collection into cluster implementation It's a nice separation to have the algorithm for syncing independent of the cluster implementation; however, when tracking things to delete it introduces some awkwardness. Firstly, it means the Cluster interface has to have a method for fetching all the candidates for garbage collection. This was `ExportByLabel`. Secondly, the method `cluster.Sync` was designed around deciding what to apply and what to delete, then supplying all the operations in one call; whereas the garbage-collecting algorithm must first apply changes, then calculate what's left over and delete it. This isn't wrong as such, but it does expose a mismatch between Sync interface and its use. While keeping the algorithm as similar as possible, this commit moves the garbage collection phase into the cluster/kubernetes implementation of `Sync(...)`. This means `Cluster.Sync` and its argument `SyncDef` can be simplified since much of the bookkeeping will be done inside the implemention. This commit also wires a flag through from the command line to the cluster implementation, so one can switch garbage collection on. The wiring bypasses the sync package, for simplicity -- it will always calculate checksums. Some included tidies: - move the ambiguously situated and named type apiObject to where it's used, renames it, and narrows its scope to just the file it's in. - use map-valued mixins rather than assembling text - turn the temporary Printfs into logger.Log invocations, and remove messages that are just reporting business as usual. --- cluster/cluster.go | 4 +- cluster/kubernetes/kubernetes.go | 148 +++++++++++++------------- cluster/kubernetes/sync.go | 44 +++++--- cluster/sync.go | 20 ++-- cmd/fluxd/main.go | 3 + daemon/loop.go | 4 +- policy/policy.go | 1 + sync/sync.go | 172 +++++++------------------------ 8 files changed, 156 insertions(+), 240 deletions(-) diff --git a/cluster/cluster.go b/cluster/cluster.go index a91e2d294..d153ff328 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -4,7 +4,6 @@ import ( "errors" "github.com/weaveworks/flux" - "github.com/weaveworks/flux/policy" "github.com/weaveworks/flux/resource" "github.com/weaveworks/flux/ssh" "github.com/weaveworks/flux/policy" @@ -30,8 +29,7 @@ type Cluster interface { SomeControllers([]flux.ResourceID) ([]Controller, error) Ping() error Export() ([]byte, error) - ExportByLabel(string, string) ([]byte, error) - Sync(SyncDef, map[string]policy.Update, map[string]policy.Update) error + Sync(SyncDef) error PublicSSHKey(regenerate bool) (ssh.PublicKey, error) } diff --git a/cluster/kubernetes/kubernetes.go b/cluster/kubernetes/kubernetes.go index 73daf2aeb..582366edb 100644 --- a/cluster/kubernetes/kubernetes.go +++ b/cluster/kubernetes/kubernetes.go @@ -37,30 +37,6 @@ type extendedClient struct { fluxHelmClient } -// --- internal types for keeping track of syncing - -type metadata struct { - Name string `yaml:"name"` - Namespace string `yaml:"namespace"` -} - -type apiObject struct { - OriginalResource resource.Resource - Payload []byte - Kind string `yaml:"kind"` - Metadata metadata `yaml:"metadata"` -} - -// A convenience for getting an minimal object from some bytes. -func parseObj(def []byte) (*apiObject, error) { - obj := apiObject{} - return &obj, yaml.Unmarshal(def, &obj) -} - -func (o *apiObject) hasNamespace() bool { - return o.Metadata.Namespace != "" -} - // --- add-ons // Kubernetes has a mechanism of "Add-ons", whereby manifest files @@ -99,6 +75,9 @@ func isAddon(obj k8sObject) bool { // Cluster is a handle to a Kubernetes API server. // (Typically, this code is deployed into the same cluster.) type Cluster struct { + // Do garbage collection when syncing resources + GC bool + client extendedClient applier Applier version string // string response for the version command. @@ -223,79 +202,57 @@ func (c *Cluster) AllControllers(namespace string) (res []cluster.Controller, er return allControllers, nil } -func applyMetadata(res resource.Resource, resourceLabels map[string]policy.Update, resourcePolicyUpdates map[string]policy.Update) ([]byte, error) { - id := res.ResourceID().String() - - definition := make(map[interface{}]interface{}) +func applyMetadata(res resource.Resource, stack, checksum string) ([]byte, error) { + definition := map[interface{}]interface{}{} if err := yaml.Unmarshal(res.Bytes(), &definition); err != nil { return nil, errors.Wrap(err, fmt.Sprintf("failed to parse yaml from %s", res.Source())) } - if update, ok := resourceLabels[id]; ok { - mixin := make(map[interface{}]interface{}) - var mixinBuffer bytes.Buffer - mixinBuffer.WriteString("metadata:\n labels:\n") - for key, value := range update.Add.ToStringMap() { - mixinBuffer.WriteString(fmt.Sprintf(" %s: %s\n", fmt.Sprintf("%s%s", kresource.PolicyPrefix, key), value)) - } - if err := yaml.Unmarshal(mixinBuffer.Bytes(), &mixin); err != nil { - return nil, errors.Wrap(err, "failed to parse yaml for mixin") - } - mergo.Merge(&definition, mixin) + mixin := map[string]interface{}{} + + if stack != "" { + mixinLabels := map[string]string{} + mixinLabels[fmt.Sprintf("%s%s", kresource.PolicyPrefix, policy.Stack)] = stack + mixin["labels"] = mixinLabels } - if update, ok := resourcePolicyUpdates[id]; ok { - mixin := make(map[interface{}]interface{}) - var mixinBuffer bytes.Buffer - mixinBuffer.WriteString("metadata:\n annotations:\n") - for key, value := range update.Add.ToStringMap() { - mixinBuffer.WriteString(fmt.Sprintf(" %s: %s\n", fmt.Sprintf("%s%s", kresource.PolicyPrefix, key), value)) - } - if err := yaml.Unmarshal(mixinBuffer.Bytes(), &mixin); err != nil { - return nil, errors.Wrap(err, "failed to parse yaml for mixin") - } - mergo.Merge(&definition, mixin) + if checksum != "" { + mixinAnnotations := map[string]string{} + mixinAnnotations[fmt.Sprintf("%s%s", kresource.PolicyPrefix, policy.StackChecksum)] = checksum + mixin["annotations"] = mixinAnnotations } + mergo.Merge(&definition, map[interface{}]interface{}{ + "metadata": mixin, + }) + bytes, err := yaml.Marshal(definition) if err != nil { return nil, errors.Wrap(err, "failed to serialize yaml after applying metadata") } - return bytes, nil } // Sync performs the given actions on resources. Operations are -// asynchronous, but serialised. -func (c *Cluster) Sync(spec cluster.SyncDef, resourceLabels map[string]policy.Update, resourcePolicyUpdates map[string]policy.Update) error { +// asynchronous (applications may take a while to be processed), but +// serialised. +func (c *Cluster) Sync(spec cluster.SyncDef) error { logger := log.With(c.logger, "method", "Sync") + // Keep track of the checksum each resource gets, so we can + // compare them during garbage collection. + checksums := map[string]string{} + cs := makeChangeSet() var errs cluster.SyncError - for _, action := range spec.Actions { - stages := []struct { - res resource.Resource - cmd string - }{ - {action.Delete, "delete"}, - {action.Apply, "apply"}, - } - for _, stage := range stages { - if stage.res == nil { - continue - } - - var obj *apiObject - resBytes, err := applyMetadata(stage.res, resourceLabels, resourcePolicyUpdates) - if err == nil { - obj, err = parseObj(resBytes) - } + for _, stack := range spec.Stacks { + for _, res := range stack.Resources { + resBytes, err := applyMetadata(res, stack.Name, stack.Checksum) if err == nil { - obj.OriginalResource = stage.res - obj.Payload = resBytes - cs.stage(stage.cmd, obj) + checksums[res.ResourceID().String()] = stack.Checksum + cs.stage("apply", res, resBytes) } else { - errs = append(errs, cluster.ResourceError{Resource: stage.res, Error: err}) + errs = append(errs, cluster.ResourceError{Resource: res, Error: err}) break } } @@ -309,7 +266,39 @@ func (c *Cluster) Sync(spec cluster.SyncDef, resourceLabels map[string]policy.Up } c.muSyncErrors.RUnlock() - // If `nil`, errs is a cluster.SyncError(nil) rather than error(nil) + if c.GC { + orphanedResources := makeChangeSet() + + clusterResourceBytes, err := c.exportByLabel(fmt.Sprintf("%s%s", kresource.PolicyPrefix, policy.Stack)) + if err != nil { + return errors.Wrap(err, "exporting resource defs from cluster for garbage collection") + } + clusterResources, err := kresource.ParseMultidoc(clusterResourceBytes, "exported") + if err != nil { + return errors.Wrap(err, "parsing exported resources during garbage collection") + } + + for resourceID, res := range clusterResources { + expected := checksums[resourceID] // shall be "" if no such resource was applied earlier + actual, ok := res.Policy().Get(policy.StackChecksum) + switch { + case !ok: + stack, _ := res.Policy().Get(policy.Stack) + c.logger.Log("warning", "cluster resource has stack but no checksum; skipping", "resource", resourceID, "stack", stack) + case actual != expected: // including if checksum is "" + c.logger.Log("info", "cluster resource has out-of-date checksum; deleting", "resource", resourceID, "actual", actual, "expected", expected) + orphanedResources.stage("delete", res, res.Bytes()) + default: + // all good; proceed + } + } + + if deleteErrs := c.applier.apply(logger, orphanedResources, nil); len(deleteErrs) > 0 { + errs = append(errs, deleteErrs...) + } + } + + // If `nil`, errs is a cluster.SyncError(nil) rather than error(nil), so it cannot be returned directly. if errs == nil { return nil } @@ -390,7 +379,9 @@ func contains(a []string, x string) bool { return false } -func (c *Cluster) ExportByLabel(labelName string, labelValue string) ([]byte, error) { +// exportByLabel collates all the resources that have a particular +// label (regardless of the value). +func (c *Cluster) exportByLabel(labelName string) ([]byte, error) { var config bytes.Buffer resources, err := c.client.coreClient.Discovery().ServerResources() @@ -423,7 +414,7 @@ func (c *Cluster) ExportByLabel(labelName string, labelValue string) ([]byte, er Resource: apiResource.Name, }) data, err := resourceClient.List(meta_v1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", labelName, labelValue), + LabelSelector: labelName, // exists <> }) if err != nil { return nil, err @@ -438,6 +429,7 @@ func (c *Cluster) ExportByLabel(labelName string, labelValue string) ([]byte, er if itemDesc == "v1:ComponentStatus" || itemDesc == "v1:Endpoints" { continue } + // TODO(michael) also exclude anything that has an ownerReference (that isn't "standard"?) yamlBytes, err := k8syaml.Marshal(item.Object) if err != nil { diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index 53f2a5373..ae894e4de 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -15,18 +15,30 @@ import ( "github.com/pkg/errors" "github.com/weaveworks/flux" "github.com/weaveworks/flux/cluster" + "github.com/weaveworks/flux/resource" ) +// --- internal types for keeping track of syncing + +type applyObject struct { + OriginalResource resource.Resource + Payload []byte +} + +func (obj applyObject) Components() (namespace, kind, name string) { + return obj.OriginalResource.ResourceID().Components() +} + type changeSet struct { - objs map[string][]*apiObject + objs map[string][]applyObject } func makeChangeSet() changeSet { - return changeSet{objs: make(map[string][]*apiObject)} + return changeSet{objs: make(map[string][]applyObject)} } -func (c *changeSet) stage(cmd string, o *apiObject) { - c.objs[cmd] = append(c.objs[cmd], o) +func (c *changeSet) stage(cmd string, res resource.Resource, bytes []byte) { + c.objs[cmd] = append(c.objs[cmd], applyObject{res, bytes}) } // Applier is something that will apply a changeset to the cluster. @@ -76,18 +88,18 @@ func (c *Kubectl) connectArgs() []string { // in the partial ordering of Kubernetes resources, according to which // kinds depend on which (derived by hand). func rankOfKind(kind string) int { - switch kind { + switch strings.ToLower(kind) { // Namespaces answer to NOONE - case "Namespace": + case "namespace": return 0 // These don't go in namespaces; or do, but don't depend on anything else - case "CustomResourceDefinition", "ServiceAccount", "ClusterRole", "Role", "PersistentVolume", "Service": + case "customresourcedefinition", "serviceaccount", "clusterrole", "role", "persistentvolume", "service": return 1 // These depend on something above, but not each other - case "ResourceQuota", "LimitRange", "Secret", "ConfigMap", "RoleBinding", "ClusterRoleBinding", "PersistentVolumeClaim", "Ingress": + case "resourcequota", "limitrange", "secret", "configmap", "rolebinding", "clusterrolebinding", "persistentvolumeclaim", "ingress": return 2 // Same deal, next layer - case "DaemonSet", "Deployment", "ReplicationController", "ReplicaSet", "Job", "CronJob", "StatefulSet": + case "daemonset", "deployment", "replicationcontroller", "replicaset", "job", "cronjob", "statefulset": return 3 // Assumption: anything not mentioned isn't depended _upon_, so // can come last. @@ -96,7 +108,7 @@ func rankOfKind(kind string) int { } } -type applyOrder []*apiObject +type applyOrder []applyObject func (objs applyOrder) Len() int { return len(objs) @@ -107,22 +119,24 @@ func (objs applyOrder) Swap(i, j int) { } func (objs applyOrder) Less(i, j int) bool { - ranki, rankj := rankOfKind(objs[i].Kind), rankOfKind(objs[j].Kind) + _, ki, ni := objs[i].Components() + _, kj, nj := objs[j].Components() + ranki, rankj := rankOfKind(ki), rankOfKind(kj) if ranki == rankj { - return objs[i].Metadata.Name < objs[j].Metadata.Name + return ni < nj } return ranki < rankj } func (c *Kubectl) apply(logger log.Logger, cs changeSet, errored map[flux.ResourceID]error) (errs cluster.SyncError) { - f := func(objs []*apiObject, cmd string, args ...string) { + f := func(objs []applyObject, cmd string, args ...string) { if len(objs) == 0 { return } logger.Log("cmd", cmd, "args", strings.Join(args, " "), "count", len(objs)) args = append(args, cmd) - var multi, single []*apiObject + var multi, single []applyObject if len(errored) == 0 { multi = objs } else { @@ -185,7 +199,7 @@ func (c *Kubectl) doCommand(logger log.Logger, r io.Reader, args ...string) erro return err } -func makeMultidoc(objs []*apiObject) *bytes.Buffer { +func makeMultidoc(objs []applyObject) *bytes.Buffer { buf := &bytes.Buffer{} for _, obj := range objs { buf.WriteString("\n---\n") diff --git a/cluster/sync.go b/cluster/sync.go index 9859ee00f..7c9c9d889 100644 --- a/cluster/sync.go +++ b/cluster/sync.go @@ -8,16 +8,22 @@ import ( // Definitions for use in synchronising a cluster with a git repo. -// SyncAction represents either the deletion or application (create or -// update) of a resource. -type SyncAction struct { - Delete resource.Resource // ) one of these - Apply resource.Resource // ) +// SyncStack groups a set of resources to be updated. The purpose of +// the grouping is to limit the "blast radius" of changes. For +// example, if we calculate a checksum for each stack and annotate the +// resources within it, any single change will affect only the +// resources in the same stack, meaning fewer things to annotate. (So +// why not do these individually? This, too, can be expensive, since +// it involves examining each resource individually). +type SyncStack struct { + Name string + Checksum string + Resources []resource.Resource } type SyncDef struct { - // The actions to undertake - Actions []SyncAction + // The applications to undertake + Stacks []SyncStack } type ResourceError struct { diff --git a/cmd/fluxd/main.go b/cmd/fluxd/main.go index e8f977d57..15e9370bf 100644 --- a/cmd/fluxd/main.go +++ b/cmd/fluxd/main.go @@ -95,8 +95,10 @@ func main() { gitPollInterval = fs.Duration("git-poll-interval", 5*time.Minute, "period at which to poll git repo for new commits") gitTimeout = fs.Duration("git-timeout", 20*time.Second, "duration after which git operations time out") + // syncing syncInterval = fs.Duration("sync-interval", 5*time.Minute, "apply config in git to cluster at least this often, even if there are no new commits") + syncGC = fs.Bool("sync-garbage-collection", false, "experimental; delete resources that are no longer in the git repo") // registry memcachedHostname = fs.String("memcached-hostname", "memcached", "hostname for memcached service.") @@ -268,6 +270,7 @@ func main() { kubectlApplier := kubernetes.NewKubectl(kubectl, restClientConfig) k8sInst := kubernetes.NewCluster(clientset, dynamicClientset, ifclientset, kubectlApplier, sshKeyRing, logger, *k8sNamespaceWhitelist, *registryExcludeImage) + k8sInst.GC = *syncGC if err := k8sInst.Ping(); err != nil { logger.Log("ping", err) diff --git a/daemon/loop.go b/daemon/loop.go index 1c15c523c..745356c00 100644 --- a/daemon/loop.go +++ b/daemon/loop.go @@ -205,9 +205,7 @@ func (d *Daemon) doSync(logger log.Logger, lastKnownSyncTagRev *string, warnedAb } var resourceErrors []event.ResourceError - // TODO supply deletes argument from somewhere (command-line?) - // TODO: supply tracking argument from somewhere - if err := fluxsync.Sync(logger, d.Manifests, allResources, d.Cluster, true, false); err != nil { + if err := fluxsync.Sync(logger, d.Manifests, allResources, d.Cluster); err != nil { logger.Log("err", err) switch syncerr := err.(type) { case cluster.SyncError: diff --git a/policy/policy.go b/policy/policy.go index bfdbb1df6..f43d8d787 100644 --- a/policy/policy.go +++ b/policy/policy.go @@ -14,6 +14,7 @@ const ( LockedMsg = Policy("locked_msg") Automated = Policy("automated") TagAll = Policy("tag_all") + Stack = Policy("stack") StackChecksum = Policy("stack_checksum") ) diff --git a/sync/sync.go b/sync/sync.go index 0aec0d2db..4ee452bb5 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -3,52 +3,18 @@ package sync import ( "crypto/sha1" "encoding/hex" - "fmt" "sort" "github.com/go-kit/kit/log" "github.com/pkg/errors" "github.com/weaveworks/flux/cluster" - kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" "github.com/weaveworks/flux/policy" "github.com/weaveworks/flux/resource" ) -// Checksum generates a unique identifier for all apply actions in the stack -func getStackChecksum(repoResources map[string]resource.Resource) string { - checksum := sha1.New() - - sortedKeys := make([]string, 0, len(repoResources)) - for resourceID := range repoResources { - sortedKeys = append(sortedKeys, resourceID) - } - sort.Strings(sortedKeys) - for resourceIDIndex := range sortedKeys { - checksum.Write(repoResources[sortedKeys[resourceIDIndex]].Bytes()) - } - return hex.EncodeToString(checksum.Sum(nil)) -} - -func garbageCollect(orphans []string, clusterResources map[string]resource.Resource, clus cluster.Cluster, logger log.Logger) error { - garbageCollect := cluster.SyncDef{} - emptyResources := map[string]resource.Resource{"noop/noop:noop": nil} - for _, id := range orphans { - res, ok := clusterResources[id] - if !ok { - return errors.Errorf("invariant: unable to find resource %s\n", id) - } - if prepareSyncDelete(logger, emptyResources, id, res, &garbageCollect) { - // TODO: use logger - fmt.Printf("[stack-tracking] marking resource %s for deletion\n", id) - } - } - return clus.Sync(garbageCollect, map[string]policy.Update{}, map[string]policy.Update{}) -} - -// Sync synchronises the cluster to the files in a directory -func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resource.Resource, clus cluster.Cluster, - deletes, tracks bool) error { +// Sync synchronises the cluster to the files under a directory. +func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resource.Resource, clus cluster.Cluster) error { // Get a map of resources defined in the cluster clusterBytes, err := clus.Export() @@ -60,113 +26,51 @@ func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resou return errors.Wrap(err, "parsing exported resources") } - // Everything that's in the cluster but not in the repo, delete; - // everything that's in the repo, apply. This is an approximation - // to figuring out what's changed, and applying that. We're - // relying on Kubernetes to decide for each application if it is a - // no-op. - sync := cluster.SyncDef{} - - var stackName, stackChecksum string - resourceLabels := map[string]policy.Update{} - resourcePolicyUpdates := map[string]policy.Update{} - if tracks { - stackName = "default" // TODO: multiple stack support - stackChecksum = getStackChecksum(repoResources) - - fmt.Printf("[stack-tracking] stack=%s, checksum=%s\n", stackName, stackChecksum) - - for id := range repoResources { - fmt.Printf("[stack-tracking] resource=%s, applying checksum=%s\n", id, stackChecksum) - resourceLabels[id] = policy.Update{ - Add: policy.Set{"stack": stackName}, - } - resourcePolicyUpdates[id] = policy.Update{ - Add: policy.Set{policy.StackChecksum: stackChecksum}, - } - } - } - - // DANGER ZONE (tamara) This works and is dangerous. At the moment will delete Flux and - // other pods unless the relevant manifests are part of the user repo. Needs a lot of thought - // before this cleanup cluster feature can be unleashed on the world. - if deletes { - for id, res := range clusterResources { - prepareSyncDelete(logger, repoResources, id, res, &sync) - } - } - - for id, res := range repoResources { - prepareSyncApply(logger, clusterResources, id, res, &sync) - } + // TODO: multiple stack support. This will involve partitioning + // the resources into disjoint maps, then passing each to + // makeStack. + defaultStack := makeStack("default", repoResources, clusterResources, logger) - if err := clus.Sync(sync, resourceLabels, resourcePolicyUpdates); err != nil { + sync := cluster.SyncDef{Stacks: []cluster.SyncStack{defaultStack}} + if err := clus.Sync(sync); err != nil { return err } - if tracks { - orphanedResources := make([]string, 0) - - fmt.Printf("[stack-tracking] scanning stack (%s) for orphaned resources\n", stackName) - clusterResourceBytes, err := clus.ExportByLabel(fmt.Sprintf("%s%s", kresource.PolicyPrefix, "stack"), stackName) - if err != nil { - return errors.Wrap(err, "exporting resource defs from cluster post-sync") - } - clusterResources, err = m.ParseManifests(clusterResourceBytes) - if err != nil { - return errors.Wrap(err, "parsing exported resources post-sync") - } - - for resourceID, res := range clusterResources { - if res.Policy().Has(policy.StackChecksum) { - val, _ := res.Policy().Get(policy.StackChecksum) - if val != stackChecksum { - fmt.Printf("[stack-tracking] cluster resource=%s, invalid checksum=%s\n", resourceID, val) - orphanedResources = append(orphanedResources, resourceID) - } else { - fmt.Printf("[stack-tracking] cluster resource ok: %s\n", resourceID) - } - } else { - fmt.Printf("warning: [stack-tracking] cluster resource=%s, missing policy=%s\n", resourceID, policy.StackChecksum) - } - } - - if len(orphanedResources) > 0 { - return garbageCollect(orphanedResources, clusterResources, clus, logger) - } - } - return nil } -func prepareSyncDelete(logger log.Logger, repoResources map[string]resource.Resource, id string, res resource.Resource, sync *cluster.SyncDef) bool { - if len(repoResources) == 0 { - return false - } - if res.Policy().Has(policy.Ignore) { - logger.Log("resource", res.ResourceID(), "ignore", "delete") - return false - } - if _, ok := repoResources[id]; !ok { - sync.Actions = append(sync.Actions, cluster.SyncAction{ - Delete: res, - }) - return true - } - return false -} +func makeStack(name string, repoResources, clusterResources map[string]resource.Resource, logger log.Logger) cluster.SyncStack { + stack := cluster.SyncStack{Name: name} + var resources []resource.Resource -func prepareSyncApply(logger log.Logger, clusterResources map[string]resource.Resource, id string, res resource.Resource, sync *cluster.SyncDef) { - if res.Policy().Has(policy.Ignore) { - logger.Log("resource", res.ResourceID(), "ignore", "apply") - return + // To get a stable checksum, we have to sort the resources. + var ids []string + for id, _ := range repoResources { + ids = append(ids, id) } - if cres, ok := clusterResources[id]; ok { - if cres.Policy().Has(policy.Ignore) { + sort.Strings(ids) + + checksum := sha1.New() + for _, id := range ids { + res := repoResources[id] + if res.Policy().Has(policy.Ignore) { logger.Log("resource", res.ResourceID(), "ignore", "apply") - return + continue } + // It may be ignored in the cluster, but it isn't in the repo; + // and we don't want what happens in the cluster to affect the + // checksum. + checksum.Write(res.Bytes()) + + if cres, ok := clusterResources[id]; ok { + if cres.Policy().Has(policy.Ignore) { + logger.Log("resource", res.ResourceID(), "ignore", "apply") + continue + } + } + resources = append(resources, res) } - sync.Actions = append(sync.Actions, cluster.SyncAction{ - Apply: res, - }) + + stack.Resources = resources + stack.Checksum = hex.EncodeToString(checksum.Sum(nil)) + return stack } From e2b140f0ce927e8bf88b5d39a463f4b1e9a24655 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 8 Nov 2018 17:22:09 +0000 Subject: [PATCH 05/24] Bring tests up to date with sync changes To start, this changes types/definitions to be in line with the changes to `cluster.Cluster.Sync()` and `cluster.SyncDef`. After that, to verify that the cluster implementation does in fact apply and delete resources appropriately, we must have some fairly elaborate scaffolding. The cluster must be able to ask for the resources extant, as well as have apply and delete _stick_ (i.e, be observable in subsequent queries). To do that, we have to create fake clients, and make sure they are primed to respond in the expected way. This amounts to - making sure the core client can enumerate the namespaces used in the test; and, - giving the dynamic client the appropriate API resources, so it will know to ask for the objects used in the test. It's also necessary to have an up-to-date implementation of the fake dynamic client; prior versions had a bug that made these tests impossible. To avoid having a cascading dependency on the very latest kubernetes code, I've simply taken one file we need, at the version that works (see cluster/kubernetes/fakedynamicclient_test.go). The pay-off is that having set up this scaffolding, we can test syncing quite directly: the test itself boils down to "sync these fake manifests, then see whether they got applied in the cluster". --- Gopkg.lock | 4 + cluster/kubernetes/doc.go | 1 - cluster/kubernetes/fakedynamicclient_test.go | 387 +++++++++++++++++++ cluster/kubernetes/kubernetes.go | 7 +- cluster/kubernetes/kubernetes_test.go | 3 +- cluster/kubernetes/sync_test.go | 288 +++++++++++--- cluster/mock.go | 6 +- daemon/daemon_test.go | 11 +- daemon/loop_test.go | 13 +- sync/sync.go | 9 +- sync/sync_test.go | 183 +-------- 11 files changed, 657 insertions(+), 255 deletions(-) create mode 100644 cluster/kubernetes/fakedynamicclient_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 041107b45..13a20ad06 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -907,6 +907,7 @@ packages = [ "discovery", "discovery/fake", + "dynamic", "kubernetes", "kubernetes/fake", "kubernetes/scheme", @@ -1206,7 +1207,9 @@ "k8s.io/api/batch/v1beta1", "k8s.io/api/core/v1", "k8s.io/apimachinery/pkg/api/errors", + "k8s.io/apimachinery/pkg/api/meta", "k8s.io/apimachinery/pkg/apis/meta/v1", + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured", "k8s.io/apimachinery/pkg/labels", "k8s.io/apimachinery/pkg/runtime", "k8s.io/apimachinery/pkg/runtime/schema", @@ -1217,6 +1220,7 @@ "k8s.io/apimachinery/pkg/watch", "k8s.io/client-go/discovery", "k8s.io/client-go/discovery/fake", + "k8s.io/client-go/dynamic", "k8s.io/client-go/kubernetes", "k8s.io/client-go/kubernetes/fake", "k8s.io/client-go/kubernetes/scheme", diff --git a/cluster/kubernetes/doc.go b/cluster/kubernetes/doc.go index 9d8d86e63..230cdba94 100644 --- a/cluster/kubernetes/doc.go +++ b/cluster/kubernetes/doc.go @@ -3,5 +3,4 @@ Package kubernetes provides implementations of `Cluster` and `Manifests` that interact with the Kubernetes API (using kubectl or the k8s API client). */ - package kubernetes diff --git a/cluster/kubernetes/fakedynamicclient_test.go b/cluster/kubernetes/fakedynamicclient_test.go new file mode 100644 index 000000000..7e4f3df42 --- /dev/null +++ b/cluster/kubernetes/fakedynamicclient_test.go @@ -0,0 +1,387 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +/* +This file was obtained from +https://github.com/kubernetes/client-go/blob/4b43750b963d2b6e0f7527fe558e71c47bfc5045/dynamic/fake/simple.go +and modified in the following way(s): + + - the package was changed to `kubernetes` + +This file is here because it has a fix for +https://github.com/kubernetes/client-go/issues/465, which is included +in client-go v9, which we're not able to vendor at this time. + +It can be removed, and the fake clientset from the original package +used, when we are ready to vendor client-go v9 and kubernetes-1.12. +*/ + +import ( + "strings" + gotesting "testing" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/testing" +) + +func TestFakeClientConstruction(t *gotesting.T) { +} + +func NewSimpleDynamicClient(scheme *runtime.Scheme, objects ...runtime.Object) *FakeDynamicClient { + // In order to use List with this client, you have to have the v1.List registered in your scheme. Neat thing though + // it does NOT have to be the *same* list + scheme.AddKnownTypeWithName(schema.GroupVersionKind{Group: "fake-dynamic-client-group", Version: "v1", Kind: "List"}, &unstructured.UnstructuredList{}) + + codecs := serializer.NewCodecFactory(scheme) + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &FakeDynamicClient{} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type FakeDynamicClient struct { + testing.Fake + scheme *runtime.Scheme +} + +type dynamicResourceClient struct { + client *FakeDynamicClient + namespace string + resource schema.GroupVersionResource +} + +var _ dynamic.Interface = &FakeDynamicClient{} + +func (c *FakeDynamicClient) Resource(resource schema.GroupVersionResource) dynamic.NamespaceableResourceInterface { + return &dynamicResourceClient{client: c, resource: resource} +} + +func (c *dynamicResourceClient) Namespace(ns string) dynamic.ResourceInterface { + ret := *c + ret.namespace = ns + return &ret +} + +func (c *dynamicResourceClient) Create(obj *unstructured.Unstructured, subresources ...string) (*unstructured.Unstructured, error) { + var uncastRet runtime.Object + var err error + switch { + case len(c.namespace) == 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootCreateAction(c.resource, obj), obj) + + case len(c.namespace) == 0 && len(subresources) > 0: + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, err + } + name := accessor.GetName() + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootCreateSubresourceAction(c.resource, name, strings.Join(subresources, "/"), obj), obj) + + case len(c.namespace) > 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewCreateAction(c.resource, c.namespace, obj), obj) + + case len(c.namespace) > 0 && len(subresources) > 0: + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, err + } + name := accessor.GetName() + uncastRet, err = c.client.Fake. + Invokes(testing.NewCreateSubresourceAction(c.resource, name, strings.Join(subresources, "/"), c.namespace, obj), obj) + + } + + if err != nil { + return nil, err + } + if uncastRet == nil { + return nil, err + } + + ret := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil { + return nil, err + } + return ret, err +} + +func (c *dynamicResourceClient) Update(obj *unstructured.Unstructured, subresources ...string) (*unstructured.Unstructured, error) { + var uncastRet runtime.Object + var err error + switch { + case len(c.namespace) == 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootUpdateAction(c.resource, obj), obj) + + case len(c.namespace) == 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(c.resource, strings.Join(subresources, "/"), obj), obj) + + case len(c.namespace) > 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewUpdateAction(c.resource, c.namespace, obj), obj) + + case len(c.namespace) > 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewUpdateSubresourceAction(c.resource, strings.Join(subresources, "/"), c.namespace, obj), obj) + + } + + if err != nil { + return nil, err + } + if uncastRet == nil { + return nil, err + } + + ret := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil { + return nil, err + } + return ret, err +} + +func (c *dynamicResourceClient) UpdateStatus(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + var uncastRet runtime.Object + var err error + switch { + case len(c.namespace) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(c.resource, "status", obj), obj) + + case len(c.namespace) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewUpdateSubresourceAction(c.resource, "status", c.namespace, obj), obj) + + } + + if err != nil { + return nil, err + } + if uncastRet == nil { + return nil, err + } + + ret := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil { + return nil, err + } + return ret, err +} + +func (c *dynamicResourceClient) Delete(name string, opts *metav1.DeleteOptions, subresources ...string) error { + var err error + switch { + case len(c.namespace) == 0 && len(subresources) == 0: + _, err = c.client.Fake. + Invokes(testing.NewRootDeleteAction(c.resource, name), &metav1.Status{Status: "dynamic delete fail"}) + + case len(c.namespace) == 0 && len(subresources) > 0: + _, err = c.client.Fake. + Invokes(testing.NewRootDeleteSubresourceAction(c.resource, strings.Join(subresources, "/"), name), &metav1.Status{Status: "dynamic delete fail"}) + + case len(c.namespace) > 0 && len(subresources) == 0: + _, err = c.client.Fake. + Invokes(testing.NewDeleteAction(c.resource, c.namespace, name), &metav1.Status{Status: "dynamic delete fail"}) + + case len(c.namespace) > 0 && len(subresources) > 0: + _, err = c.client.Fake. + Invokes(testing.NewDeleteSubresourceAction(c.resource, strings.Join(subresources, "/"), c.namespace, name), &metav1.Status{Status: "dynamic delete fail"}) + } + + return err +} + +func (c *dynamicResourceClient) DeleteCollection(opts *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var err error + switch { + case len(c.namespace) == 0: + action := testing.NewRootDeleteCollectionAction(c.resource, listOptions) + _, err = c.client.Fake.Invokes(action, &metav1.Status{Status: "dynamic deletecollection fail"}) + + case len(c.namespace) > 0: + action := testing.NewDeleteCollectionAction(c.resource, c.namespace, listOptions) + _, err = c.client.Fake.Invokes(action, &metav1.Status{Status: "dynamic deletecollection fail"}) + + } + + return err +} + +func (c *dynamicResourceClient) Get(name string, opts metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) { + var uncastRet runtime.Object + var err error + switch { + case len(c.namespace) == 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootGetAction(c.resource, name), &metav1.Status{Status: "dynamic get fail"}) + + case len(c.namespace) == 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootGetSubresourceAction(c.resource, strings.Join(subresources, "/"), name), &metav1.Status{Status: "dynamic get fail"}) + + case len(c.namespace) > 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewGetAction(c.resource, c.namespace, name), &metav1.Status{Status: "dynamic get fail"}) + + case len(c.namespace) > 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewGetSubresourceAction(c.resource, c.namespace, strings.Join(subresources, "/"), name), &metav1.Status{Status: "dynamic get fail"}) + } + + if err != nil { + return nil, err + } + if uncastRet == nil { + return nil, err + } + + ret := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil { + return nil, err + } + return ret, err +} + +func (c *dynamicResourceClient) List(opts metav1.ListOptions) (*unstructured.UnstructuredList, error) { + var obj runtime.Object + var err error + switch { + case len(c.namespace) == 0: + obj, err = c.client.Fake. + Invokes(testing.NewRootListAction(c.resource, schema.GroupVersionKind{Group: "fake-dynamic-client-group", Version: "v1", Kind: "" /*List is appended by the tracker automatically*/}, opts), &metav1.Status{Status: "dynamic list fail"}) + + case len(c.namespace) > 0: + obj, err = c.client.Fake. + Invokes(testing.NewListAction(c.resource, schema.GroupVersionKind{Group: "fake-dynamic-client-group", Version: "v1", Kind: "" /*List is appended by the tracker automatically*/}, c.namespace, opts), &metav1.Status{Status: "dynamic list fail"}) + + } + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + + retUnstructured := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(obj, retUnstructured, nil); err != nil { + return nil, err + } + entireList, err := retUnstructured.ToList() + if err != nil { + return nil, err + } + + list := &unstructured.UnstructuredList{} + for i := range entireList.Items { + item := &entireList.Items[i] + metadata, err := meta.Accessor(item) + if err != nil { + return nil, err + } + if label.Matches(labels.Set(metadata.GetLabels())) { + list.Items = append(list.Items, *item) + } + } + return list, nil +} + +func (c *dynamicResourceClient) Watch(opts metav1.ListOptions) (watch.Interface, error) { + switch { + case len(c.namespace) == 0: + return c.client.Fake. + InvokesWatch(testing.NewRootWatchAction(c.resource, opts)) + + case len(c.namespace) > 0: + return c.client.Fake. + InvokesWatch(testing.NewWatchAction(c.resource, c.namespace, opts)) + + } + + panic("math broke") +} + +func (c *dynamicResourceClient) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (*unstructured.Unstructured, error) { + var uncastRet runtime.Object + var err error + switch { + case len(c.namespace) == 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootPatchAction(c.resource, name, data), &metav1.Status{Status: "dynamic patch fail"}) + + case len(c.namespace) == 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootPatchSubresourceAction(c.resource, name, data, subresources...), &metav1.Status{Status: "dynamic patch fail"}) + + case len(c.namespace) > 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewPatchAction(c.resource, c.namespace, name, data), &metav1.Status{Status: "dynamic patch fail"}) + + case len(c.namespace) > 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewPatchSubresourceAction(c.resource, c.namespace, name, data, subresources...), &metav1.Status{Status: "dynamic patch fail"}) + + } + + if err != nil { + return nil, err + } + if uncastRet == nil { + return nil, err + } + + ret := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil { + return nil, err + } + return ret, err +} diff --git a/cluster/kubernetes/kubernetes.go b/cluster/kubernetes/kubernetes.go index 582366edb..15a8e1256 100644 --- a/cluster/kubernetes/kubernetes.go +++ b/cluster/kubernetes/kubernetes.go @@ -269,7 +269,7 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { if c.GC { orphanedResources := makeChangeSet() - clusterResourceBytes, err := c.exportByLabel(fmt.Sprintf("%s%s", kresource.PolicyPrefix, policy.Stack)) + clusterResourceBytes, err := c.exportResourcesInStack() if err != nil { return errors.Wrap(err, "exporting resource defs from cluster for garbage collection") } @@ -379,9 +379,10 @@ func contains(a []string, x string) bool { return false } -// exportByLabel collates all the resources that have a particular +// exportResourcesInStack collates all the resources that have a particular // label (regardless of the value). -func (c *Cluster) exportByLabel(labelName string) ([]byte, error) { +func (c *Cluster) exportResourcesInStack() ([]byte, error) { + labelName := fmt.Sprintf("%s%s", kresource.PolicyPrefix, policy.Stack) var config bytes.Buffer resources, err := c.client.coreClient.Discovery().ServerResources() diff --git a/cluster/kubernetes/kubernetes_test.go b/cluster/kubernetes/kubernetes_test.go index e11538d17..9143da643 100644 --- a/cluster/kubernetes/kubernetes_test.go +++ b/cluster/kubernetes/kubernetes_test.go @@ -25,8 +25,7 @@ func newNamespace(name string) *apiv1.Namespace { func testGetAllowedNamespaces(t *testing.T, namespace []string, expected []string) { clientset := fakekubernetes.NewSimpleClientset(newNamespace("default"), newNamespace("kube-system")) - - c := NewCluster(clientset, nil, nil, nil, log.NewNopLogger(), namespace, []string{}) + c := NewCluster(clientset, nil, nil, nil, nil, log.NewNopLogger(), namespace, []string{}) namespaces, err := c.getAllowedNamespaces() if err != nil { diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index 4847ca350..ddd4fe51b 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -2,53 +2,151 @@ package kubernetes import ( "sort" + "strings" "testing" + "github.com/ghodss/yaml" "github.com/go-kit/kit/log" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + // "k8s.io/apimachinery/pkg/runtime/serializer" + // "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" + // dynamicfake "k8s.io/client-go/dynamic/fake" + // k8sclient "k8s.io/client-go/kubernetes" + corefake "k8s.io/client-go/kubernetes/fake" + k8s_testing "k8s.io/client-go/testing" "github.com/weaveworks/flux" "github.com/weaveworks/flux/cluster" + fluxfake "github.com/weaveworks/flux/integrations/client/clientset/versioned/fake" "github.com/weaveworks/flux/policy" + "github.com/weaveworks/flux/sync" ) -type mockApplier struct { - commandRun bool -} +func fakeClients() extendedClient { + scheme := runtime.NewScheme() + + // Set this to `true` to output a trace of the API actions called + // while running the tests + const debug = false -func (m *mockApplier) apply(_ log.Logger, c changeSet, errored map[flux.ResourceID]error) cluster.SyncError { - if len(c.objs) != 0 { - m.commandRun = true + getAndList := metav1.Verbs([]string{"get", "list"}) + // Adding these means the fake dynamic client will find them, and + // be able to enumerate (list and get) the resources that we care + // about + apiResources := []*metav1.APIResourceList{ + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + {Name: "deployments", SingularName: "deployment", Namespaced: true, Kind: "Deployment", Verbs: getAndList}, + }, + }, } - return nil -} -type rsc struct { - id string - bytes []byte -} + coreClient := corefake.NewSimpleClientset(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "foobar"}}) + fluxClient := fluxfake.NewSimpleClientset() + dynamicClient := NewSimpleDynamicClient(scheme) // NB from this package, rather than the official one, since we needed a patched version -func (r rsc) ResourceID() flux.ResourceID { - return flux.MustParseResourceID(r.id) -} + // Assigned here, since this is _also_ used by the (fake) + // discovery client therein, and ultimately by + // exportResourcesInStack since that uses the core clientset to + // enumerate the namespaces. + coreClient.Fake.Resources = apiResources -func (r rsc) Bytes() []byte { - return r.bytes + if debug { + for _, fake := range []*k8s_testing.Fake{&coreClient.Fake, &fluxClient.Fake, &dynamicClient.Fake} { + fake.PrependReactor("*", "*", func(action k8s_testing.Action) (bool, runtime.Object, error) { + gvr := action.GetResource() + println("[DEBUG] action: ", action.GetVerb(), gvr.Group, gvr.Version, gvr.Resource) + return false, nil, nil + }) + } + } + + return extendedClient{ + coreClient: coreClient, + fluxHelmClient: fluxClient, + dynamicClient: dynamicClient, + } } -func (r rsc) Policy() policy.Set { - return nil +// fakeApplier is an Applier that just forwards changeset operations +// to a dynamic client. It doesn't try to properly patch resources +// when that might be expected; it just overwrites them. But this is +// enough for checking whether sync operations succeeded and had the +// correct effect, which is either to "upsert", or delete, resources. +type fakeApplier struct { + client dynamic.Interface + commandRun bool } -func (r rsc) Source() string { - return "test" +func (a fakeApplier) apply(_ log.Logger, cs changeSet, errored map[flux.ResourceID]error) cluster.SyncError { + var errs []cluster.ResourceError + + operate := func(obj applyObject, cmd string) { + a.commandRun = true + var unstruct map[string]interface{} + if err := yaml.Unmarshal(obj.Payload, &unstruct); err != nil { + errs = append(errs, cluster.ResourceError{obj.OriginalResource, err}) + return + } + res := &unstructured.Unstructured{Object: unstruct} + gvk := res.GetObjectKind().GroupVersionKind() + gvr := schema.GroupVersionResource{Group: gvk.Group, Version: gvk.Version, Resource: strings.ToLower(gvk.Kind) + "s"} + c := a.client.Resource(gvr) + var dc dynamic.ResourceInterface = c + if ns := res.GetNamespace(); ns != "" { + dc = c.Namespace(ns) + } + name := res.GetName() + + if cmd == "apply" { + _, err := dc.Get(name, metav1.GetOptions{}) + switch { + case errors.IsNotFound(err): + _, err = dc.Create(res) //, &metav1.CreateOptions{}) + case err == nil: + _, err = dc.Update(res) //, &metav1.UpdateOptions{}) + } + if err != nil { + errs = append(errs, cluster.ResourceError{obj.OriginalResource, err}) + return + } + } else if cmd == "delete" { + if err := dc.Delete(name, &metav1.DeleteOptions{}); err != nil { + errs = append(errs, cluster.ResourceError{obj.OriginalResource, err}) + return + } + } else { + panic("unknown action: " + cmd) + } + } + + for _, obj := range cs.objs["delete"] { + operate(obj, "delete") + } + for _, obj := range cs.objs["apply"] { + operate(obj, "apply") + } + if len(errs) == 0 { + return nil + } + return errs } // --- -func setup(t *testing.T) (*Cluster, *mockApplier) { - applier := &mockApplier{} +func setup(t *testing.T) (*Cluster, *fakeApplier) { + clients := fakeClients() + applier := &fakeApplier{client: clients.dynamicClient} kube := &Cluster{ applier: applier, + client: clients, logger: log.NewNopLogger(), } return kube, applier @@ -56,7 +154,7 @@ func setup(t *testing.T) (*Cluster, *mockApplier) { func TestSyncNop(t *testing.T) { kube, mock := setup(t) - if err := kube.Sync(cluster.SyncDef{}, map[string]policy.Update{}, map[string]policy.Update{}); err != nil { + if err := kube.Sync(cluster.SyncDef{}); err != nil { t.Errorf("%#v", err) } if mock.commandRun { @@ -64,49 +162,121 @@ func TestSyncNop(t *testing.T) { } } -func TestSyncMalformed(t *testing.T) { - kube, mock := setup(t) - err := kube.Sync(cluster.SyncDef{ - Actions: []cluster.SyncAction{ - cluster.SyncAction{ - Apply: rsc{"default:deployment/trash", []byte("garbage")}, - }, - }, - }, map[string]policy.Update{}, map[string]policy.Update{}) - if err == nil { - t.Error("expected error because malformed resource def, but got nil") - } - if mock.commandRun { - t.Error("expected no commands run") +func TestSync(t *testing.T) { + const defs1 = `--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dep1 + namespace: foobar +` + + const defs2 = `--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dep2 + namespace: foobar +` + + const defs3 = `--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dep3 + namespace: other +` + + kube, _ := setup(t) + + test := func(defs, expectedAfterSync string) { + manifests := &Manifests{} + resources, err := manifests.ParseManifests([]byte(defs)) + if err != nil { + t.Fatal(err) + } + + err = sync.Sync(log.NewNopLogger(), manifests, resources, kube) + if err != nil { + t.Error(err) + } + // Now check that the resources were created + exported, err := kube.exportResourcesInStack() + if err != nil { + t.Fatal(err) + } + + resources0, err := manifests.ParseManifests([]byte(expectedAfterSync)) + if err != nil { + panic(err) + } + resources1, err := manifests.ParseManifests([]byte(exported)) + if err != nil { + panic(err) + } + + for id := range resources1 { + if _, ok := resources0[id]; !ok { + t.Errorf("resource present after sync but not in resources applied: %q", id) + } + } + for id := range resources0 { + if _, ok := resources1[id]; !ok { + t.Errorf("resource supposed to be synced but not present: %q", id) + } + } } + + // without GC on, resources persist if they are not mentioned in subsequent syncs. + test(defs1, defs1) + test(defs1+defs2, defs1+defs2) + test(defs3, defs1+defs2+defs3) + + // Now with GC switched on. That means if we don't include a + // resource in a sync, it should be deleted. + kube.GC = true + test(defs2+defs3, defs3+defs2) +} + +// ---- + +type rsc struct { + id string + bytes []byte +} + +func (r rsc) ResourceID() flux.ResourceID { + return flux.MustParseResourceID(r.id) +} + +func (r rsc) Bytes() []byte { + return r.bytes +} + +func (r rsc) Policy() policy.Set { + return nil +} + +func (r rsc) Source() string { + return "test" +} + +func mkResource(kind, name string) rsc { + return rsc{id: "default:" + kind + "/" + name} } // TestApplyOrder checks that applyOrder works as expected. func TestApplyOrder(t *testing.T) { - objs := []*apiObject{ - { - Kind: "Deployment", - Metadata: metadata{ - Name: "deploy", - }, - }, - { - Kind: "Secret", - Metadata: metadata{ - Name: "secret", - }, - }, - { - Kind: "Namespace", - Metadata: metadata{ - Name: "namespace", - }, - }, + objs := []applyObject{ + {OriginalResource: mkResource("Deployment", "deploy")}, + {OriginalResource: mkResource("Secret", "secret")}, + {OriginalResource: mkResource("Namespace", "namespace")}, } sort.Sort(applyOrder(objs)) for i, name := range []string{"namespace", "secret", "deploy"} { - if objs[i].Metadata.Name != name { - t.Errorf("Expected %q at position %d, got %q", name, i, objs[i].Metadata.Name) + _, _, objName := objs[i].OriginalResource.ResourceID().Components() + if objName != name { + t.Errorf("Expected %q at position %d, got %q", name, i, objName) } } } diff --git a/cluster/mock.go b/cluster/mock.go index 9d44467e1..4943bce44 100644 --- a/cluster/mock.go +++ b/cluster/mock.go @@ -14,7 +14,7 @@ type Mock struct { SomeServicesFunc func([]flux.ResourceID) ([]Controller, error) PingFunc func() error ExportFunc func() ([]byte, error) - SyncFunc func(SyncDef, map[string]policy.Update, map[string]policy.Update) error + SyncFunc func(SyncDef) error PublicSSHKeyFunc func(regenerate bool) (ssh.PublicKey, error) UpdateImageFunc func(def []byte, id flux.ResourceID, container string, newImageID image.Ref) ([]byte, error) LoadManifestsFunc func(base string, paths []string) (map[string]resource.Resource, error) @@ -39,8 +39,8 @@ func (m *Mock) Export() ([]byte, error) { return m.ExportFunc() } -func (m *Mock) Sync(c SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { - return m.SyncFunc(c, l, p) +func (m *Mock) Sync(c SyncDef) error { + return m.SyncFunc(c) } func (m *Mock) PublicSSHKey(regenerate bool) (ssh.PublicKey, error) { diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go index 62dd9f22d..d5d5454ab 100644 --- a/daemon/daemon_test.go +++ b/daemon/daemon_test.go @@ -21,7 +21,6 @@ import ( "github.com/weaveworks/flux/cluster" "github.com/weaveworks/flux/cluster/kubernetes" kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" - "github.com/weaveworks/flux/cluster/kubernetes/testfiles" "github.com/weaveworks/flux/event" "github.com/weaveworks/flux/git" "github.com/weaveworks/flux/git/gittest" @@ -290,7 +289,7 @@ func TestDaemon_ListImagesWithOptions(t *testing.T) { { name: "Override container field selection", opts: v10.ListImagesOptions{ - Spec: specAll, + Spec: specAll, OverrideContainerFields: []string{"Name", "Current", "NewAvailableImagesCount"}, }, expectedImages: []v6.ImageStatus{ @@ -320,7 +319,7 @@ func TestDaemon_ListImagesWithOptions(t *testing.T) { { name: "Override container field selection with invalid field", opts: v10.ListImagesOptions{ - Spec: specAll, + Spec: specAll, OverrideContainerFields: []string{"InvalidField"}, }, expectedImages: nil, @@ -359,7 +358,7 @@ func TestDaemon_NotifyChange(t *testing.T) { var syncCalled int var syncDef *cluster.SyncDef var syncMu sync.Mutex - mockK8s.SyncFunc = func(def cluster.SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { + mockK8s.SyncFunc = func(def cluster.SyncDef) error { syncMu.Lock() syncCalled++ syncDef = &def @@ -384,8 +383,6 @@ func TestDaemon_NotifyChange(t *testing.T) { t.Errorf("Sync was not called once, was called %d times", syncCalled) } else if syncDef == nil { t.Errorf("Sync was called with a nil syncDef") - } else if len(syncDef.Actions) != len(testfiles.ResourceMap) { - t.Errorf("Expected Sync called with %d actions (resources), was called with %d", len(testfiles.ResourceMap), len(syncDef.Actions)) } // Check that history was written to @@ -660,7 +657,7 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *cluster.Mock, *mockEven singleService, }, nil } - k8s.SyncFunc = func(def cluster.SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { return nil } + k8s.SyncFunc = func(def cluster.SyncDef) error { return nil } k8s.UpdatePoliciesFunc = (&kubernetes.Manifests{}).UpdatePolicies k8s.UpdateImageFunc = (&kubernetes.Manifests{}).UpdateImage } diff --git a/daemon/loop_test.go b/daemon/loop_test.go index 697ea2f76..810763bb0 100644 --- a/daemon/loop_test.go +++ b/daemon/loop_test.go @@ -21,7 +21,6 @@ import ( "github.com/weaveworks/flux/git" "github.com/weaveworks/flux/git/gittest" "github.com/weaveworks/flux/job" - "github.com/weaveworks/flux/policy" registryMock "github.com/weaveworks/flux/registry/mock" "github.com/weaveworks/flux/resource" ) @@ -101,7 +100,7 @@ func TestPullAndSync_InitialSync(t *testing.T) { expectedResourceIDs = append(expectedResourceIDs, id) } expectedResourceIDs.Sort() - k8s.SyncFunc = func(def cluster.SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { + k8s.SyncFunc = func(def cluster.SyncDef) error { syncCalled++ syncDef = &def return nil @@ -118,8 +117,6 @@ func TestPullAndSync_InitialSync(t *testing.T) { t.Errorf("Sync was not called once, was called %d times", syncCalled) } else if syncDef == nil { t.Errorf("Sync was called with a nil syncDef") - } else if len(syncDef.Actions) != len(expectedResourceIDs) { - t.Errorf("Sync was not called with %d actions (resources), was called with %d", len(expectedResourceIDs), len(syncDef.Actions)) } // The emitted event has all service ids @@ -174,7 +171,7 @@ func TestDoSync_NoNewCommits(t *testing.T) { expectedResourceIDs = append(expectedResourceIDs, id) } expectedResourceIDs.Sort() - k8s.SyncFunc = func(def cluster.SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { + k8s.SyncFunc = func(def cluster.SyncDef) error { syncCalled++ syncDef = &def return nil @@ -193,8 +190,6 @@ func TestDoSync_NoNewCommits(t *testing.T) { t.Errorf("Sync was not called once, was called %d times", syncCalled) } else if syncDef == nil { t.Errorf("Sync was called with a nil syncDef") - } else if len(syncDef.Actions) != len(expectedResourceIDs) { - t.Errorf("Sync was not called with %d actions, was called with: %d", len(expectedResourceIDs), len(syncDef.Actions)) } // The emitted event has no service ids @@ -272,7 +267,7 @@ func TestDoSync_WithNewCommit(t *testing.T) { expectedResourceIDs = append(expectedResourceIDs, id) } expectedResourceIDs.Sort() - k8s.SyncFunc = func(def cluster.SyncDef, l map[string]policy.Update, p map[string]policy.Update) error { + k8s.SyncFunc = func(def cluster.SyncDef) error { syncCalled++ syncDef = &def return nil @@ -289,8 +284,6 @@ func TestDoSync_WithNewCommit(t *testing.T) { t.Errorf("Sync was not called once, was called %d times", syncCalled) } else if syncDef == nil { t.Errorf("Sync was called with a nil syncDef") - } else if len(syncDef.Actions) != len(expectedResourceIDs) { - t.Errorf("Sync was not called with %d actions, was called with %d", len(expectedResourceIDs), len(syncDef.Actions)) } // The emitted event has no service ids diff --git a/sync/sync.go b/sync/sync.go index 4ee452bb5..991cb1041 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -13,8 +13,15 @@ import ( "github.com/weaveworks/flux/resource" ) +// Syncer has the methods we need to be able to compile and run a sync +type Syncer interface { + // TODO(michael) this could be less leaky as `() -> map[string]resource.Resource` + Export() ([]byte, error) + Sync(cluster.SyncDef) error +} + // Sync synchronises the cluster to the files under a directory. -func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resource.Resource, clus cluster.Cluster) error { +func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resource.Resource, clus Syncer) error { // Get a map of resources defined in the cluster clusterBytes, err := clus.Export() diff --git a/sync/sync_test.go b/sync/sync_test.go index cf3d0846e..a3f3f618a 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -2,38 +2,27 @@ package sync import ( "bytes" - "fmt" - "os" - "os/exec" - "reflect" - "strings" "testing" "github.com/go-kit/kit/log" - - // "github.com/weaveworks/flux" - "context" + "github.com/stretchr/testify/assert" "github.com/weaveworks/flux/cluster" "github.com/weaveworks/flux/cluster/kubernetes" - "github.com/weaveworks/flux/cluster/kubernetes/testfiles" "github.com/weaveworks/flux/git" "github.com/weaveworks/flux/git/gittest" - "github.com/weaveworks/flux/policy" "github.com/weaveworks/flux/resource" ) +// Test that cluster.Sync gets called with the appropriate things when +// run. func TestSync(t *testing.T) { checkout, cleanup := setup(t) defer cleanup() - // Let's test that cluster.Sync gets called with the appropriate - // things when we add and remove resources from the config. - // Start with nothing running. We should be told to apply all the things. - mockCluster := &cluster.Mock{} manifests := &kubernetes.Manifests{} - var clus cluster.Cluster = &syncCluster{mockCluster, map[string][]byte{}} + clus := &syncCluster{map[string][]byte{}} dirs := checkout.ManifestDirs() resources, err := manifests.LoadManifests(checkout.Dir(), dirs) @@ -41,136 +30,12 @@ func TestSync(t *testing.T) { t.Fatal(err) } - if err := Sync(log.NewNopLogger(), manifests, resources, clus, false, true); err != nil { - t.Fatal(err) - } - checkClusterMatchesFiles(t, manifests, clus, checkout.Dir(), dirs) - - for _, res := range testfiles.ServiceMap(checkout.Dir()) { - if err := execCommand("rm", res[0]); err != nil { - t.Fatal(err) - } - commitAction := git.CommitAction{Author: "", Message: "deleted " + res[0]} - if err := checkout.CommitAndPush(context.Background(), commitAction, nil); err != nil { - t.Fatal(err) - } - break - } - - resources, err = manifests.LoadManifests(checkout.Dir(), dirs) - if err != nil { - t.Fatal(err) - } - if err := Sync(log.NewNopLogger(), manifests, resources, clus, false, true); err != nil { + if err := Sync(log.NewNopLogger(), manifests, resources, clus); err != nil { t.Fatal(err) } checkClusterMatchesFiles(t, manifests, clus, checkout.Dir(), dirs) } -func TestPrepareSyncDelete(t *testing.T) { - var tests = []struct { - msg string - repoRes map[string]resource.Resource - res resource.Resource - expected *cluster.SyncDef - }{ - { - msg: "No repo resources provided during sync delete", - repoRes: map[string]resource.Resource{}, - res: mockResourceWithIgnorePolicy("service", "ns1", "s2"), - expected: &cluster.SyncDef{}, - }, - { - msg: "No policy to ignore in place during sync delete", - repoRes: map[string]resource.Resource{ - "res1": mockResourceWithoutIgnorePolicy("namespace", "ns1", "ns1"), - "res2": mockResourceWithoutIgnorePolicy("namespace", "ns2", "ns2"), - "res3": mockResourceWithoutIgnorePolicy("namespace", "ns3", "ns3"), - "res4": mockResourceWithoutIgnorePolicy("deployment", "ns1", "d1"), - "res5": mockResourceWithoutIgnorePolicy("deployment", "ns2", "d2"), - "res6": mockResourceWithoutIgnorePolicy("service", "ns3", "s1"), - }, - res: mockResourceWithIgnorePolicy("service", "ns1", "s2"), - expected: &cluster.SyncDef{}, - }, - { - msg: "No policy to ignore during sync delete", - repoRes: map[string]resource.Resource{ - "res1": mockResourceWithoutIgnorePolicy("namespace", "ns1", "ns1"), - "res2": mockResourceWithoutIgnorePolicy("namespace", "ns2", "ns2"), - "res3": mockResourceWithoutIgnorePolicy("namespace", "ns3", "ns3"), - "res4": mockResourceWithoutIgnorePolicy("deployment", "ns1", "d1"), - "res5": mockResourceWithoutIgnorePolicy("deployment", "ns2", "d2"), - "res6": mockResourceWithoutIgnorePolicy("service", "ns3", "s1"), - }, - res: mockResourceWithoutIgnorePolicy("service", "ns1", "s2"), - expected: &cluster.SyncDef{Actions: []cluster.SyncAction{cluster.SyncAction{Delete: mockResourceWithoutIgnorePolicy("service", "ns1", "s2")}}}, - }, - } - - logger := log.NewNopLogger() - for _, sc := range tests { - sync := &cluster.SyncDef{} - prepareSyncDelete(logger, sc.repoRes, sc.res.ResourceID().String(), sc.res, sync) - - if !reflect.DeepEqual(sc.expected, sync) { - t.Errorf("%s: expected %+v, got %+v\n", sc.msg, sc.expected, sync) - } - } -} - -func TestPrepareSyncApply(t *testing.T) { - var tests = []struct { - msg string - clusRes map[string]resource.Resource - res resource.Resource - expected *cluster.SyncDef - }{ - { - msg: "No repo resources provided during sync apply", - clusRes: map[string]resource.Resource{}, - res: mockResourceWithIgnorePolicy("service", "ns1", "s2"), - expected: &cluster.SyncDef{}, - }, - { - msg: "No policy to ignore in place during sync apply", - clusRes: map[string]resource.Resource{ - "res1": mockResourceWithoutIgnorePolicy("namespace", "ns1", "ns1"), - "res2": mockResourceWithoutIgnorePolicy("namespace", "ns2", "ns2"), - "res3": mockResourceWithoutIgnorePolicy("namespace", "ns3", "ns3"), - "res4": mockResourceWithoutIgnorePolicy("deployment", "ns1", "d1"), - "res5": mockResourceWithoutIgnorePolicy("deployment", "ns2", "d2"), - "res6": mockResourceWithoutIgnorePolicy("service", "ns3", "s1"), - }, - res: mockResourceWithIgnorePolicy("service", "ns1", "s2"), - expected: &cluster.SyncDef{}, - }, - { - msg: "No policy to ignore during sync apply", - clusRes: map[string]resource.Resource{ - "res1": mockResourceWithoutIgnorePolicy("namespace", "ns1", "ns1"), - "res2": mockResourceWithoutIgnorePolicy("namespace", "ns2", "ns2"), - "res3": mockResourceWithoutIgnorePolicy("namespace", "ns3", "ns3"), - "res4": mockResourceWithoutIgnorePolicy("deployment", "ns1", "d1"), - "res5": mockResourceWithoutIgnorePolicy("deployment", "ns2", "d2"), - "res6": mockResourceWithoutIgnorePolicy("service", "ns3", "s1"), - }, - res: mockResourceWithoutIgnorePolicy("service", "ns1", "s2"), - expected: &cluster.SyncDef{Actions: []cluster.SyncAction{cluster.SyncAction{Apply: mockResourceWithoutIgnorePolicy("service", "ns1", "s2")}}}, - }, - } - - logger := log.NewNopLogger() - for _, sc := range tests { - sync := &cluster.SyncDef{} - prepareSyncApply(logger, sc.clusRes, sc.res.ResourceID().String(), sc.res, sync) - - if !reflect.DeepEqual(sc.expected, sync) { - t.Errorf("%s: expected %+v, got %+v\n", sc.msg, sc.expected, sync) - } - } -} - // --- var gitconf = git.Config{ @@ -184,33 +49,18 @@ func setup(t *testing.T) (*git.Checkout, func()) { return gittest.Checkout(t) } -func execCommand(cmd string, args ...string) error { - c := exec.Command(cmd, args...) - fmt.Printf("exec: %s %s\n", cmd, strings.Join(args, " ")) - c.Stderr = os.Stderr - c.Stdout = os.Stdout - return c.Run() -} - // A cluster that keeps track of exactly what it's been told to apply // or delete and parrots it back when asked to Export. This is as -// mechanically simple as possible! +// mechanically simple as possible. -type syncCluster struct { - *cluster.Mock - resources map[string][]byte -} +type syncCluster struct{ resources map[string][]byte } -func (p *syncCluster) Sync(def cluster.SyncDef, l map[string]policy.Update, pl map[string]policy.Update) error { +func (p *syncCluster) Sync(def cluster.SyncDef) error { println("=== Syncing ===") - for _, action := range def.Actions { - if action.Delete != nil { - println("Deleting " + action.Delete.ResourceID().String()) - delete(p.resources, action.Delete.ResourceID().String()) - } - if action.Apply != nil { - println("Applying " + action.Apply.ResourceID().String()) - p.resources[action.Apply.ResourceID().String()] = action.Apply.Bytes() + for _, stack := range def.Stacks { + for _, resource := range stack.Resources { + println("Applying " + resource.ResourceID().String()) + p.resources[resource.ResourceID().String()] = resource.Bytes() } } println("=== Done syncing ===") @@ -237,7 +87,7 @@ func resourcesToStrings(resources map[string]resource.Resource) map[string]strin // Our invariant is that the model we can export from the cluster // should always reflect what's in git. So, let's check that. -func checkClusterMatchesFiles(t *testing.T, m cluster.Manifests, c cluster.Cluster, base string, dirs []string) { +func checkClusterMatchesFiles(t *testing.T, m cluster.Manifests, c Syncer, base string, dirs []string) { conf, err := c.Export() if err != nil { t.Fatal(err) @@ -251,10 +101,5 @@ func checkClusterMatchesFiles(t *testing.T, m cluster.Manifests, c cluster.Clust t.Fatal(err) } - expected := resourcesToStrings(files) - got := resourcesToStrings(resources) - - if !reflect.DeepEqual(expected, got) { - t.Errorf("expected:\n%#v\ngot:\n%#v", expected, got) - } + assert.Equal(t, resourcesToStrings(resources), resourcesToStrings(files)) } From 9fd2441a0689a98aee4122b269fe5606afb6bb9d Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Mon, 26 Nov 2018 13:53:11 +0000 Subject: [PATCH 06/24] Clear up use of annotations and labels for sync Although it would be cluster-neutral to use policies, the checksum and stack have to be implemented as distinct bits of metadata, since policies get encoded as annotations and we want to select (via the Kubernetes API) by labels. Therefore, deal with the annotation and label used, explicitly. This _does_ mean we don't have to create `resource.Resource`s when we want to examine what is in the cluster, though -- we can just pass around the things that we got from the Kubernetes API, with a shim for getting the metadata we care about. --- cluster/kubernetes/kubernetes.go | 116 ++++++++++++++++++++----------- cluster/kubernetes/sync.go | 26 +++---- cluster/kubernetes/sync_test.go | 83 ++++++++-------------- cluster/sync.go | 8 ++- daemon/loop.go | 4 +- policy/policy.go | 14 ++-- 6 files changed, 131 insertions(+), 120 deletions(-) diff --git a/cluster/kubernetes/kubernetes.go b/cluster/kubernetes/kubernetes.go index 15a8e1256..f6258668d 100644 --- a/cluster/kubernetes/kubernetes.go +++ b/cluster/kubernetes/kubernetes.go @@ -12,11 +12,11 @@ import ( "github.com/pkg/errors" kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" fhrclient "github.com/weaveworks/flux/integrations/client/clientset/versioned" - "github.com/weaveworks/flux/policy" "gopkg.in/yaml.v2" apiv1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" k8sclientdynamic "k8s.io/client-go/dynamic" k8sclient "k8s.io/client-go/kubernetes" @@ -37,6 +37,11 @@ type extendedClient struct { fluxHelmClient } +const ( + stackLabel = kresource.PolicyPrefix + "stack" + checksumAnnotation = kresource.PolicyPrefix + "stack_checksum" +) + // --- add-ons // Kubernetes has a mechanism of "Add-ons", whereby manifest files @@ -212,13 +217,13 @@ func applyMetadata(res resource.Resource, stack, checksum string) ([]byte, error if stack != "" { mixinLabels := map[string]string{} - mixinLabels[fmt.Sprintf("%s%s", kresource.PolicyPrefix, policy.Stack)] = stack + mixinLabels[stackLabel] = stack mixin["labels"] = mixinLabels } if checksum != "" { mixinAnnotations := map[string]string{} - mixinAnnotations[fmt.Sprintf("%s%s", kresource.PolicyPrefix, policy.StackChecksum)] = checksum + mixinAnnotations[checksumAnnotation] = checksum mixin["annotations"] = mixinAnnotations } @@ -233,26 +238,28 @@ func applyMetadata(res resource.Resource, stack, checksum string) ([]byte, error return bytes, nil } -// Sync performs the given actions on resources. Operations are -// asynchronous (applications may take a while to be processed), but -// serialised. +// Sync takes a definition of what should be running in the cluster, +// and attempts to make the cluster conform. An error return does not +// necessarily indicate complete failure; some resources may succeed +// in being synced, and some may fail (for example, they may be +// malformed). func (c *Cluster) Sync(spec cluster.SyncDef) error { logger := log.With(c.logger, "method", "Sync") - // Keep track of the checksum each resource gets, so we can - // compare them during garbage collection. + // Keep track of the checksum of each stack, so we can compare + // them during garbage collection. checksums := map[string]string{} cs := makeChangeSet() var errs cluster.SyncError for _, stack := range spec.Stacks { + checksums[stack.Name] = stack.Checksum for _, res := range stack.Resources { resBytes, err := applyMetadata(res, stack.Name, stack.Checksum) if err == nil { - checksums[res.ResourceID().String()] = stack.Checksum - cs.stage("apply", res, resBytes) + cs.stage("apply", res.ResourceID(), res.Source(), resBytes) } else { - errs = append(errs, cluster.ResourceError{Resource: res, Error: err}) + errs = append(errs, cluster.ResourceError{ResourceID: res.ResourceID(), Source: res.Source(), Error: err}) break } } @@ -269,27 +276,29 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { if c.GC { orphanedResources := makeChangeSet() - clusterResourceBytes, err := c.exportResourcesInStack() + clusterResources, err := c.getResourcesInStack() if err != nil { return errors.Wrap(err, "exporting resource defs from cluster for garbage collection") } - clusterResources, err := kresource.ParseMultidoc(clusterResourceBytes, "exported") - if err != nil { - return errors.Wrap(err, "parsing exported resources during garbage collection") - } for resourceID, res := range clusterResources { - expected := checksums[resourceID] // shall be "" if no such resource was applied earlier - actual, ok := res.Policy().Get(policy.StackChecksum) + stack := res.GetStack() + if stack == "" { + c.logger.Log("warning", "cluster resource with empty stack label; skipping", "resource", resourceID) + continue + } + + expected := checksums[stack] // shall be "" if no such resource was applied earlier + actual := res.GetChecksum() switch { - case !ok: - stack, _ := res.Policy().Get(policy.Stack) - c.logger.Log("warning", "cluster resource has stack but no checksum; skipping", "resource", resourceID, "stack", stack) + case expected == "": + c.logger.Log("info", "cluster resource was not applied this sync; deleting", "resource", resourceID, "actual", actual, "stack", stack) + orphanedResources.stage("delete", res.ResourceID(), "", res.Bytes()) case actual != expected: // including if checksum is "" - c.logger.Log("info", "cluster resource has out-of-date checksum; deleting", "resource", resourceID, "actual", actual, "expected", expected) - orphanedResources.stage("delete", res, res.Bytes()) + c.logger.Log("warning", "cluster resource has out-of-date checksum; deleting", "resource", resourceID, "actual", actual, "expected", expected) + orphanedResources.stage("delete", res.ResourceID(), "", res.Bytes()) default: - // all good; proceed + // the checksum is the same, indicating that it was applied earlier. Leave it alone. } } @@ -314,7 +323,7 @@ func (c *Cluster) setSyncErrors(errs cluster.SyncError) { defer c.muSyncErrors.Unlock() c.syncErrors = make(map[flux.ResourceID]error) for _, e := range errs { - c.syncErrors[e.ResourceID()] = e.Error + c.syncErrors[e.ResourceID] = e.Error } } @@ -379,16 +388,48 @@ func contains(a []string, x string) bool { return false } -// exportResourcesInStack collates all the resources that have a particular -// label (regardless of the value). -func (c *Cluster) exportResourcesInStack() ([]byte, error) { - labelName := fmt.Sprintf("%s%s", kresource.PolicyPrefix, policy.Stack) - var config bytes.Buffer +type kuberesource struct { + obj *unstructured.Unstructured +} + +func (r *kuberesource) ResourceID() flux.ResourceID { + ns, kind, name := r.obj.GetNamespace(), r.obj.GetKind(), r.obj.GetName() + return flux.MakeResourceID(ns, kind, name) +} +// Bytes returns a byte slice description +func (r *kuberesource) Bytes() []byte { + return []byte(fmt.Sprintf(` +apiVersion: %s +kind: %s +metadata: + namespace: %q + name: %q +`, r.obj.GetAPIVersion(), r.obj.GetKind(), r.obj.GetNamespace(), r.obj.GetName())) +} + +// GetChecksum returns the checksum recorded on the resource from +// Kubernetes, or an empty string if it's not present. +func (r *kuberesource) GetChecksum() string { + return r.obj.GetAnnotations()[checksumAnnotation] +} + +// GetStack returns the stack recorded on the the resource from +// Kubernetes, or an empty string if it's not present. +func (r *kuberesource) GetStack() string { + return r.obj.GetLabels()[stackLabel] +} + +// exportResourcesInStack collates all the resources that belong to a +// stack, i.e., were applied by flux. +func (c *Cluster) getResourcesInStack() (map[string]*kuberesource, error) { resources, err := c.client.coreClient.Discovery().ServerResources() if err != nil { return nil, err } + + result := map[string]*kuberesource{} + for _, resource := range resources { for _, apiResource := range resource.APIResources { verbs := apiResource.Verbs @@ -415,13 +456,13 @@ func (c *Cluster) exportResourcesInStack() ([]byte, error) { Resource: apiResource.Name, }) data, err := resourceClient.List(meta_v1.ListOptions{ - LabelSelector: labelName, // exists <> + LabelSelector: stackLabel, // means "has label <>" }) if err != nil { return nil, err } - for _, item := range data.Items { + for i, item := range data.Items { apiVersion := item.GetAPIVersion() kind := item.GetKind() @@ -432,18 +473,13 @@ func (c *Cluster) exportResourcesInStack() ([]byte, error) { } // TODO(michael) also exclude anything that has an ownerReference (that isn't "standard"?) - yamlBytes, err := k8syaml.Marshal(item.Object) - if err != nil { - return nil, err - } - config.WriteString("---\n") - config.Write(yamlBytes) - config.WriteString("\n") + res := &kuberesource{obj: &data.Items[i]} + result[res.ResourceID().String()] = res } } } - return config.Bytes(), nil + return result, nil } // kind & apiVersion must be passed separately as the object's TypeMeta is not populated diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index ae894e4de..6dbf28002 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -15,18 +15,14 @@ import ( "github.com/pkg/errors" "github.com/weaveworks/flux" "github.com/weaveworks/flux/cluster" - "github.com/weaveworks/flux/resource" ) // --- internal types for keeping track of syncing type applyObject struct { - OriginalResource resource.Resource - Payload []byte -} - -func (obj applyObject) Components() (namespace, kind, name string) { - return obj.OriginalResource.ResourceID().Components() + ResourceID flux.ResourceID + Source string + Payload []byte } type changeSet struct { @@ -37,8 +33,8 @@ func makeChangeSet() changeSet { return changeSet{objs: make(map[string][]applyObject)} } -func (c *changeSet) stage(cmd string, res resource.Resource, bytes []byte) { - c.objs[cmd] = append(c.objs[cmd], applyObject{res, bytes}) +func (c *changeSet) stage(cmd string, id flux.ResourceID, source string, bytes []byte) { + c.objs[cmd] = append(c.objs[cmd], applyObject{id, source, bytes}) } // Applier is something that will apply a changeset to the cluster. @@ -119,8 +115,8 @@ func (objs applyOrder) Swap(i, j int) { } func (objs applyOrder) Less(i, j int) bool { - _, ki, ni := objs[i].Components() - _, kj, nj := objs[j].Components() + _, ki, ni := objs[i].ResourceID.Components() + _, kj, nj := objs[j].ResourceID.Components() ranki, rankj := rankOfKind(ki), rankOfKind(kj) if ranki == rankj { return ni < nj @@ -141,7 +137,7 @@ func (c *Kubectl) apply(logger log.Logger, cs changeSet, errored map[flux.Resour multi = objs } else { for _, obj := range objs { - if _, ok := errored[obj.OriginalResource.ResourceID()]; ok { + if _, ok := errored[obj.ResourceID]; ok { // Resources that errored before shall be applied separately single = append(single, obj) } else { @@ -159,7 +155,11 @@ func (c *Kubectl) apply(logger log.Logger, cs changeSet, errored map[flux.Resour for _, obj := range single { r := bytes.NewReader(obj.Payload) if err := c.doCommand(logger, r, args...); err != nil { - errs = append(errs, cluster.ResourceError{obj.OriginalResource, err}) + errs = append(errs, cluster.ResourceError{ + ResourceID: obj.ResourceID, + Source: obj.Source, + Error: err, + }) } } } diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index ddd4fe51b..badc65a58 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -24,7 +24,6 @@ import ( "github.com/weaveworks/flux" "github.com/weaveworks/flux/cluster" fluxfake "github.com/weaveworks/flux/integrations/client/clientset/versioned/fake" - "github.com/weaveworks/flux/policy" "github.com/weaveworks/flux/sync" ) @@ -54,7 +53,7 @@ func fakeClients() extendedClient { // Assigned here, since this is _also_ used by the (fake) // discovery client therein, and ultimately by - // exportResourcesInStack since that uses the core clientset to + // getResourcesInStack since that uses the core clientset to // enumerate the namespaces. coreClient.Fake.Resources = apiResources @@ -92,7 +91,7 @@ func (a fakeApplier) apply(_ log.Logger, cs changeSet, errored map[flux.Resource a.commandRun = true var unstruct map[string]interface{} if err := yaml.Unmarshal(obj.Payload, &unstruct); err != nil { - errs = append(errs, cluster.ResourceError{obj.OriginalResource, err}) + errs = append(errs, cluster.ResourceError{obj.ResourceID, obj.Source, err}) return } res := &unstructured.Unstructured{Object: unstruct} @@ -114,12 +113,12 @@ func (a fakeApplier) apply(_ log.Logger, cs changeSet, errored map[flux.Resource _, err = dc.Update(res) //, &metav1.UpdateOptions{}) } if err != nil { - errs = append(errs, cluster.ResourceError{obj.OriginalResource, err}) + errs = append(errs, cluster.ResourceError{obj.ResourceID, obj.Source, err}) return } } else if cmd == "delete" { if err := dc.Delete(name, &metav1.DeleteOptions{}); err != nil { - errs = append(errs, cluster.ResourceError{obj.OriginalResource, err}) + errs = append(errs, cluster.ResourceError{obj.ResourceID, obj.Source, err}) return } } else { @@ -187,9 +186,7 @@ metadata: namespace: other ` - kube, _ := setup(t) - - test := func(defs, expectedAfterSync string) { + test := func(kube *Cluster, defs, expectedAfterSync string) { manifests := &Manifests{} resources, err := manifests.ParseManifests([]byte(defs)) if err != nil { @@ -200,19 +197,15 @@ metadata: if err != nil { t.Error(err) } - // Now check that the resources were created - exported, err := kube.exportResourcesInStack() - if err != nil { - t.Fatal(err) - } - resources0, err := manifests.ParseManifests([]byte(expectedAfterSync)) if err != nil { panic(err) } - resources1, err := manifests.ParseManifests([]byte(exported)) + + // Now check that the resources were created + resources1, err := kube.getResourcesInStack() if err != nil { - panic(err) + t.Fatal(err) } for id := range resources1 { @@ -227,54 +220,36 @@ metadata: } } - // without GC on, resources persist if they are not mentioned in subsequent syncs. - test(defs1, defs1) - test(defs1+defs2, defs1+defs2) - test(defs3, defs1+defs2+defs3) - - // Now with GC switched on. That means if we don't include a - // resource in a sync, it should be deleted. - kube.GC = true - test(defs2+defs3, defs3+defs2) + t.Run("sync adds and GCs resources", func(t *testing.T) { + kube, _ := setup(t) + + // without GC on, resources persist if they are not mentioned in subsequent syncs. + test(kube, "", "") + test(kube, defs1, defs1) + test(kube, defs1+defs2, defs1+defs2) + test(kube, defs3, defs1+defs2+defs3) + + // Now with GC switched on. That means if we don't include a + // resource in a sync, it should be deleted. + kube.GC = true + test(kube, defs2+defs3, defs3+defs2) + test(kube, defs1+defs2, defs1+defs2) + test(kube, "", "") + }) } // ---- -type rsc struct { - id string - bytes []byte -} - -func (r rsc) ResourceID() flux.ResourceID { - return flux.MustParseResourceID(r.id) -} - -func (r rsc) Bytes() []byte { - return r.bytes -} - -func (r rsc) Policy() policy.Set { - return nil -} - -func (r rsc) Source() string { - return "test" -} - -func mkResource(kind, name string) rsc { - return rsc{id: "default:" + kind + "/" + name} -} - // TestApplyOrder checks that applyOrder works as expected. func TestApplyOrder(t *testing.T) { objs := []applyObject{ - {OriginalResource: mkResource("Deployment", "deploy")}, - {OriginalResource: mkResource("Secret", "secret")}, - {OriginalResource: mkResource("Namespace", "namespace")}, + {ResourceID: flux.MakeResourceID("test", "Deployment", "deploy")}, + {ResourceID: flux.MakeResourceID("test", "Secret", "secret")}, + {ResourceID: flux.MakeResourceID("", "Namespace", "namespace")}, } sort.Sort(applyOrder(objs)) for i, name := range []string{"namespace", "secret", "deploy"} { - _, _, objName := objs[i].OriginalResource.ResourceID().Components() + _, _, objName := objs[i].ResourceID.Components() if objName != name { t.Errorf("Expected %q at position %d, got %q", name, i, objName) } diff --git a/cluster/sync.go b/cluster/sync.go index 7c9c9d889..bd4057e8f 100644 --- a/cluster/sync.go +++ b/cluster/sync.go @@ -3,6 +3,7 @@ package cluster import ( "strings" + "github.com/weaveworks/flux" "github.com/weaveworks/flux/resource" ) @@ -27,8 +28,9 @@ type SyncDef struct { } type ResourceError struct { - resource.Resource - Error error + ResourceID flux.ResourceID + Source string + Error error } type SyncError []ResourceError @@ -36,7 +38,7 @@ type SyncError []ResourceError func (err SyncError) Error() string { var errs []string for _, e := range err { - errs = append(errs, e.ResourceID().String()+": "+e.Error.Error()) + errs = append(errs, e.ResourceID.String()+": "+e.Error.Error()) } return strings.Join(errs, "; ") } diff --git a/daemon/loop.go b/daemon/loop.go index 745356c00..a145561d6 100644 --- a/daemon/loop.go +++ b/daemon/loop.go @@ -211,8 +211,8 @@ func (d *Daemon) doSync(logger log.Logger, lastKnownSyncTagRev *string, warnedAb case cluster.SyncError: for _, e := range syncerr { resourceErrors = append(resourceErrors, event.ResourceError{ - ID: e.ResourceID(), - Path: e.Source(), + ID: e.ResourceID, + Path: e.Source, Error: e.Error.Error(), }) } diff --git a/policy/policy.go b/policy/policy.go index f43d8d787..ef846858a 100644 --- a/policy/policy.go +++ b/policy/policy.go @@ -8,14 +8,12 @@ import ( ) const ( - Ignore = Policy("ignore") - Locked = Policy("locked") - LockedUser = Policy("locked_user") - LockedMsg = Policy("locked_msg") - Automated = Policy("automated") - TagAll = Policy("tag_all") - Stack = Policy("stack") - StackChecksum = Policy("stack_checksum") + Ignore = Policy("ignore") + Locked = Policy("locked") + LockedUser = Policy("locked_user") + LockedMsg = Policy("locked_msg") + Automated = Policy("automated") + TagAll = Policy("tag_all") ) // Policy is an string, denoting the current deployment policy of a service, From 9c15d5c827d9c2d789636e3a11bb0a51d84e7dfa Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Mon, 26 Nov 2018 17:20:12 +0000 Subject: [PATCH 07/24] Avoid deleting resources that did not apply OK It's possible that when we go to apply a particular manifest, there's some reason it doesn't work. In that case, we don't want to delete it, since it's clearly intended to still exist. In other words, anything that was represented by a manifest is kept, whether it was applied successfully or not. I still have a case checking if the checksum matches, so it can be reported in a log message. --- cluster/kubernetes/kubernetes.go | 31 +++++++++++----------- cluster/kubernetes/sync_test.go | 44 +++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/cluster/kubernetes/kubernetes.go b/cluster/kubernetes/kubernetes.go index f6258668d..863445bc4 100644 --- a/cluster/kubernetes/kubernetes.go +++ b/cluster/kubernetes/kubernetes.go @@ -246,17 +246,20 @@ func applyMetadata(res resource.Resource, stack, checksum string) ([]byte, error func (c *Cluster) Sync(spec cluster.SyncDef) error { logger := log.With(c.logger, "method", "Sync") + type checksum struct { + stack, sum string + } // Keep track of the checksum of each stack, so we can compare // them during garbage collection. - checksums := map[string]string{} + checksums := map[string]checksum{} cs := makeChangeSet() var errs cluster.SyncError for _, stack := range spec.Stacks { - checksums[stack.Name] = stack.Checksum for _, res := range stack.Resources { resBytes, err := applyMetadata(res, stack.Name, stack.Checksum) if err == nil { + checksums[res.ResourceID().String()] = checksum{stack.Name, stack.Checksum} cs.stage("apply", res.ResourceID(), res.Source(), resBytes) } else { errs = append(errs, cluster.ResourceError{ResourceID: res.ResourceID(), Source: res.Source(), Error: err}) @@ -282,23 +285,21 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { } for resourceID, res := range clusterResources { - stack := res.GetStack() - if stack == "" { - c.logger.Log("warning", "cluster resource with empty stack label; skipping", "resource", resourceID) - continue - } + actual := checksum{res.GetStack(), res.GetChecksum()} + expected, ok := checksums[resourceID] - expected := checksums[stack] // shall be "" if no such resource was applied earlier - actual := res.GetChecksum() switch { - case expected == "": - c.logger.Log("info", "cluster resource was not applied this sync; deleting", "resource", resourceID, "actual", actual, "stack", stack) - orphanedResources.stage("delete", res.ResourceID(), "", res.Bytes()) - case actual != expected: // including if checksum is "" - c.logger.Log("warning", "cluster resource has out-of-date checksum; deleting", "resource", resourceID, "actual", actual, "expected", expected) + case !ok: // was not recorded as having been staged for application + c.logger.Log("info", "cluster resource not in resources to be synced; deleting", "resource", resourceID) orphanedResources.stage("delete", res.ResourceID(), "", res.Bytes()) + case actual.stack == "": // the label has been removed, out of band (or due to a bug). Best to leave it. + c.logger.Log("warning", "cluster resource with empty stack label; skipping", "resource", resourceID) + continue + case actual != expected: + c.logger.Log("warning", "resource to be synced has not been updated; skipping", "resource", resourceID) + continue default: - // the checksum is the same, indicating that it was applied earlier. Leave it alone. + // The stack and checksum are the same, indicating that it was applied earlier. Leave it alone. } } diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index badc65a58..23a3a0d5e 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -1,6 +1,7 @@ package kubernetes import ( + "fmt" "sort" "strings" "testing" @@ -95,6 +96,14 @@ func (a fakeApplier) apply(_ log.Logger, cs changeSet, errored map[flux.Resource return } res := &unstructured.Unstructured{Object: unstruct} + + // This is a special case trapdoor, for testing failure to + // apply a resource. + if errStr := res.GetAnnotations()["error"]; errStr != "" { + errs = append(errs, cluster.ResourceError{obj.ResourceID, obj.Source, fmt.Errorf(errStr)}) + return + } + gvk := res.GetObjectKind().GroupVersionKind() gvr := schema.GroupVersionResource{Group: gvk.Group, Version: gvk.Version, Resource: strings.ToLower(gvk.Kind) + "s"} c := a.client.Resource(gvr) @@ -186,7 +195,7 @@ metadata: namespace: other ` - test := func(kube *Cluster, defs, expectedAfterSync string) { + test := func(t *testing.T, kube *Cluster, defs, expectedAfterSync string, expectErrors bool) { manifests := &Manifests{} resources, err := manifests.ParseManifests([]byte(defs)) if err != nil { @@ -194,7 +203,7 @@ metadata: } err = sync.Sync(log.NewNopLogger(), manifests, resources, kube) - if err != nil { + if !expectErrors && err != nil { t.Error(err) } resources0, err := manifests.ParseManifests([]byte(expectedAfterSync)) @@ -224,17 +233,34 @@ metadata: kube, _ := setup(t) // without GC on, resources persist if they are not mentioned in subsequent syncs. - test(kube, "", "") - test(kube, defs1, defs1) - test(kube, defs1+defs2, defs1+defs2) - test(kube, defs3, defs1+defs2+defs3) + test(t, kube, "", "", false) + test(t, kube, defs1, defs1, false) + test(t, kube, defs1+defs2, defs1+defs2, false) + test(t, kube, defs3, defs1+defs2+defs3, false) // Now with GC switched on. That means if we don't include a // resource in a sync, it should be deleted. kube.GC = true - test(kube, defs2+defs3, defs3+defs2) - test(kube, defs1+defs2, defs1+defs2) - test(kube, "", "") + test(t, kube, defs2+defs3, defs3+defs2, false) + test(t, kube, defs1+defs2, defs1+defs2, false) + test(t, kube, "", "", false) + }) + + t.Run("sync won't doesn't delete if apply failed", func(t *testing.T) { + kube, _ := setup(t) + kube.GC = true + + const defs1invalid = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: foobar + name: dep1 + annotations: + error: fail to apply this +` + test(t, kube, defs1, defs1, false) + test(t, kube, defs1invalid, defs1, true) }) } From 5810dd2b7fee3e1e38f37095a3b1a07219e211af Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Wed, 19 Dec 2018 18:01:48 +0000 Subject: [PATCH 08/24] Respect "ignore" when syncing and GCing Prior to chopping and changing how Sync works, and adding garbage collection, the "ignore" annotation was implemented by exporting the cluster resources (which only yielded namespaces and workloads!) and checking for each manifest in the repo whether it, or its namesake in the cluster, had the "ignore" annotation. I have reinstated this logic in the Sync method. If the cluster resource or repo manifest is marked with "ignore", the manifest will not be staged for application; and, if a cluster resource is marked with "ignore", it will not be deleted during GC. To check the cluster resources, we have to have _all_ of them, and not just those that were synced by flux in the past, thus `getResourcesBySelector` (which will get all resources, when given an empty selector). When it comes to GC, we only care about things that were previously synced by flux, so using `getResourcesInStack` is fine. --- cluster/kubernetes/kubernetes.go | 44 +++++- cluster/kubernetes/resource/resource.go | 8 +- cluster/kubernetes/sync_test.go | 202 ++++++++++++++++++++++-- daemon/loop.go | 2 +- sync/sync.go | 37 ++--- sync/sync_test.go | 9 +- 6 files changed, 250 insertions(+), 52 deletions(-) diff --git a/cluster/kubernetes/kubernetes.go b/cluster/kubernetes/kubernetes.go index 863445bc4..a76b34cd0 100644 --- a/cluster/kubernetes/kubernetes.go +++ b/cluster/kubernetes/kubernetes.go @@ -23,6 +23,7 @@ import ( "github.com/weaveworks/flux" "github.com/weaveworks/flux/cluster" + "github.com/weaveworks/flux/policy" "github.com/weaveworks/flux/resource" "github.com/weaveworks/flux/ssh" ) @@ -253,13 +254,29 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { // them during garbage collection. checksums := map[string]checksum{} + // NB we get all resources, since we care about leaving unsynced, + // _ignored_ resources alone. + clusterResources, err := c.getResourcesBySelector("") + if err != nil { + return errors.Wrap(err, "collating resources in cluster for sync") + } + cs := makeChangeSet() var errs cluster.SyncError for _, stack := range spec.Stacks { for _, res := range stack.Resources { + id := res.ResourceID().String() + // make a record of the checksum, whether we stage it to + // be applied or not, so that we don't delete it later. + checksums[id] = checksum{stack.Name, stack.Checksum} + if res.Policy().Has(policy.Ignore) { + continue + } + if cres, ok := clusterResources[id]; ok && cres.Policy().Has(policy.Ignore) { + continue + } resBytes, err := applyMetadata(res, stack.Name, stack.Checksum) if err == nil { - checksums[res.ResourceID().String()] = checksum{stack.Name, stack.Checksum} cs.stage("apply", res.ResourceID(), res.Source(), resBytes) } else { errs = append(errs, cluster.ResourceError{ResourceID: res.ResourceID(), Source: res.Source(), Error: err}) @@ -281,7 +298,7 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { clusterResources, err := c.getResourcesInStack() if err != nil { - return errors.Wrap(err, "exporting resource defs from cluster for garbage collection") + return errors.Wrap(err, "collating resources in cluster for calculating garbage collection") } for resourceID, res := range clusterResources { @@ -409,6 +426,10 @@ metadata: `, r.obj.GetAPIVersion(), r.obj.GetKind(), r.obj.GetNamespace(), r.obj.GetName())) } +func (r *kuberesource) Policy() policy.Set { + return kresource.PolicyFromAnnotations(r.obj.GetAnnotations()) +} + // GetChecksum returns the checksum recorded on the resource from // Kubernetes, or an empty string if it's not present. func (r *kuberesource) GetChecksum() string { @@ -421,9 +442,12 @@ func (r *kuberesource) GetStack() string { return r.obj.GetLabels()[stackLabel] } -// exportResourcesInStack collates all the resources that belong to a -// stack, i.e., were applied by flux. -func (c *Cluster) getResourcesInStack() (map[string]*kuberesource, error) { +func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesource, error) { + listOptions := meta_v1.ListOptions{} + if selector != "" { + listOptions.LabelSelector = selector + } + resources, err := c.client.coreClient.Discovery().ServerResources() if err != nil { return nil, err @@ -456,9 +480,7 @@ func (c *Cluster) getResourcesInStack() (map[string]*kuberesource, error) { Version: version, Resource: apiResource.Name, }) - data, err := resourceClient.List(meta_v1.ListOptions{ - LabelSelector: stackLabel, // means "has label <>" - }) + data, err := resourceClient.List(listOptions) if err != nil { return nil, err } @@ -483,6 +505,12 @@ func (c *Cluster) getResourcesInStack() (map[string]*kuberesource, error) { return result, nil } +// exportResourcesInStack collates all the resources that belong to a +// stack, i.e., were applied by flux. +func (c *Cluster) getResourcesInStack() (map[string]*kuberesource, error) { + return c.getResourcesBySelector(stackLabel) // means "has label <>" +} + // kind & apiVersion must be passed separately as the object's TypeMeta is not populated func appendYAML(buffer *bytes.Buffer, apiVersion, kind string, object interface{}) error { yamlBytes, err := k8syaml.Marshal(object) diff --git a/cluster/kubernetes/resource/resource.go b/cluster/kubernetes/resource/resource.go index f73d90b67..0d18f5ad1 100644 --- a/cluster/kubernetes/resource/resource.go +++ b/cluster/kubernetes/resource/resource.go @@ -43,9 +43,9 @@ func (o *baseObject) debyte() { o.bytes = nil } -func (o baseObject) Policy() policy.Set { +func PolicyFromAnnotations(annotations map[string]string) policy.Set { set := policy.Set{} - for k, v := range o.Meta.Annotations { + for k, v := range annotations { if strings.HasPrefix(k, PolicyPrefix) { p := strings.TrimPrefix(k, PolicyPrefix) if v == "true" { @@ -58,6 +58,10 @@ func (o baseObject) Policy() policy.Set { return set } +func (o baseObject) Policy() policy.Set { + return PolicyFromAnnotations(o.Meta.Annotations) +} + func (o baseObject) Source() string { return o.source } diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index 23a3a0d5e..c8b235195 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -19,6 +19,7 @@ import ( "k8s.io/client-go/dynamic" // dynamicfake "k8s.io/client-go/dynamic/fake" // k8sclient "k8s.io/client-go/kubernetes" + "github.com/stretchr/testify/assert" corefake "k8s.io/client-go/kubernetes/fake" k8s_testing "k8s.io/client-go/testing" @@ -62,7 +63,7 @@ func fakeClients() extendedClient { for _, fake := range []*k8s_testing.Fake{&coreClient.Fake, &fluxClient.Fake, &dynamicClient.Fake} { fake.PrependReactor("*", "*", func(action k8s_testing.Action) (bool, runtime.Object, error) { gvr := action.GetResource() - println("[DEBUG] action: ", action.GetVerb(), gvr.Group, gvr.Version, gvr.Resource) + println("[DEBUG] action:", action.GetVerb(), gvr.Group, gvr.Version, gvr.Resource) return false, nil, nil }) } @@ -85,6 +86,11 @@ type fakeApplier struct { commandRun bool } +func groupVersionResource(res *unstructured.Unstructured) schema.GroupVersionResource { + gvk := res.GetObjectKind().GroupVersionKind() + return schema.GroupVersionResource{Group: gvk.Group, Version: gvk.Version, Resource: strings.ToLower(gvk.Kind) + "s"} +} + func (a fakeApplier) apply(_ log.Logger, cs changeSet, errored map[flux.ResourceID]error) cluster.SyncError { var errs []cluster.ResourceError @@ -104,8 +110,7 @@ func (a fakeApplier) apply(_ log.Logger, cs changeSet, errored map[flux.Resource return } - gvk := res.GetObjectKind().GroupVersionKind() - gvr := schema.GroupVersionResource{Group: gvk.Group, Version: gvk.Version, Resource: strings.ToLower(gvk.Kind) + "s"} + gvr := groupVersionResource(res) c := a.client.Resource(gvr) var dc dynamic.ResourceInterface = c if ns := res.GetNamespace(); ns != "" { @@ -195,6 +200,21 @@ metadata: namespace: other ` + // checkSame is a check that a result returned from the cluster is + // the same as an expected. labels and annotations may be altered + // by the sync process; we'll look at the "spec" field as an + // indication of whether the resources are equivalent or not. + checkSame := func(t *testing.T, expected []byte, actual *unstructured.Unstructured) { + var expectedSpec struct{ Spec map[string]interface{} } + if err := yaml.Unmarshal(expected, &expectedSpec); err != nil { + t.Error(err) + return + } + if expectedSpec.Spec != nil { + assert.Equal(t, expectedSpec.Spec, actual.Object["spec"]) + } + } + test := func(t *testing.T, kube *Cluster, defs, expectedAfterSync string, expectErrors bool) { manifests := &Manifests{} resources, err := manifests.ParseManifests([]byte(defs)) @@ -202,30 +222,37 @@ metadata: t.Fatal(err) } - err = sync.Sync(log.NewNopLogger(), manifests, resources, kube) + err = sync.Sync(log.NewNopLogger(), resources, kube) if !expectErrors && err != nil { t.Error(err) } - resources0, err := manifests.ParseManifests([]byte(expectedAfterSync)) + expected, err := manifests.ParseManifests([]byte(expectedAfterSync)) if err != nil { panic(err) } // Now check that the resources were created - resources1, err := kube.getResourcesInStack() + actual, err := kube.getResourcesInStack() if err != nil { t.Fatal(err) } - for id := range resources1 { - if _, ok := resources0[id]; !ok { + for id := range actual { + if _, ok := expected[id]; !ok { t.Errorf("resource present after sync but not in resources applied: %q", id) + if j, err := yaml.Marshal(actual[id].obj); err == nil { + println(string(j)) + } + continue } + checkSame(t, expected[id].Bytes(), actual[id].obj) } - for id := range resources0 { - if _, ok := resources1[id]; !ok { + for id := range expected { + if _, ok := actual[id]; !ok { t.Errorf("resource supposed to be synced but not present: %q", id) } + // no need to compare values, since we already considered + // the intersection of actual and expected above. } } @@ -246,7 +273,7 @@ metadata: test(t, kube, "", "", false) }) - t.Run("sync won't doesn't delete if apply failed", func(t *testing.T) { + t.Run("sync won't delete if apply failed", func(t *testing.T) { kube, _ := setup(t) kube.GC = true @@ -262,6 +289,159 @@ metadata: test(t, kube, defs1, defs1, false) test(t, kube, defs1invalid, defs1, true) }) + + t.Run("sync doesn't apply or delete manifests marked with ignore", func(t *testing.T) { + kube, _ := setup(t) + kube.GC = true + + const dep1 = `--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: foobar + name: dep1 +spec: + metadata: + labels: {app: foo} +` + + const dep2 = `--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: foobar + name: dep2 + annotations: {flux.weave.works/ignore: "true"} +` + + // dep1 is created, but dep2 is ignored + test(t, kube, dep1+dep2, dep1, false) + + const dep1ignored = `--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: foobar + name: dep1 + annotations: + flux.weave.works/ignore: "true" +spec: + metadata: + labels: {app: bar} +` + // dep1 is not updated, but neither is it deleted + test(t, kube, dep1ignored+dep2, dep1, false) + }) + + t.Run("sync doesn't update a cluster resource marked with ignore", func(t *testing.T) { + const dep1 = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: foobar + name: dep1 +spec: + metadata: + labels: + app: original +` + kube, _ := setup(t) + // This just checks the starting assumption: dep1 exists in the cluster + test(t, kube, dep1, dep1, false) + + // Now we'll mark it as ignored _in the cluster_ (i.e., the + // equivalent of `kubectl annotate`) + dc := kube.client.dynamicClient + rc := dc.Resource(schema.GroupVersionResource{ + Group: "apps", + Version: "v1", + Resource: "deployments", + }) + res, err := rc.Namespace("foobar").Get("dep1", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + annots := res.GetAnnotations() + annots["flux.weave.works/ignore"] = "true" + res.SetAnnotations(annots) + if _, err = rc.Namespace("foobar").Update(res); err != nil { + t.Fatal(err) + } + + const mod1 = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: foobar + name: dep1 +spec: + metadata: + labels: + app: modified +` + // Check that dep1, which is marked ignore in the cluster, is + // neither updated or deleted + test(t, kube, mod1, dep1, false) + }) + + t.Run("sync doesn't update or delete a pre-existing resource marked with ignore", func(t *testing.T) { + kube, _ := setup(t) + + const existing = `--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: foobar + name: dep1 + annotations: {flux.weave.works/ignore: "true"} +spec: + metadata: + labels: {foo: original} +` + var dep1obj map[string]interface{} + err := yaml.Unmarshal([]byte(existing), &dep1obj) + assert.NoError(t, err) + dep1res := &unstructured.Unstructured{Object: dep1obj} + gvr := groupVersionResource(dep1res) + // Put the pre-existing resource in the cluster + dc := kube.client.dynamicClient.Resource(gvr).Namespace(dep1res.GetNamespace()) + _, err = dc.Create(dep1res) + assert.NoError(t, err) + + // Check that our resource-getting also sees the pre-existing resource + resources, err := kube.getResourcesBySelector("") + assert.NoError(t, err) + assert.Contains(t, resources, "foobar:deployment/dep1") + + // NB test checks the _synced_ resources, so this just asserts + // the precondition, that nothing is synced + test(t, kube, "", "", false) + + // .. but, our resource is still there. + r, err := dc.Get(dep1res.GetName(), metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, r) + + const update = `--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: foobar + name: dep1 +spec: + metadata: + labels: {foo: modified} +` + + // Check that it's not been synced (i.e., still not included in synced resources) + test(t, kube, update, "", false) + + // Check that it still exists, as created + r, err = dc.Get(dep1res.GetName(), metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, r) + checkSame(t, []byte(existing), r) + }) } // ---- diff --git a/daemon/loop.go b/daemon/loop.go index a145561d6..d84d366a9 100644 --- a/daemon/loop.go +++ b/daemon/loop.go @@ -205,7 +205,7 @@ func (d *Daemon) doSync(logger log.Logger, lastKnownSyncTagRev *string, warnedAb } var resourceErrors []event.ResourceError - if err := fluxsync.Sync(logger, d.Manifests, allResources, d.Cluster); err != nil { + if err := fluxsync.Sync(logger, allResources, d.Cluster); err != nil { logger.Log("err", err) switch syncerr := err.(type) { case cluster.SyncError: diff --git a/sync/sync.go b/sync/sync.go index 991cb1041..7d6f8ac51 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -6,7 +6,6 @@ import ( "sort" "github.com/go-kit/kit/log" - "github.com/pkg/errors" "github.com/weaveworks/flux/cluster" "github.com/weaveworks/flux/policy" @@ -15,28 +14,15 @@ import ( // Syncer has the methods we need to be able to compile and run a sync type Syncer interface { - // TODO(michael) this could be less leaky as `() -> map[string]resource.Resource` - Export() ([]byte, error) Sync(cluster.SyncDef) error } // Sync synchronises the cluster to the files under a directory. -func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resource.Resource, clus Syncer) error { - // Get a map of resources defined in the cluster - clusterBytes, err := clus.Export() - - if err != nil { - return errors.Wrap(err, "exporting resource defs from cluster") - } - clusterResources, err := m.ParseManifests(clusterBytes) - if err != nil { - return errors.Wrap(err, "parsing exported resources") - } - +func Sync(logger log.Logger, repoResources map[string]resource.Resource, clus Syncer) error { // TODO: multiple stack support. This will involve partitioning // the resources into disjoint maps, then passing each to // makeStack. - defaultStack := makeStack("default", repoResources, clusterResources, logger) + defaultStack := makeStack("default", repoResources, logger) sync := cluster.SyncDef{Stacks: []cluster.SyncStack{defaultStack}} if err := clus.Sync(sync); err != nil { @@ -45,7 +31,7 @@ func Sync(logger log.Logger, m cluster.Manifests, repoResources map[string]resou return nil } -func makeStack(name string, repoResources, clusterResources map[string]resource.Resource, logger log.Logger) cluster.SyncStack { +func makeStack(name string, repoResources map[string]resource.Resource, logger log.Logger) cluster.SyncStack { stack := cluster.SyncStack{Name: name} var resources []resource.Resource @@ -59,22 +45,17 @@ func makeStack(name string, repoResources, clusterResources map[string]resource. checksum := sha1.New() for _, id := range ids { res := repoResources[id] + resources = append(resources, res) if res.Policy().Has(policy.Ignore) { logger.Log("resource", res.ResourceID(), "ignore", "apply") continue } - // It may be ignored in the cluster, but it isn't in the repo; - // and we don't want what happens in the cluster to affect the - // checksum. + // Ignored resources are not included in the checksum; this + // means if you mark something as ignored, the checksum will + // come out differently. But the alternative is that adding + // ignored resources changes the checksum even though they are + // not intended to be created. checksum.Write(res.Bytes()) - - if cres, ok := clusterResources[id]; ok { - if cres.Policy().Has(policy.Ignore) { - logger.Log("resource", res.ResourceID(), "ignore", "apply") - continue - } - } - resources = append(resources, res) } stack.Resources = resources diff --git a/sync/sync_test.go b/sync/sync_test.go index a3f3f618a..6578a3cec 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -30,7 +30,7 @@ func TestSync(t *testing.T) { t.Fatal(err) } - if err := Sync(log.NewNopLogger(), manifests, resources, clus); err != nil { + if err := Sync(log.NewNopLogger(), resources, clus); err != nil { t.Fatal(err) } checkClusterMatchesFiles(t, manifests, clus, checkout.Dir(), dirs) @@ -85,9 +85,14 @@ func resourcesToStrings(resources map[string]resource.Resource) map[string]strin return res } +type SyncExporter interface { + Syncer + Export() ([]byte, error) +} + // Our invariant is that the model we can export from the cluster // should always reflect what's in git. So, let's check that. -func checkClusterMatchesFiles(t *testing.T, m cluster.Manifests, c Syncer, base string, dirs []string) { +func checkClusterMatchesFiles(t *testing.T, m cluster.Manifests, c SyncExporter, base string, dirs []string) { conf, err := c.Export() if err != nil { t.Fatal(err) From 6173f6b986c7f7214b9abde60157b5ee64617c49 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 20 Dec 2018 13:31:11 +0000 Subject: [PATCH 09/24] Move kubernetes Sync code into sync.go Just a rearrangement, no logic changes. --- cluster/kubernetes/kubernetes.go | 291 ++----------------------------- cluster/kubernetes/sync.go | 260 ++++++++++++++++++++++++++- 2 files changed, 277 insertions(+), 274 deletions(-) diff --git a/cluster/kubernetes/kubernetes.go b/cluster/kubernetes/kubernetes.go index a76b34cd0..074a038a0 100644 --- a/cluster/kubernetes/kubernetes.go +++ b/cluster/kubernetes/kubernetes.go @@ -3,28 +3,22 @@ package kubernetes import ( "bytes" "fmt" - "strings" "sync" k8syaml "github.com/ghodss/yaml" "github.com/go-kit/kit/log" - "github.com/imdario/mergo" "github.com/pkg/errors" - kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" - fhrclient "github.com/weaveworks/flux/integrations/client/clientset/versioned" - "gopkg.in/yaml.v2" apiv1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" k8sclientdynamic "k8s.io/client-go/dynamic" k8sclient "k8s.io/client-go/kubernetes" + kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" + fhrclient "github.com/weaveworks/flux/integrations/client/clientset/versioned" + "github.com/weaveworks/flux" "github.com/weaveworks/flux/cluster" - "github.com/weaveworks/flux/policy" - "github.com/weaveworks/flux/resource" "github.com/weaveworks/flux/ssh" ) @@ -38,11 +32,6 @@ type extendedClient struct { fluxHelmClient } -const ( - stackLabel = kresource.PolicyPrefix + "stack" - checksumAnnotation = kresource.PolicyPrefix + "stack_checksum" -) - // --- add-ons // Kubernetes has a mechanism of "Add-ons", whereby manifest files @@ -208,134 +197,6 @@ func (c *Cluster) AllControllers(namespace string) (res []cluster.Controller, er return allControllers, nil } -func applyMetadata(res resource.Resource, stack, checksum string) ([]byte, error) { - definition := map[interface{}]interface{}{} - if err := yaml.Unmarshal(res.Bytes(), &definition); err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("failed to parse yaml from %s", res.Source())) - } - - mixin := map[string]interface{}{} - - if stack != "" { - mixinLabels := map[string]string{} - mixinLabels[stackLabel] = stack - mixin["labels"] = mixinLabels - } - - if checksum != "" { - mixinAnnotations := map[string]string{} - mixinAnnotations[checksumAnnotation] = checksum - mixin["annotations"] = mixinAnnotations - } - - mergo.Merge(&definition, map[interface{}]interface{}{ - "metadata": mixin, - }) - - bytes, err := yaml.Marshal(definition) - if err != nil { - return nil, errors.Wrap(err, "failed to serialize yaml after applying metadata") - } - return bytes, nil -} - -// Sync takes a definition of what should be running in the cluster, -// and attempts to make the cluster conform. An error return does not -// necessarily indicate complete failure; some resources may succeed -// in being synced, and some may fail (for example, they may be -// malformed). -func (c *Cluster) Sync(spec cluster.SyncDef) error { - logger := log.With(c.logger, "method", "Sync") - - type checksum struct { - stack, sum string - } - // Keep track of the checksum of each stack, so we can compare - // them during garbage collection. - checksums := map[string]checksum{} - - // NB we get all resources, since we care about leaving unsynced, - // _ignored_ resources alone. - clusterResources, err := c.getResourcesBySelector("") - if err != nil { - return errors.Wrap(err, "collating resources in cluster for sync") - } - - cs := makeChangeSet() - var errs cluster.SyncError - for _, stack := range spec.Stacks { - for _, res := range stack.Resources { - id := res.ResourceID().String() - // make a record of the checksum, whether we stage it to - // be applied or not, so that we don't delete it later. - checksums[id] = checksum{stack.Name, stack.Checksum} - if res.Policy().Has(policy.Ignore) { - continue - } - if cres, ok := clusterResources[id]; ok && cres.Policy().Has(policy.Ignore) { - continue - } - resBytes, err := applyMetadata(res, stack.Name, stack.Checksum) - if err == nil { - cs.stage("apply", res.ResourceID(), res.Source(), resBytes) - } else { - errs = append(errs, cluster.ResourceError{ResourceID: res.ResourceID(), Source: res.Source(), Error: err}) - break - } - } - } - - c.mu.Lock() - defer c.mu.Unlock() - c.muSyncErrors.RLock() - if applyErrs := c.applier.apply(logger, cs, c.syncErrors); len(applyErrs) > 0 { - errs = append(errs, applyErrs...) - } - c.muSyncErrors.RUnlock() - - if c.GC { - orphanedResources := makeChangeSet() - - clusterResources, err := c.getResourcesInStack() - if err != nil { - return errors.Wrap(err, "collating resources in cluster for calculating garbage collection") - } - - for resourceID, res := range clusterResources { - actual := checksum{res.GetStack(), res.GetChecksum()} - expected, ok := checksums[resourceID] - - switch { - case !ok: // was not recorded as having been staged for application - c.logger.Log("info", "cluster resource not in resources to be synced; deleting", "resource", resourceID) - orphanedResources.stage("delete", res.ResourceID(), "", res.Bytes()) - case actual.stack == "": // the label has been removed, out of band (or due to a bug). Best to leave it. - c.logger.Log("warning", "cluster resource with empty stack label; skipping", "resource", resourceID) - continue - case actual != expected: - c.logger.Log("warning", "resource to be synced has not been updated; skipping", "resource", resourceID) - continue - default: - // The stack and checksum are the same, indicating that it was applied earlier. Leave it alone. - } - } - - if deleteErrs := c.applier.apply(logger, orphanedResources, nil); len(deleteErrs) > 0 { - errs = append(errs, deleteErrs...) - } - } - - // If `nil`, errs is a cluster.SyncError(nil) rather than error(nil), so it cannot be returned directly. - if errs == nil { - return nil - } - - // It is expected that Cluster.Sync is invoked with *all* resources. - // Otherwise it will override previously recorded sync errors. - c.setSyncErrors(errs) - return errs -} - func (c *Cluster) setSyncErrors(errs cluster.SyncError) { c.muSyncErrors.Lock() defer c.muSyncErrors.Unlock() @@ -397,136 +258,6 @@ func (c *Cluster) Export() ([]byte, error) { return config.Bytes(), nil } -func contains(a []string, x string) bool { - for _, n := range a { - if x == n { - return true - } - } - return false -} - -type kuberesource struct { - obj *unstructured.Unstructured -} - -func (r *kuberesource) ResourceID() flux.ResourceID { - ns, kind, name := r.obj.GetNamespace(), r.obj.GetKind(), r.obj.GetName() - return flux.MakeResourceID(ns, kind, name) -} - -// Bytes returns a byte slice description -func (r *kuberesource) Bytes() []byte { - return []byte(fmt.Sprintf(` -apiVersion: %s -kind: %s -metadata: - namespace: %q - name: %q -`, r.obj.GetAPIVersion(), r.obj.GetKind(), r.obj.GetNamespace(), r.obj.GetName())) -} - -func (r *kuberesource) Policy() policy.Set { - return kresource.PolicyFromAnnotations(r.obj.GetAnnotations()) -} - -// GetChecksum returns the checksum recorded on the resource from -// Kubernetes, or an empty string if it's not present. -func (r *kuberesource) GetChecksum() string { - return r.obj.GetAnnotations()[checksumAnnotation] -} - -// GetStack returns the stack recorded on the the resource from -// Kubernetes, or an empty string if it's not present. -func (r *kuberesource) GetStack() string { - return r.obj.GetLabels()[stackLabel] -} - -func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesource, error) { - listOptions := meta_v1.ListOptions{} - if selector != "" { - listOptions.LabelSelector = selector - } - - resources, err := c.client.coreClient.Discovery().ServerResources() - if err != nil { - return nil, err - } - - result := map[string]*kuberesource{} - - for _, resource := range resources { - for _, apiResource := range resource.APIResources { - verbs := apiResource.Verbs - // skip resources that can't be listed - if !contains(verbs, "list") { - continue - } - - // get group and version - var group, version string - groupVersion := resource.GroupVersion - if strings.Contains(groupVersion, "/") { - a := strings.SplitN(groupVersion, "/", 2) - group = a[0] - version = a[1] - } else { - group = "" - version = groupVersion - } - - resourceClient := c.client.dynamicClient.Resource(schema.GroupVersionResource{ - Group: group, - Version: version, - Resource: apiResource.Name, - }) - data, err := resourceClient.List(listOptions) - if err != nil { - return nil, err - } - - for i, item := range data.Items { - apiVersion := item.GetAPIVersion() - kind := item.GetKind() - - itemDesc := fmt.Sprintf("%s:%s", apiVersion, kind) - // https://github.com/kontena/k8s-client/blob/6e9a7ba1f03c255bd6f06e8724a1c7286b22e60f/lib/k8s/stack.rb#L17-L22 - if itemDesc == "v1:ComponentStatus" || itemDesc == "v1:Endpoints" { - continue - } - // TODO(michael) also exclude anything that has an ownerReference (that isn't "standard"?) - - res := &kuberesource{obj: &data.Items[i]} - result[res.ResourceID().String()] = res - } - } - } - - return result, nil -} - -// exportResourcesInStack collates all the resources that belong to a -// stack, i.e., were applied by flux. -func (c *Cluster) getResourcesInStack() (map[string]*kuberesource, error) { - return c.getResourcesBySelector(stackLabel) // means "has label <>" -} - -// kind & apiVersion must be passed separately as the object's TypeMeta is not populated -func appendYAML(buffer *bytes.Buffer, apiVersion, kind string, object interface{}) error { - yamlBytes, err := k8syaml.Marshal(object) - if err != nil { - return err - } - buffer.WriteString("---\n") - buffer.WriteString("apiVersion: ") - buffer.WriteString(apiVersion) - buffer.WriteString("\nkind: ") - buffer.WriteString(kind) - buffer.WriteString("\n") - buffer.Write(yamlBytes) - return nil -} - func (c *Cluster) PublicSSHKey(regenerate bool) (ssh.PublicKey, error) { if regenerate { if err := c.sshKeyRing.Regenerate(); err != nil { @@ -569,3 +300,19 @@ func (c *Cluster) getAllowedNamespaces() ([]apiv1.Namespace, error) { } return namespaces.Items, nil } + +// kind & apiVersion must be passed separately as the object's TypeMeta is not populated +func appendYAML(buffer *bytes.Buffer, apiVersion, kind string, object interface{}) error { + yamlBytes, err := k8syaml.Marshal(object) + if err != nil { + return err + } + buffer.WriteString("---\n") + buffer.WriteString("apiVersion: ") + buffer.WriteString(apiVersion) + buffer.WriteString("\nkind: ") + buffer.WriteString(kind) + buffer.WriteString("\n") + buffer.Write(yamlBytes) + return nil +} diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index 6dbf28002..8cb38e799 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -9,14 +9,270 @@ import ( "strings" "time" - rest "k8s.io/client-go/rest" - "github.com/go-kit/kit/log" + "github.com/imdario/mergo" "github.com/pkg/errors" + "gopkg.in/yaml.v2" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + rest "k8s.io/client-go/rest" + "github.com/weaveworks/flux" "github.com/weaveworks/flux/cluster" + kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" + "github.com/weaveworks/flux/policy" + "github.com/weaveworks/flux/resource" +) + +const ( + stackLabel = kresource.PolicyPrefix + "stack" + checksumAnnotation = kresource.PolicyPrefix + "stack_checksum" ) +// Sync takes a definition of what should be running in the cluster, +// and attempts to make the cluster conform. An error return does not +// necessarily indicate complete failure; some resources may succeed +// in being synced, and some may fail (for example, they may be +// malformed). +func (c *Cluster) Sync(spec cluster.SyncDef) error { + logger := log.With(c.logger, "method", "Sync") + + type checksum struct { + stack, sum string + } + // Keep track of the checksum of each stack, so we can compare + // them during garbage collection. + checksums := map[string]checksum{} + + // NB we get all resources, since we care about leaving unsynced, + // _ignored_ resources alone. + clusterResources, err := c.getResourcesBySelector("") + if err != nil { + return errors.Wrap(err, "collating resources in cluster for sync") + } + + cs := makeChangeSet() + var errs cluster.SyncError + for _, stack := range spec.Stacks { + for _, res := range stack.Resources { + id := res.ResourceID().String() + // make a record of the checksum, whether we stage it to + // be applied or not, so that we don't delete it later. + checksums[id] = checksum{stack.Name, stack.Checksum} + if res.Policy().Has(policy.Ignore) { + continue + } + if cres, ok := clusterResources[id]; ok && cres.Policy().Has(policy.Ignore) { + continue + } + resBytes, err := applyMetadata(res, stack.Name, stack.Checksum) + if err == nil { + cs.stage("apply", res.ResourceID(), res.Source(), resBytes) + } else { + errs = append(errs, cluster.ResourceError{ResourceID: res.ResourceID(), Source: res.Source(), Error: err}) + break + } + } + } + + c.mu.Lock() + defer c.mu.Unlock() + c.muSyncErrors.RLock() + if applyErrs := c.applier.apply(logger, cs, c.syncErrors); len(applyErrs) > 0 { + errs = append(errs, applyErrs...) + } + c.muSyncErrors.RUnlock() + + if c.GC { + orphanedResources := makeChangeSet() + + clusterResources, err := c.getResourcesInStack() + if err != nil { + return errors.Wrap(err, "collating resources in cluster for calculating garbage collection") + } + + for resourceID, res := range clusterResources { + actual := checksum{res.GetStack(), res.GetChecksum()} + expected, ok := checksums[resourceID] + + switch { + case !ok: // was not recorded as having been staged for application + c.logger.Log("info", "cluster resource not in resources to be synced; deleting", "resource", resourceID) + orphanedResources.stage("delete", res.ResourceID(), "", res.Bytes()) + case actual.stack == "": // the label has been removed, out of band (or due to a bug). Best to leave it. + c.logger.Log("warning", "cluster resource with empty stack label; skipping", "resource", resourceID) + continue + case actual != expected: + c.logger.Log("warning", "resource to be synced has not been updated; skipping", "resource", resourceID) + continue + default: + // The stack and checksum are the same, indicating that it was applied earlier. Leave it alone. + } + } + + if deleteErrs := c.applier.apply(logger, orphanedResources, nil); len(deleteErrs) > 0 { + errs = append(errs, deleteErrs...) + } + } + + // If `nil`, errs is a cluster.SyncError(nil) rather than error(nil), so it cannot be returned directly. + if errs == nil { + return nil + } + + // It is expected that Cluster.Sync is invoked with *all* resources. + // Otherwise it will override previously recorded sync errors. + c.setSyncErrors(errs) + return errs +} + +// --- internals in support of Sync + +type kuberesource struct { + obj *unstructured.Unstructured +} + +func (r *kuberesource) ResourceID() flux.ResourceID { + ns, kind, name := r.obj.GetNamespace(), r.obj.GetKind(), r.obj.GetName() + return flux.MakeResourceID(ns, kind, name) +} + +// Bytes returns a byte slice description +func (r *kuberesource) Bytes() []byte { + return []byte(fmt.Sprintf(` +apiVersion: %s +kind: %s +metadata: + namespace: %q + name: %q +`, r.obj.GetAPIVersion(), r.obj.GetKind(), r.obj.GetNamespace(), r.obj.GetName())) +} + +func (r *kuberesource) Policy() policy.Set { + return kresource.PolicyFromAnnotations(r.obj.GetAnnotations()) +} + +// GetChecksum returns the checksum recorded on the resource from +// Kubernetes, or an empty string if it's not present. +func (r *kuberesource) GetChecksum() string { + return r.obj.GetAnnotations()[checksumAnnotation] +} + +// GetStack returns the stack recorded on the the resource from +// Kubernetes, or an empty string if it's not present. +func (r *kuberesource) GetStack() string { + return r.obj.GetLabels()[stackLabel] +} + +func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesource, error) { + listOptions := meta_v1.ListOptions{} + if selector != "" { + listOptions.LabelSelector = selector + } + + resources, err := c.client.coreClient.Discovery().ServerResources() + if err != nil { + return nil, err + } + + result := map[string]*kuberesource{} + + contains := func(a []string, x string) bool { + for _, n := range a { + if x == n { + return true + } + } + return false + } + + for _, resource := range resources { + for _, apiResource := range resource.APIResources { + verbs := apiResource.Verbs + if !contains(verbs, "list") { + continue + } + + // get group and version + var group, version string + groupVersion := resource.GroupVersion + if strings.Contains(groupVersion, "/") { + a := strings.SplitN(groupVersion, "/", 2) + group = a[0] + version = a[1] + } else { + group = "" + version = groupVersion + } + + resourceClient := c.client.dynamicClient.Resource(schema.GroupVersionResource{ + Group: group, + Version: version, + Resource: apiResource.Name, + }) + data, err := resourceClient.List(listOptions) + if err != nil { + return nil, err + } + + for i, item := range data.Items { + apiVersion := item.GetAPIVersion() + kind := item.GetKind() + + itemDesc := fmt.Sprintf("%s:%s", apiVersion, kind) + // https://github.com/kontena/k8s-client/blob/6e9a7ba1f03c255bd6f06e8724a1c7286b22e60f/lib/k8s/stack.rb#L17-L22 + if itemDesc == "v1:ComponentStatus" || itemDesc == "v1:Endpoints" { + continue + } + // TODO(michael) also exclude anything that has an ownerReference (that isn't "standard"?) + + res := &kuberesource{obj: &data.Items[i]} + result[res.ResourceID().String()] = res + } + } + } + + return result, nil +} + +// exportResourcesInStack collates all the resources that belong to a +// stack, i.e., were applied by flux. +func (c *Cluster) getResourcesInStack() (map[string]*kuberesource, error) { + return c.getResourcesBySelector(stackLabel) // means "has label <>" +} + +func applyMetadata(res resource.Resource, stack, checksum string) ([]byte, error) { + definition := map[interface{}]interface{}{} + if err := yaml.Unmarshal(res.Bytes(), &definition); err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("failed to parse yaml from %s", res.Source())) + } + + mixin := map[string]interface{}{} + + if stack != "" { + mixinLabels := map[string]string{} + mixinLabels[stackLabel] = stack + mixin["labels"] = mixinLabels + } + + if checksum != "" { + mixinAnnotations := map[string]string{} + mixinAnnotations[checksumAnnotation] = checksum + mixin["annotations"] = mixinAnnotations + } + + mergo.Merge(&definition, map[interface{}]interface{}{ + "metadata": mixin, + }) + + bytes, err := yaml.Marshal(definition) + if err != nil { + return nil, errors.Wrap(err, "failed to serialize yaml after applying metadata") + } + return bytes, nil +} + // --- internal types for keeping track of syncing type applyObject struct { From b26ad33c8e6f203aa081a33cfa51a77e77f3c34c Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 20 Dec 2018 13:53:06 +0000 Subject: [PATCH 10/24] Use checksum per resource In using a checksum for a whole stack (which at present is everything in the repo), it makes the syncing very sensitive to changes. Since it would mostly be just changing annotations, I don't think this is a problem for correctness, but it will keep Kubernetes busy, updating annotations every time anything changes. It's not expensive to calculate the checksum for each resource, as it goes past. We don't actually use it for change detection -- it's only consulted when it comes to garbage collect items, to check if it was updated earlier (and if not, a warning is issued). --- cluster/kubernetes/kubernetes.go | 4 +--- cluster/kubernetes/sync.go | 8 ++++++-- cluster/sync.go | 1 - sync/sync.go | 23 +---------------------- 4 files changed, 8 insertions(+), 28 deletions(-) diff --git a/cluster/kubernetes/kubernetes.go b/cluster/kubernetes/kubernetes.go index 074a038a0..50dd04cc0 100644 --- a/cluster/kubernetes/kubernetes.go +++ b/cluster/kubernetes/kubernetes.go @@ -14,11 +14,9 @@ import ( k8sclientdynamic "k8s.io/client-go/dynamic" k8sclient "k8s.io/client-go/kubernetes" - kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" - fhrclient "github.com/weaveworks/flux/integrations/client/clientset/versioned" - "github.com/weaveworks/flux" "github.com/weaveworks/flux/cluster" + fhrclient "github.com/weaveworks/flux/integrations/client/clientset/versioned" "github.com/weaveworks/flux/ssh" ) diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index 8cb38e799..baf7472b8 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -2,6 +2,8 @@ package kubernetes import ( "bytes" + "crypto/sha1" + "encoding/hex" "fmt" "io" "os/exec" @@ -59,14 +61,16 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { id := res.ResourceID().String() // make a record of the checksum, whether we stage it to // be applied or not, so that we don't delete it later. - checksums[id] = checksum{stack.Name, stack.Checksum} + csum := sha1.Sum(res.Bytes()) + checkHex := hex.EncodeToString(csum[:]) + checksums[id] = checksum{stack.Name, checkHex} if res.Policy().Has(policy.Ignore) { continue } if cres, ok := clusterResources[id]; ok && cres.Policy().Has(policy.Ignore) { continue } - resBytes, err := applyMetadata(res, stack.Name, stack.Checksum) + resBytes, err := applyMetadata(res, stack.Name, checkHex) if err == nil { cs.stage("apply", res.ResourceID(), res.Source(), resBytes) } else { diff --git a/cluster/sync.go b/cluster/sync.go index bd4057e8f..6588bf0fe 100644 --- a/cluster/sync.go +++ b/cluster/sync.go @@ -18,7 +18,6 @@ import ( // it involves examining each resource individually). type SyncStack struct { Name string - Checksum string Resources []resource.Resource } diff --git a/sync/sync.go b/sync/sync.go index 7d6f8ac51..08313adf1 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -1,10 +1,6 @@ package sync import ( - "crypto/sha1" - "encoding/hex" - "sort" - "github.com/go-kit/kit/log" "github.com/weaveworks/flux/cluster" @@ -34,31 +30,14 @@ func Sync(logger log.Logger, repoResources map[string]resource.Resource, clus Sy func makeStack(name string, repoResources map[string]resource.Resource, logger log.Logger) cluster.SyncStack { stack := cluster.SyncStack{Name: name} var resources []resource.Resource - - // To get a stable checksum, we have to sort the resources. - var ids []string - for id, _ := range repoResources { - ids = append(ids, id) - } - sort.Strings(ids) - - checksum := sha1.New() - for _, id := range ids { - res := repoResources[id] + for _, res := range repoResources { resources = append(resources, res) if res.Policy().Has(policy.Ignore) { logger.Log("resource", res.ResourceID(), "ignore", "apply") continue } - // Ignored resources are not included in the checksum; this - // means if you mark something as ignored, the checksum will - // come out differently. But the alternative is that adding - // ignored resources changes the checksum even though they are - // not intended to be created. - checksum.Write(res.Bytes()) } stack.Resources = resources - stack.Checksum = hex.EncodeToString(checksum.Sum(nil)) return stack } From d52f7994d2cf7b0571b50c51c6062159df361018 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 10 Jan 2019 12:09:40 +0000 Subject: [PATCH 11/24] Rationalise logging of sync ignore annotation There's no particular reason to log "ignore" annotations when constructing the stack to be synced, since we'll include them anyway, and they will get logged later. This commit also makes the log messages when ignoring things to sync to be more like other informational messages. --- cluster/kubernetes/sync.go | 2 ++ cluster/kubernetes/sync_test.go | 2 +- daemon/loop.go | 2 +- sync/sync.go | 13 +++---------- sync/sync_test.go | 3 +-- 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index baf7472b8..b599b8ec4 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -65,9 +65,11 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { checkHex := hex.EncodeToString(csum[:]) checksums[id] = checksum{stack.Name, checkHex} if res.Policy().Has(policy.Ignore) { + logger.Log("info", "not applying resource; ignore annotation in file", "resource", res.ResourceID(), "source", res.Source()) continue } if cres, ok := clusterResources[id]; ok && cres.Policy().Has(policy.Ignore) { + logger.Log("info", "not applying resource; ignore annotation in cluster resource", "resource", cres.ResourceID()) continue } resBytes, err := applyMetadata(res, stack.Name, checkHex) diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index c8b235195..6b34d3022 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -222,7 +222,7 @@ metadata: t.Fatal(err) } - err = sync.Sync(log.NewNopLogger(), resources, kube) + err = sync.Sync(resources, kube) if !expectErrors && err != nil { t.Error(err) } diff --git a/daemon/loop.go b/daemon/loop.go index d84d366a9..229df7350 100644 --- a/daemon/loop.go +++ b/daemon/loop.go @@ -205,7 +205,7 @@ func (d *Daemon) doSync(logger log.Logger, lastKnownSyncTagRev *string, warnedAb } var resourceErrors []event.ResourceError - if err := fluxsync.Sync(logger, allResources, d.Cluster); err != nil { + if err := fluxsync.Sync(allResources, d.Cluster); err != nil { logger.Log("err", err) switch syncerr := err.(type) { case cluster.SyncError: diff --git a/sync/sync.go b/sync/sync.go index 08313adf1..0c1688238 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -1,10 +1,7 @@ package sync import ( - "github.com/go-kit/kit/log" - "github.com/weaveworks/flux/cluster" - "github.com/weaveworks/flux/policy" "github.com/weaveworks/flux/resource" ) @@ -14,11 +11,11 @@ type Syncer interface { } // Sync synchronises the cluster to the files under a directory. -func Sync(logger log.Logger, repoResources map[string]resource.Resource, clus Syncer) error { +func Sync(repoResources map[string]resource.Resource, clus Syncer) error { // TODO: multiple stack support. This will involve partitioning // the resources into disjoint maps, then passing each to // makeStack. - defaultStack := makeStack("default", repoResources, logger) + defaultStack := makeStack("default", repoResources) sync := cluster.SyncDef{Stacks: []cluster.SyncStack{defaultStack}} if err := clus.Sync(sync); err != nil { @@ -27,15 +24,11 @@ func Sync(logger log.Logger, repoResources map[string]resource.Resource, clus Sy return nil } -func makeStack(name string, repoResources map[string]resource.Resource, logger log.Logger) cluster.SyncStack { +func makeStack(name string, repoResources map[string]resource.Resource) cluster.SyncStack { stack := cluster.SyncStack{Name: name} var resources []resource.Resource for _, res := range repoResources { resources = append(resources, res) - if res.Policy().Has(policy.Ignore) { - logger.Log("resource", res.ResourceID(), "ignore", "apply") - continue - } } stack.Resources = resources diff --git a/sync/sync_test.go b/sync/sync_test.go index 6578a3cec..c424d226c 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -4,7 +4,6 @@ import ( "bytes" "testing" - "github.com/go-kit/kit/log" "github.com/stretchr/testify/assert" "github.com/weaveworks/flux/cluster" @@ -30,7 +29,7 @@ func TestSync(t *testing.T) { t.Fatal(err) } - if err := Sync(log.NewNopLogger(), resources, clus); err != nil { + if err := Sync(resources, clus); err != nil { t.Fatal(err) } checkClusterMatchesFiles(t, manifests, clus, checkout.Dir(), dirs) From 1ed6e2330afac5f045f8e1fc8d9d552f8f3a8a94 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Mon, 14 Jan 2019 15:58:28 +0000 Subject: [PATCH 12/24] Remove Manifests#ParseManifests Since we're no longer relying on Export when syncing (which was always a little circuitous), there's no need for ParseManifests. Where it was used for testing, it's OK to use the Kubernetes-specific procedure instead (cluster/kubernetes/resource#ParseMultidoc), or avoid the Export->ParseManifests sequence altogether and treat things as opaque strings/bytes. --- cluster/kubernetes/manifests.go | 4 ---- cluster/kubernetes/sync_test.go | 6 +++--- cluster/manifests.go | 2 -- cluster/mock.go | 5 ----- daemon/daemon_test.go | 12 +++-------- daemon/loop_test.go | 4 ---- sync/sync_test.go | 36 ++++++--------------------------- 7 files changed, 12 insertions(+), 57 deletions(-) diff --git a/cluster/kubernetes/manifests.go b/cluster/kubernetes/manifests.go index 18c424161..53df474a3 100644 --- a/cluster/kubernetes/manifests.go +++ b/cluster/kubernetes/manifests.go @@ -14,10 +14,6 @@ func (c *Manifests) LoadManifests(base string, paths []string) (map[string]resou return kresource.Load(base, paths) } -func (c *Manifests) ParseManifests(allDefs []byte) (map[string]resource.Resource, error) { - return kresource.ParseMultidoc(allDefs, "exported") -} - func (c *Manifests) UpdateImage(def []byte, id flux.ResourceID, container string, image image.Ref) ([]byte, error) { return updatePodController(def, id, container, image) } diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index 6b34d3022..657fcefa0 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -25,6 +25,7 @@ import ( "github.com/weaveworks/flux" "github.com/weaveworks/flux/cluster" + kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" fluxfake "github.com/weaveworks/flux/integrations/client/clientset/versioned/fake" "github.com/weaveworks/flux/sync" ) @@ -216,8 +217,7 @@ metadata: } test := func(t *testing.T, kube *Cluster, defs, expectedAfterSync string, expectErrors bool) { - manifests := &Manifests{} - resources, err := manifests.ParseManifests([]byte(defs)) + resources, err := kresource.ParseMultidoc([]byte(defs), "before") if err != nil { t.Fatal(err) } @@ -226,7 +226,7 @@ metadata: if !expectErrors && err != nil { t.Error(err) } - expected, err := manifests.ParseManifests([]byte(expectedAfterSync)) + expected, err := kresource.ParseMultidoc([]byte(expectedAfterSync), "after") if err != nil { panic(err) } diff --git a/cluster/manifests.go b/cluster/manifests.go index 8314dfe70..fe74a4863 100644 --- a/cluster/manifests.go +++ b/cluster/manifests.go @@ -31,8 +31,6 @@ type Manifests interface { // supplied as absolute paths to directories or files; at least // one path should be supplied, even if it is the same as `baseDir`. LoadManifests(baseDir string, paths []string) (map[string]resource.Resource, error) - // Parse the manifests given in an exported blob - ParseManifests([]byte) (map[string]resource.Resource, error) // UpdatePolicies modifies a manifest to apply the policy update specified UpdatePolicies([]byte, flux.ResourceID, policy.Update) ([]byte, error) } diff --git a/cluster/mock.go b/cluster/mock.go index 4943bce44..629776238 100644 --- a/cluster/mock.go +++ b/cluster/mock.go @@ -18,7 +18,6 @@ type Mock struct { PublicSSHKeyFunc func(regenerate bool) (ssh.PublicKey, error) UpdateImageFunc func(def []byte, id flux.ResourceID, container string, newImageID image.Ref) ([]byte, error) LoadManifestsFunc func(base string, paths []string) (map[string]resource.Resource, error) - ParseManifestsFunc func([]byte) (map[string]resource.Resource, error) UpdateManifestFunc func(path, resourceID string, f func(def []byte) ([]byte, error)) error UpdatePoliciesFunc func([]byte, flux.ResourceID, policy.Update) ([]byte, error) } @@ -55,10 +54,6 @@ func (m *Mock) LoadManifests(base string, paths []string) (map[string]resource.R return m.LoadManifestsFunc(base, paths) } -func (m *Mock) ParseManifests(def []byte) (map[string]resource.Resource, error) { - return m.ParseManifestsFunc(def) -} - func (m *Mock) UpdateManifest(path string, resourceID string, f func(def []byte) ([]byte, error)) error { return m.UpdateManifestFunc(path, resourceID, f) } diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go index d5d5454ab..6dd2cc46b 100644 --- a/daemon/daemon_test.go +++ b/daemon/daemon_test.go @@ -289,7 +289,7 @@ func TestDaemon_ListImagesWithOptions(t *testing.T) { { name: "Override container field selection", opts: v10.ListImagesOptions{ - Spec: specAll, + Spec: specAll, OverrideContainerFields: []string{"Name", "Current", "NewAvailableImagesCount"}, }, expectedImages: []v6.ImageStatus{ @@ -319,7 +319,7 @@ func TestDaemon_ListImagesWithOptions(t *testing.T) { { name: "Override container field selection with invalid field", opts: v10.ListImagesOptions{ - Spec: specAll, + Spec: specAll, OverrideContainerFields: []string{"InvalidField"}, }, expectedImages: nil, @@ -648,9 +648,6 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *cluster.Mock, *mockEven } k8s.ExportFunc = func() ([]byte, error) { return testBytes, nil } k8s.LoadManifestsFunc = kresource.Load - k8s.ParseManifestsFunc = func(allDefs []byte) (map[string]resource.Resource, error) { - return kresource.ParseMultidoc(allDefs, "test") - } k8s.PingFunc = func() error { return nil } k8s.SomeServicesFunc = func([]flux.ResourceID) ([]cluster.Controller, error) { return []cluster.Controller{ @@ -823,10 +820,7 @@ func (w *wait) ForImageTag(t *testing.T, d *Daemon, service, container, tag stri defer co.Clean() dirs := co.ManifestDirs() - m, err := d.Manifests.LoadManifests(co.Dir(), dirs) - assert.NoError(t, err) - - resources, err := d.Manifests.ParseManifests(m[service].Bytes()) + resources, err := d.Manifests.LoadManifests(co.Dir(), dirs) assert.NoError(t, err) workload, ok := resources[service].(resource.Workload) diff --git a/daemon/loop_test.go b/daemon/loop_test.go index 810763bb0..05da7692d 100644 --- a/daemon/loop_test.go +++ b/daemon/loop_test.go @@ -22,7 +22,6 @@ import ( "github.com/weaveworks/flux/git/gittest" "github.com/weaveworks/flux/job" registryMock "github.com/weaveworks/flux/registry/mock" - "github.com/weaveworks/flux/resource" ) const ( @@ -43,9 +42,6 @@ func daemon(t *testing.T) (*Daemon, func()) { k8s = &cluster.Mock{} k8s.LoadManifestsFunc = kresource.Load - k8s.ParseManifestsFunc = func(allDefs []byte) (map[string]resource.Resource, error) { - return kresource.ParseMultidoc(allDefs, "exported") - } k8s.ExportFunc = func() ([]byte, error) { return nil, nil } events = &mockEventWriter{} diff --git a/sync/sync_test.go b/sync/sync_test.go index c424d226c..3821ca303 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -1,7 +1,6 @@ package sync import ( - "bytes" "testing" "github.com/stretchr/testify/assert" @@ -21,7 +20,7 @@ func TestSync(t *testing.T) { // Start with nothing running. We should be told to apply all the things. manifests := &kubernetes.Manifests{} - clus := &syncCluster{map[string][]byte{}} + clus := &syncCluster{map[string]string{}} dirs := checkout.ManifestDirs() resources, err := manifests.LoadManifests(checkout.Dir(), dirs) @@ -32,7 +31,7 @@ func TestSync(t *testing.T) { if err := Sync(resources, clus); err != nil { t.Fatal(err) } - checkClusterMatchesFiles(t, manifests, clus, checkout.Dir(), dirs) + checkClusterMatchesFiles(t, manifests, clus.resources, checkout.Dir(), dirs) } // --- @@ -52,30 +51,20 @@ func setup(t *testing.T) (*git.Checkout, func()) { // or delete and parrots it back when asked to Export. This is as // mechanically simple as possible. -type syncCluster struct{ resources map[string][]byte } +type syncCluster struct{ resources map[string]string } func (p *syncCluster) Sync(def cluster.SyncDef) error { println("=== Syncing ===") for _, stack := range def.Stacks { for _, resource := range stack.Resources { println("Applying " + resource.ResourceID().String()) - p.resources[resource.ResourceID().String()] = resource.Bytes() + p.resources[resource.ResourceID().String()] = string(resource.Bytes()) } } println("=== Done syncing ===") return nil } -func (p *syncCluster) Export() ([]byte, error) { - // We need a response for Export, which is supposed to supply the - // entire configuration as a lump of bytes. - var configs [][]byte - for _, config := range p.resources { - configs = append(configs, config) - } - return bytes.Join(configs, []byte("\n---\n")), nil -} - func resourcesToStrings(resources map[string]resource.Resource) map[string]string { res := map[string]string{} for k, r := range resources { @@ -84,26 +73,13 @@ func resourcesToStrings(resources map[string]resource.Resource) map[string]strin return res } -type SyncExporter interface { - Syncer - Export() ([]byte, error) -} - // Our invariant is that the model we can export from the cluster // should always reflect what's in git. So, let's check that. -func checkClusterMatchesFiles(t *testing.T, m cluster.Manifests, c SyncExporter, base string, dirs []string) { - conf, err := c.Export() - if err != nil { - t.Fatal(err) - } - resources, err := m.ParseManifests(conf) - if err != nil { - t.Fatal(err) - } +func checkClusterMatchesFiles(t *testing.T, m cluster.Manifests, resources map[string]string, base string, dirs []string) { files, err := m.LoadManifests(base, dirs) if err != nil { t.Fatal(err) } - assert.Equal(t, resourcesToStrings(resources), resourcesToStrings(files)) + assert.Equal(t, resources, resourcesToStrings(files)) } From a6238ba6c6c34cf6240011f31b99b9ec143c25d1 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Wed, 16 Jan 2019 17:33:44 +0000 Subject: [PATCH 13/24] Account for namespace defaulting It's possible to have manifests for namespaces resources in your git repo, but not specify a namespace in them. In this case, `kubectl`, and thereby fluxd, will transparently assign a default namespace (either from kubeconfig, or if not given there, the default fallback of `"default"`). When we're syncing, we need to be able to know which namespace these resources will end up in, so we can identify them during garbage collection. The basic scheme is: in a few strategic places, let a fallback namespace be supplied, which is used for anything that doesn't have one. To avoid having to check which resources are supposed to have namespaces (e.g., namespaces themselves), use the fallback to identify everything that doesn't have one, either from a file or from the cluster. (The alternative is to figure out precisely which things need to have namespaces, e.g., using the discovery API, and give only those a default namespace. So long as the namespace is used _only_ to identify resources, giving everything a namespace is equivalent. And since lots of existing code assumes there's always a namespace, I leant that way.) Since the fallback namespace is decided by `kubectl`, that's what we use to discover it. --- cluster/kubernetes/kubernetes.go | 5 +- cluster/kubernetes/manifests.go | 3 +- cluster/kubernetes/policies.go | 2 +- .../resource/fluxhelmrelease_test.go | 14 +-- cluster/kubernetes/resource/load.go | 13 ++- cluster/kubernetes/resource/load_test.go | 30 +++-- cluster/kubernetes/resource/resource.go | 25 ++-- cluster/kubernetes/sync.go | 56 ++++++++- cluster/kubernetes/sync_test.go | 108 ++++++++++++++++-- daemon/daemon_test.go | 3 +- daemon/loop_test.go | 4 +- 11 files changed, 206 insertions(+), 57 deletions(-) diff --git a/cluster/kubernetes/kubernetes.go b/cluster/kubernetes/kubernetes.go index 50dd04cc0..0d0b21760 100644 --- a/cluster/kubernetes/kubernetes.go +++ b/cluster/kubernetes/kubernetes.go @@ -71,8 +71,9 @@ type Cluster struct { // Do garbage collection when syncing resources GC bool - client extendedClient - applier Applier + client extendedClient + applier Applier + version string // string response for the version command. logger log.Logger sshKeyRing ssh.KeyRing diff --git a/cluster/kubernetes/manifests.go b/cluster/kubernetes/manifests.go index 53df474a3..973197471 100644 --- a/cluster/kubernetes/manifests.go +++ b/cluster/kubernetes/manifests.go @@ -8,10 +8,11 @@ import ( ) type Manifests struct { + FallbackNamespace string } func (c *Manifests) LoadManifests(base string, paths []string) (map[string]resource.Resource, error) { - return kresource.Load(base, paths) + return kresource.Load(base, c.FallbackNamespace, paths) } func (c *Manifests) UpdateImage(def []byte, id flux.ResourceID, container string, image image.Ref) ([]byte, error) { diff --git a/cluster/kubernetes/policies.go b/cluster/kubernetes/policies.go index 79d51221c..5fbdf4b32 100644 --- a/cluster/kubernetes/policies.go +++ b/cluster/kubernetes/policies.go @@ -67,7 +67,7 @@ func extractAnnotations(def []byte) (map[string]string, error) { } func extractContainers(def []byte, id flux.ResourceID) ([]resource.Container, error) { - resources, err := kresource.ParseMultidoc(def, "stdin") + resources, err := kresource.ParseMultidoc(def, "", "stdin") if err != nil { return nil, err } diff --git a/cluster/kubernetes/resource/fluxhelmrelease_test.go b/cluster/kubernetes/resource/fluxhelmrelease_test.go index 8cd037e6e..4f2abbede 100644 --- a/cluster/kubernetes/resource/fluxhelmrelease_test.go +++ b/cluster/kubernetes/resource/fluxhelmrelease_test.go @@ -26,7 +26,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "test") + resources, err := ParseMultidoc([]byte(doc), "", "test") if err != nil { t.Fatal(err) } @@ -72,7 +72,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "test") + resources, err := ParseMultidoc([]byte(doc), "", "test") if err != nil { t.Fatal(err) } @@ -116,7 +116,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "test") + resources, err := ParseMultidoc([]byte(doc), "", "test") if err != nil { t.Fatal(err) } @@ -186,7 +186,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "test") + resources, err := ParseMultidoc([]byte(doc), "", "test") if err != nil { t.Fatal(err) } @@ -253,7 +253,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "test") + resources, err := ParseMultidoc([]byte(doc), "", "test") if err != nil { t.Fatal(err) } @@ -304,7 +304,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "test") + resources, err := ParseMultidoc([]byte(doc), "", "test") if err != nil { t.Fatal(err) } @@ -391,7 +391,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "test") + resources, err := ParseMultidoc([]byte(doc), "", "test") if err != nil { t.Fatal(err) } diff --git a/cluster/kubernetes/resource/load.go b/cluster/kubernetes/resource/load.go index 8803f2a3b..0757ce7f8 100644 --- a/cluster/kubernetes/resource/load.go +++ b/cluster/kubernetes/resource/load.go @@ -14,8 +14,11 @@ import ( // Load takes paths to directories or files, and creates an object set // based on the file(s) therein. Resources are named according to the -// file content, rather than the file name of directory structure. -func Load(base string, paths []string) (map[string]resource.Resource, error) { +// file content, rather than the file name of directory structure. The +// `fallbackNamespace` is assigned to any resource that doesn't +// specify a namespace; it's only used for identification, and not put +// in the definition. +func Load(base, fallbackNamespace string, paths []string) (map[string]resource.Resource, error) { if _, err := os.Stat(base); os.IsNotExist(err) { return nil, fmt.Errorf("git path %q not found", base) } @@ -47,7 +50,7 @@ func Load(base string, paths []string) (map[string]resource.Resource, error) { if err != nil { return errors.Wrapf(err, "path to scan %q is not under base %q", path, base) } - docsInFile, err := ParseMultidoc(bytes, source) + docsInFile, err := ParseMultidoc(bytes, source, fallbackNamespace) if err != nil { return err } @@ -127,7 +130,7 @@ func looksLikeChart(dir string) bool { // ParseMultidoc takes a dump of config (a multidoc YAML) and // constructs an object set from the resources represented therein. -func ParseMultidoc(multidoc []byte, source string) (map[string]resource.Resource, error) { +func ParseMultidoc(multidoc []byte, source, fallbackNamespace string) (map[string]resource.Resource, error) { objs := map[string]resource.Resource{} chunks := bufio.NewScanner(bytes.NewReader(multidoc)) initialBuffer := make([]byte, 4096) // Matches startBufSize in bufio/scan.go @@ -143,7 +146,7 @@ func ParseMultidoc(multidoc []byte, source string) (map[string]resource.Resource bytes := chunks.Bytes() bytes2 := make([]byte, len(bytes), cap(bytes)) copy(bytes2, bytes) - if obj, err = unmarshalObject(source, bytes2); err != nil { + if obj, err = unmarshalObject(source, fallbackNamespace, bytes2); err != nil { return nil, errors.Wrapf(err, "parsing YAML doc from %q", source) } if obj == nil { diff --git a/cluster/kubernetes/resource/load_test.go b/cluster/kubernetes/resource/load_test.go index 15b102c11..c06be8e8f 100644 --- a/cluster/kubernetes/resource/load_test.go +++ b/cluster/kubernetes/resource/load_test.go @@ -13,9 +13,13 @@ import ( "github.com/weaveworks/flux/resource" ) +const ( + fallbackNS = "fallback" +) + // for convenience func base(source, kind, namespace, name string) baseObject { - b := baseObject{source: source, Kind: kind} + b := baseObject{source: source, Kind: kind, fallbackNamespace: fallbackNS} b.Meta.Namespace = namespace b.Meta.Name = name return b @@ -24,7 +28,7 @@ func base(source, kind, namespace, name string) baseObject { func TestParseEmpty(t *testing.T) { doc := `` - objs, err := ParseMultidoc([]byte(doc), "test") + objs, err := ParseMultidoc([]byte(doc), "", "test") if err != nil { t.Error(err) } @@ -44,7 +48,7 @@ kind: Deployment metadata: name: a-deployment ` - objs, err := ParseMultidoc([]byte(docs), "test") + objs, err := ParseMultidoc([]byte(docs), "test", fallbackNS) if err != nil { t.Error(err) } @@ -76,7 +80,7 @@ kind: Deployment metadata: name: a-deployment ` - objs, err := ParseMultidoc([]byte(docs), "test") + objs, err := ParseMultidoc([]byte(docs), "test", fallbackNS) if err != nil { t.Error(err) } @@ -115,7 +119,7 @@ data: buffer.WriteString(line) } - _, err := ParseMultidoc(buffer.Bytes(), "test") + _, err := ParseMultidoc(buffer.Bytes(), "test", fallbackNS) if err != nil { t.Error(err) } @@ -137,7 +141,7 @@ spec: - name: weekly-curl-homepage image: centos:7 # Has curl installed by default ` - objs, err := ParseMultidoc([]byte(doc), "test") + objs, err := ParseMultidoc([]byte(doc), "test", fallbackNS) assert.NoError(t, err) obj, ok := objs["default:cronjob/weekly-curl-homepage"] @@ -161,11 +165,13 @@ items: - kind: Deployment metadata: name: foo + namespace: ns - kind: Service metadata: name: bar + namespace: ns ` - res, err := unmarshalObject("", []byte(doc)) + res, err := unmarshalObject("", fallbackNS, []byte(doc)) if err != nil { t.Fatal(err) } @@ -177,8 +183,8 @@ items: t.Fatalf("expected two items, got %+v", list.Items) } for i, id := range []flux.ResourceID{ - flux.MustParseResourceID("default:deployment/foo"), - flux.MustParseResourceID("default:service/bar")} { + flux.MustParseResourceID("ns:deployment/foo"), + flux.MustParseResourceID("ns:service/bar")} { if list.Items[i].ResourceID() != id { t.Errorf("At %d, expected %q, got %q", i, id, list.Items[i].ResourceID()) } @@ -200,7 +206,7 @@ func TestLoadSome(t *testing.T) { if err := testfiles.WriteTestFiles(dir); err != nil { t.Fatal(err) } - objs, err := Load(dir, []string{dir}) + objs, err := Load(dir, fallbackNS, []string{dir}) if err != nil { t.Error(err) } @@ -231,7 +237,7 @@ func TestChartTracker(t *testing.T) { if f == "garbage" { continue } - if m, err := Load(dir, []string{fq}); err != nil || len(m) == 0 { + if m, err := Load(dir, fallbackNS, []string{fq}); err != nil || len(m) == 0 { t.Errorf("Load returned 0 objs, err=%v", err) } } @@ -250,7 +256,7 @@ func TestChartTracker(t *testing.T) { } for _, f := range chartfiles { fq := filepath.Join(dir, f) - if m, err := Load(dir, []string{fq}); err != nil || len(m) != 0 { + if m, err := Load(dir, fallbackNS, []string{fq}); err != nil || len(m) != 0 { t.Errorf("%q not ignored as a chart should be", f) } } diff --git a/cluster/kubernetes/resource/resource.go b/cluster/kubernetes/resource/resource.go index 0d18f5ad1..ded2c451d 100644 --- a/cluster/kubernetes/resource/resource.go +++ b/cluster/kubernetes/resource/resource.go @@ -13,16 +13,21 @@ import ( const ( PolicyPrefix = "flux.weave.works/" + // The namespace to presume if something doesn't have one, and we + // haven't been told what to use as a fallback + FallbackFallbackNamespace = "default" ) // -- unmarshaling code for specific object and field types // struct to embed in objects, to provide default implementation type baseObject struct { - source string - bytes []byte - Kind string `yaml:"kind"` - Meta struct { + source string + fallbackNamespace string + + bytes []byte + Kind string `yaml:"kind"` + Meta struct { Namespace string `yaml:"namespace"` Name string `yaml:"name"` Annotations map[string]string `yaml:"annotations,omitempty"` @@ -32,7 +37,11 @@ type baseObject struct { func (o baseObject) ResourceID() flux.ResourceID { ns := o.Meta.Namespace if ns == "" { - ns = "default" + if o.fallbackNamespace == "" { + ns = FallbackFallbackNamespace + } else { + ns = o.fallbackNamespace + } } return flux.MakeResourceID(ns, o.Kind, o.Meta.Name) } @@ -70,8 +79,8 @@ func (o baseObject) Bytes() []byte { return o.bytes } -func unmarshalObject(source string, bytes []byte) (resource.Resource, error) { - var base = baseObject{source: source, bytes: bytes} +func unmarshalObject(source, fallbackNamespace string, bytes []byte) (resource.Resource, error) { + var base = baseObject{source: source, fallbackNamespace: fallbackNamespace, bytes: bytes} if err := yaml.Unmarshal(bytes, &base); err != nil { return nil, err } @@ -154,7 +163,7 @@ func unmarshalList(base baseObject, raw *rawList, list *List) error { if err != nil { return err } - res, err := unmarshalObject(base.source, bytes) + res, err := unmarshalObject(base.source, base.fallbackNamespace, bytes) if err != nil { return err } diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index b599b8ec4..428c612bd 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -30,6 +30,8 @@ import ( const ( stackLabel = kresource.PolicyPrefix + "stack" checksumAnnotation = kresource.PolicyPrefix + "stack_checksum" + + DefaultDefaultNamespace = "default" ) // Sync takes a definition of what should be running in the cluster, @@ -40,6 +42,8 @@ const ( func (c *Cluster) Sync(spec cluster.SyncDef) error { logger := log.With(c.logger, "method", "Sync") + fallbackNamespace := c.applier.getDefaultNamespace() + type checksum struct { stack, sum string } @@ -49,7 +53,7 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { // NB we get all resources, since we care about leaving unsynced, // _ignored_ resources alone. - clusterResources, err := c.getResourcesBySelector("") + clusterResources, err := c.getResourcesBySelector("", fallbackNamespace) if err != nil { return errors.Wrap(err, "collating resources in cluster for sync") } @@ -93,7 +97,7 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { if c.GC { orphanedResources := makeChangeSet() - clusterResources, err := c.getResourcesInStack() + clusterResources, err := c.getResourcesInStack(fallbackNamespace) // <-- FIXME(right now) if err != nil { return errors.Wrap(err, "collating resources in cluster for calculating garbage collection") } @@ -139,11 +143,27 @@ type kuberesource struct { obj *unstructured.Unstructured } +// ResourceID returns the ResourceID for this resource loaded from the +// cluster. func (r *kuberesource) ResourceID() flux.ResourceID { ns, kind, name := r.obj.GetNamespace(), r.obj.GetKind(), r.obj.GetName() return flux.MakeResourceID(ns, kind, name) } +// AssumedNamespaceResourceID returns a ResourceID which assumes the +// namespace is fallbackNamespace if it is missing. Resources returned +// from the cluster that are not namespaced (e.g., ClusterRoles) will +// have an empty string in the namespace field. To be able to compare +// them to resources we load from files, we must assume a namespace if +// none is given. +func (r *kuberesource) AssumedNamespaceResourceID(fallbackNamespace string) flux.ResourceID { + ns := r.obj.GetNamespace() + if ns == "" { + ns = fallbackNamespace + } + return flux.MakeResourceID(ns, r.obj.GetKind(), r.obj.GetName()) +} + // Bytes returns a byte slice description func (r *kuberesource) Bytes() []byte { return []byte(fmt.Sprintf(` @@ -171,7 +191,7 @@ func (r *kuberesource) GetStack() string { return r.obj.GetLabels()[stackLabel] } -func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesource, error) { +func (c *Cluster) getResourcesBySelector(selector, assumedNamespace string) (map[string]*kuberesource, error) { listOptions := meta_v1.ListOptions{} if selector != "" { listOptions.LabelSelector = selector @@ -234,7 +254,7 @@ func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesou // TODO(michael) also exclude anything that has an ownerReference (that isn't "standard"?) res := &kuberesource{obj: &data.Items[i]} - result[res.ResourceID().String()] = res + result[res.AssumedNamespaceResourceID(assumedNamespace).String()] = res } } } @@ -244,8 +264,8 @@ func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesou // exportResourcesInStack collates all the resources that belong to a // stack, i.e., were applied by flux. -func (c *Cluster) getResourcesInStack() (map[string]*kuberesource, error) { - return c.getResourcesBySelector(stackLabel) // means "has label <>" +func (c *Cluster) getResourcesInStack(assumedNamespace string) (map[string]*kuberesource, error) { + return c.getResourcesBySelector(stackLabel, assumedNamespace) // means "has label <>" } func applyMetadata(res resource.Resource, stack, checksum string) ([]byte, error) { @@ -302,6 +322,7 @@ func (c *changeSet) stage(cmd string, id flux.ResourceID, source string, bytes [ // Applier is something that will apply a changeset to the cluster. type Applier interface { apply(log.Logger, changeSet, map[flux.ResourceID]error) cluster.SyncError + getDefaultNamespace() string } type Kubectl struct { @@ -342,6 +363,29 @@ func (c *Kubectl) connectArgs() []string { return args } +// getDefaultNamespace returns the fallback namespace used by the +// applied when a namespaced resource doesn't have one specified. This +// is used when syncing to anticipate the identity of a resource in +// the cluster given the manifest from a file (which may be missing +// the namespace). +func (k *Kubectl) getDefaultNamespace() string { + cmd := k.kubectlCommand("config", "get-contexts", "--no-headers") + out, err := cmd.Output() + if err != nil { + return DefaultDefaultNamespace + } + lines := bytes.Split(out, []byte("\n")) + for _, line := range lines { + words := bytes.Fields(line) + if len(words) > 1 && string(words[0]) == "*" { + if len(words) == 5 { + return string(words[4]) + } + } + } + return DefaultDefaultNamespace +} + // rankOfKind returns an int denoting the position of the given kind // in the partial ordering of Kubernetes resources, according to which // kinds depend on which (derived by hand). diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index 657fcefa0..3f8a2b620 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -20,6 +20,7 @@ import ( // dynamicfake "k8s.io/client-go/dynamic/fake" // k8sclient "k8s.io/client-go/kubernetes" "github.com/stretchr/testify/assert" + "k8s.io/client-go/discovery" corefake "k8s.io/client-go/kubernetes/fake" k8s_testing "k8s.io/client-go/testing" @@ -30,12 +31,16 @@ import ( "github.com/weaveworks/flux/sync" ) +const ( + defaultNamespace = "unusual-default" +) + func fakeClients() extendedClient { scheme := runtime.NewScheme() // Set this to `true` to output a trace of the API actions called // while running the tests - const debug = false + const debug = true getAndList := metav1.Verbs([]string{"get", "list"}) // Adding these means the fake dynamic client will find them, and @@ -48,9 +53,15 @@ func fakeClients() extendedClient { {Name: "deployments", SingularName: "deployment", Namespaced: true, Kind: "Deployment", Verbs: getAndList}, }, }, + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + {Name: "namespaces", SingularName: "namespace", Namespaced: false, Kind: "Namespace", Verbs: getAndList}, + }, + }, } - coreClient := corefake.NewSimpleClientset(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "foobar"}}) + coreClient := corefake.NewSimpleClientset(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: defaultNamespace}}) fluxClient := fluxfake.NewSimpleClientset() dynamicClient := NewSimpleDynamicClient(scheme) // NB from this package, rather than the official one, since we needed a patched version @@ -64,7 +75,7 @@ func fakeClients() extendedClient { for _, fake := range []*k8s_testing.Fake{&coreClient.Fake, &fluxClient.Fake, &dynamicClient.Fake} { fake.PrependReactor("*", "*", func(action k8s_testing.Action) (bool, runtime.Object, error) { gvr := action.GetResource() - println("[DEBUG] action:", action.GetVerb(), gvr.Group, gvr.Version, gvr.Resource) + fmt.Printf("[DEBUG] action: %s ns:%s %s/%s %s\n", action.GetVerb(), action.GetNamespace(), gvr.Group, gvr.Version, gvr.Resource) return false, nil, nil }) } @@ -84,9 +95,15 @@ func fakeClients() extendedClient { // correct effect, which is either to "upsert", or delete, resources. type fakeApplier struct { client dynamic.Interface + discovery discovery.DiscoveryInterface + defaultNS string commandRun bool } +func (a fakeApplier) getDefaultNamespace() string { + return defaultNamespace +} + func groupVersionResource(res *unstructured.Unstructured) schema.GroupVersionResource { gvk := res.GetObjectKind().GroupVersionKind() return schema.GroupVersionResource{Group: gvk.Group, Version: gvk.Version, Resource: strings.ToLower(gvk.Kind) + "s"} @@ -113,8 +130,22 @@ func (a fakeApplier) apply(_ log.Logger, cs changeSet, errored map[flux.Resource gvr := groupVersionResource(res) c := a.client.Resource(gvr) + // This is an approximation to what `kubectl` does in filling + // in the fallback namespace (from config). In the case of + // non-namespaced entities, it will be ignored by the fake + // client (FIXME: make sure of this). + apiRes := findAPIResource(gvr, a.discovery) + if apiRes == nil { + panic("no APIResource found for " + gvr.String()) + } + var dc dynamic.ResourceInterface = c - if ns := res.GetNamespace(); ns != "" { + ns := res.GetNamespace() + if apiRes.Namespaced { + if ns == "" { + ns = a.defaultNS + res.SetNamespace(ns) + } dc = c.Namespace(ns) } name := res.GetName() @@ -153,11 +184,28 @@ func (a fakeApplier) apply(_ log.Logger, cs changeSet, errored map[flux.Resource return errs } +func findAPIResource(gvr schema.GroupVersionResource, disco discovery.DiscoveryInterface) *metav1.APIResource { + groupVersion := gvr.Version + if gvr.Group != "" { + groupVersion = gvr.Group + "/" + groupVersion + } + reses, err := disco.ServerResourcesForGroupVersion(groupVersion) + if err != nil { + return nil + } + for _, res := range reses.APIResources { + if res.Name == gvr.Resource { + return &res + } + } + return nil +} + // --- func setup(t *testing.T) (*Cluster, *fakeApplier) { clients := fakeClients() - applier := &fakeApplier{client: clients.dynamicClient} + applier := &fakeApplier{client: clients.dynamicClient, discovery: clients.coreClient.Discovery(), defaultNS: defaultNamespace} kube := &Cluster{ applier: applier, client: clients, @@ -217,7 +265,7 @@ metadata: } test := func(t *testing.T, kube *Cluster, defs, expectedAfterSync string, expectErrors bool) { - resources, err := kresource.ParseMultidoc([]byte(defs), "before") + resources, err := kresource.ParseMultidoc([]byte(defs), "before", defaultNamespace) if err != nil { t.Fatal(err) } @@ -226,20 +274,20 @@ metadata: if !expectErrors && err != nil { t.Error(err) } - expected, err := kresource.ParseMultidoc([]byte(expectedAfterSync), "after") + expected, err := kresource.ParseMultidoc([]byte(expectedAfterSync), "after", defaultNamespace) if err != nil { panic(err) } // Now check that the resources were created - actual, err := kube.getResourcesInStack() + actual, err := kube.getResourcesInStack(defaultNamespace) if err != nil { t.Fatal(err) } for id := range actual { if _, ok := expected[id]; !ok { - t.Errorf("resource present after sync but not in resources applied: %q", id) + t.Errorf("resource present after sync but not in resources applied: %q (present: %v)", id, actual) if j, err := yaml.Marshal(actual[id].obj); err == nil { println(string(j)) } @@ -249,7 +297,7 @@ metadata: } for id := range expected { if _, ok := actual[id]; !ok { - t.Errorf("resource supposed to be synced but not present: %q", id) + t.Errorf("resource supposed to be synced but not present: %q (present: %v)", id, actual) } // no need to compare values, since we already considered // the intersection of actual and expected above. @@ -273,6 +321,44 @@ metadata: test(t, kube, "", "", false) }) + t.Run("sync won't delete non-namespaced resources", func(t *testing.T) { + kube, _ := setup(t) + kube.GC = true + + const nsDef = ` +apiVersion: v1 +kind: Namespace +metadata: + name: bar-ns +` + test(t, kube, nsDef, nsDef, false) + }) + + t.Run("sync won't delete resources that got the fallback namespace when created", func(t *testing.T) { + // NB: this tests the fake client implementation to some + // extent as well. It relies on it to reflect the kubectl + // behaviour of giving things that need a namespace some + // fallback (this would come from kubeconfig usually); and, + // for things that _don't_ have a namespace to have it + // stripped out. + kube, _ := setup(t) + kube.GC = true + const withoutNS = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: depFallbackNS +` + const withNS = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: depFallbackNS + namespace: ` + defaultNamespace + ` +` + test(t, kube, withoutNS, withNS, false) + }) + t.Run("sync won't delete if apply failed", func(t *testing.T) { kube, _ := setup(t) kube.GC = true @@ -409,7 +495,7 @@ spec: assert.NoError(t, err) // Check that our resource-getting also sees the pre-existing resource - resources, err := kube.getResourcesBySelector("") + resources, err := kube.getResourcesBySelector("", defaultNamespace) assert.NoError(t, err) assert.Contains(t, resources, "foobar:deployment/dep1") diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go index 6dd2cc46b..13328ca9b 100644 --- a/daemon/daemon_test.go +++ b/daemon/daemon_test.go @@ -20,7 +20,6 @@ import ( "github.com/weaveworks/flux/api/v9" "github.com/weaveworks/flux/cluster" "github.com/weaveworks/flux/cluster/kubernetes" - kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" "github.com/weaveworks/flux/event" "github.com/weaveworks/flux/git" "github.com/weaveworks/flux/git/gittest" @@ -647,7 +646,7 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *cluster.Mock, *mockEven return []cluster.Controller{}, nil } k8s.ExportFunc = func() ([]byte, error) { return testBytes, nil } - k8s.LoadManifestsFunc = kresource.Load + k8s.LoadManifestsFunc = (&kubernetes.Manifests{}).LoadManifests k8s.PingFunc = func() error { return nil } k8s.SomeServicesFunc = func([]flux.ResourceID) ([]cluster.Controller, error) { return []cluster.Controller{ diff --git a/daemon/loop_test.go b/daemon/loop_test.go index 05da7692d..6a426eb3f 100644 --- a/daemon/loop_test.go +++ b/daemon/loop_test.go @@ -15,7 +15,7 @@ import ( "github.com/weaveworks/flux" "github.com/weaveworks/flux/cluster" - kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" + "github.com/weaveworks/flux/cluster/kubernetes" "github.com/weaveworks/flux/cluster/kubernetes/testfiles" "github.com/weaveworks/flux/event" "github.com/weaveworks/flux/git" @@ -41,7 +41,7 @@ func daemon(t *testing.T) (*Daemon, func()) { repo, repoCleanup := gittest.Repo(t) k8s = &cluster.Mock{} - k8s.LoadManifestsFunc = kresource.Load + k8s.LoadManifestsFunc = (&kubernetes.Manifests{}).LoadManifests k8s.ExportFunc = func() ([]byte, error) { return nil, nil } events = &mockEventWriter{} From d48609add92e027e21d73c972f584e3c1068228b Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Mon, 4 Feb 2019 14:48:39 +0000 Subject: [PATCH 14/24] Assign namespaces after parsing So we can separate parsing from processing at least minimally, this commit makes assigning a namespace to resources (that need one) a post-processing step, in `Manifests.LoadManifests`. It also makes it a more reliable operation than before, since it looks up whether or not a particular resource is _supposed_ to have a namespace, according to the Kubernetes API, before assigning the configured default. --- cluster/cluster.go | 2 +- cluster/kubernetes/manifests.go | 38 +++++++- cluster/kubernetes/namespacer.go | 63 ++++++++++++++ cluster/kubernetes/namespacer_test.go | 87 +++++++++++++++++++ cluster/kubernetes/policies.go | 2 +- .../resource/fluxhelmrelease_test.go | 14 +-- cluster/kubernetes/resource/list.go | 6 +- cluster/kubernetes/resource/load.go | 20 ++--- cluster/kubernetes/resource/load_test.go | 24 +++-- cluster/kubernetes/resource/resource.go | 63 ++++++++++---- cluster/kubernetes/sync.go | 55 +++++------- cluster/kubernetes/sync_test.go | 31 ++++--- cluster/mock.go | 5 -- cmd/fluxd/main.go | 7 +- daemon/daemon_test.go | 16 +++- daemon/loop_test.go | 5 +- release/releaser_test.go | 11 ++- 17 files changed, 331 insertions(+), 118 deletions(-) create mode 100644 cluster/kubernetes/namespacer.go create mode 100644 cluster/kubernetes/namespacer_test.go diff --git a/cluster/cluster.go b/cluster/cluster.go index d153ff328..78eae6870 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -74,7 +74,7 @@ type Controller struct { Rollout RolloutStatus // Errors during the recurring sync from the Git repository to the // cluster will surface here. - SyncError error + SyncError error Containers ContainersOrExcuse } diff --git a/cluster/kubernetes/manifests.go b/cluster/kubernetes/manifests.go index 973197471..4bf8eecac 100644 --- a/cluster/kubernetes/manifests.go +++ b/cluster/kubernetes/manifests.go @@ -7,12 +7,46 @@ import ( "github.com/weaveworks/flux/resource" ) +// namespacer assigns namespaces to manifests that need it (or "" if +// the manifest should not have a namespace. +type namespacer interface { + // EffectiveNamespace gives the namespace that would be used were + // the manifest to be applied. This may be "", indicating that it + // should not have a namespace (i.e., it's a cluster-level + // resource). + EffectiveNamespace(kresource.KubeManifest) (string, error) +} + +// Manifests is an implementation of cluster.Manifests, particular to +// Kubernetes. Aside from loading manifests from files, it does some +// "post-processsing" to make sure the view of the manifests is what +// would be applied; in particular, it fills in the namespace of +// manifests that would be given a default namespace when applied. type Manifests struct { - FallbackNamespace string + Namespacer namespacer +} + +func postProcess(manifests map[string]kresource.KubeManifest, nser namespacer) (map[string]resource.Resource, error) { + result := map[string]resource.Resource{} + for _, km := range manifests { + if nser != nil { + ns, err := nser.EffectiveNamespace(km) + if err != nil { + return nil, err + } + km.SetNamespace(ns) + } + result[km.ResourceID().String()] = km + } + return result, nil } func (c *Manifests) LoadManifests(base string, paths []string) (map[string]resource.Resource, error) { - return kresource.Load(base, c.FallbackNamespace, paths) + manifests, err := kresource.Load(base, paths) + if err != nil { + return nil, err + } + return postProcess(manifests, c.Namespacer) } func (c *Manifests) UpdateImage(def []byte, id flux.ResourceID, container string, image image.Ref) ([]byte, error) { diff --git a/cluster/kubernetes/namespacer.go b/cluster/kubernetes/namespacer.go new file mode 100644 index 000000000..07b2397f3 --- /dev/null +++ b/cluster/kubernetes/namespacer.go @@ -0,0 +1,63 @@ +package kubernetes + +import ( + "fmt" + + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached" + + kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" +) + +type namespaceViaDiscovery struct { + fallbackNamespace string + disco discovery.DiscoveryInterface +} + +type namespaceDefaulter interface { + GetDefaultNamespace() (string, error) +} + +// NewNamespacer creates an implementation of Namespacer +func NewNamespacer(ns namespaceDefaulter, d discovery.DiscoveryInterface) (*namespaceViaDiscovery, error) { + fallback, err := ns.GetDefaultNamespace() + if err != nil { + return nil, err + } + cachedDisco := cached.NewMemCacheClient(d) + // in client-go v9, the call of Invalidate is necessary to force + // it to query for initial data; in subsequent versions, it is a + // no-op reset of the cache validity, so safe to call. + cachedDisco.Invalidate() + return &namespaceViaDiscovery{fallbackNamespace: fallback, disco: cachedDisco}, nil +} + +// effectiveNamespace yields the namespace that would be used for this +// resource were it applied, taking into account the kind of the +// resource, and local configuration. +func (n *namespaceViaDiscovery) EffectiveNamespace(m kresource.KubeManifest) (string, error) { + namespaced, err := n.lookupNamespaced(m) + switch { + case err != nil: + return "", err + case namespaced && m.GetNamespace() == "": + return n.fallbackNamespace, nil + case !namespaced: + return "", nil + } + return m.GetNamespace(), nil +} + +func (n *namespaceViaDiscovery) lookupNamespaced(m kresource.KubeManifest) (bool, error) { + groupVersion, kind := m.GroupVersion(), m.GetKind() + resourceList, err := n.disco.ServerResourcesForGroupVersion(groupVersion) + if err != nil { + return false, err + } + for _, resource := range resourceList.APIResources { + if resource.Kind == kind { + return resource.Namespaced, nil + } + } + return false, fmt.Errorf("resource not found for API %s, kind %s", groupVersion, kind) +} diff --git a/cluster/kubernetes/namespacer_test.go b/cluster/kubernetes/namespacer_test.go new file mode 100644 index 000000000..237d0465f --- /dev/null +++ b/cluster/kubernetes/namespacer_test.go @@ -0,0 +1,87 @@ +package kubernetes + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corefake "k8s.io/client-go/kubernetes/fake" + + kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" +) + +type namespaceDefaulterFake string + +func (ns namespaceDefaulterFake) GetDefaultNamespace() (string, error) { + return string(ns), nil +} + +func TestNamespaceDefaulting(t *testing.T) { + + getAndList := metav1.Verbs([]string{"get", "list"}) + apiResources := []*metav1.APIResourceList{ + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + {Name: "deployments", SingularName: "deployment", Namespaced: true, Kind: "Deployment", Verbs: getAndList}, + }, + }, + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + {Name: "namespaces", SingularName: "namespace", Namespaced: false, Kind: "Namespace", Verbs: getAndList}, + }, + }, + } + + coreClient := corefake.NewSimpleClientset() + coreClient.Fake.Resources = apiResources + disco := coreClient.Discovery() + nser, err := NewNamespacer(namespaceDefaulterFake("fallback-ns"), disco) + if err != nil { + t.Fatal(err) + } + + const defs = `--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hasNamespace + namespace: foo-ns +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: noNamespace +--- +apiVersion: v1 +kind: Namespace +metadata: + name: notNamespaced + namespace: spurious +` + + manifests, err := kresource.ParseMultidoc([]byte(defs), "") + if err != nil { + t.Fatal(err) + } + + assertEffectiveNamespace := func(id, expected string) { + res, ok := manifests[id] + if !ok { + t.Errorf("manifest for %q not found", id) + return + } + got, err := nser.EffectiveNamespace(res) + if err != nil { + t.Errorf("error getting effective namespace for %q: %s", id, err.Error()) + return + } + if got != expected { + t.Errorf("expected effective namespace of %q, got %q", expected, got) + } + } + + assertEffectiveNamespace("foo-ns:deployment/hasNamespace", "foo-ns") + assertEffectiveNamespace(":deployment/noNamespace", "fallback-ns") + assertEffectiveNamespace("spurious:namespace/notNamespaced", "") +} diff --git a/cluster/kubernetes/policies.go b/cluster/kubernetes/policies.go index 5fbdf4b32..79d51221c 100644 --- a/cluster/kubernetes/policies.go +++ b/cluster/kubernetes/policies.go @@ -67,7 +67,7 @@ func extractAnnotations(def []byte) (map[string]string, error) { } func extractContainers(def []byte, id flux.ResourceID) ([]resource.Container, error) { - resources, err := kresource.ParseMultidoc(def, "", "stdin") + resources, err := kresource.ParseMultidoc(def, "stdin") if err != nil { return nil, err } diff --git a/cluster/kubernetes/resource/fluxhelmrelease_test.go b/cluster/kubernetes/resource/fluxhelmrelease_test.go index 4f2abbede..8cd037e6e 100644 --- a/cluster/kubernetes/resource/fluxhelmrelease_test.go +++ b/cluster/kubernetes/resource/fluxhelmrelease_test.go @@ -26,7 +26,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "", "test") + resources, err := ParseMultidoc([]byte(doc), "test") if err != nil { t.Fatal(err) } @@ -72,7 +72,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "", "test") + resources, err := ParseMultidoc([]byte(doc), "test") if err != nil { t.Fatal(err) } @@ -116,7 +116,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "", "test") + resources, err := ParseMultidoc([]byte(doc), "test") if err != nil { t.Fatal(err) } @@ -186,7 +186,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "", "test") + resources, err := ParseMultidoc([]byte(doc), "test") if err != nil { t.Fatal(err) } @@ -253,7 +253,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "", "test") + resources, err := ParseMultidoc([]byte(doc), "test") if err != nil { t.Fatal(err) } @@ -304,7 +304,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "", "test") + resources, err := ParseMultidoc([]byte(doc), "test") if err != nil { t.Fatal(err) } @@ -391,7 +391,7 @@ spec: enabled: false ` - resources, err := ParseMultidoc([]byte(doc), "", "test") + resources, err := ParseMultidoc([]byte(doc), "test") if err != nil { t.Fatal(err) } diff --git a/cluster/kubernetes/resource/list.go b/cluster/kubernetes/resource/list.go index f9f9fc743..3b53c3682 100644 --- a/cluster/kubernetes/resource/list.go +++ b/cluster/kubernetes/resource/list.go @@ -1,10 +1,6 @@ package resource -import ( - "github.com/weaveworks/flux/resource" -) - type List struct { baseObject - Items []resource.Resource + Items []KubeManifest } diff --git a/cluster/kubernetes/resource/load.go b/cluster/kubernetes/resource/load.go index 0757ce7f8..69f3efec4 100644 --- a/cluster/kubernetes/resource/load.go +++ b/cluster/kubernetes/resource/load.go @@ -9,20 +9,16 @@ import ( "path/filepath" "github.com/pkg/errors" - "github.com/weaveworks/flux/resource" ) // Load takes paths to directories or files, and creates an object set // based on the file(s) therein. Resources are named according to the -// file content, rather than the file name of directory structure. The -// `fallbackNamespace` is assigned to any resource that doesn't -// specify a namespace; it's only used for identification, and not put -// in the definition. -func Load(base, fallbackNamespace string, paths []string) (map[string]resource.Resource, error) { +// file content, rather than the file name of directory structure. +func Load(base string, paths []string) (map[string]KubeManifest, error) { if _, err := os.Stat(base); os.IsNotExist(err) { return nil, fmt.Errorf("git path %q not found", base) } - objs := map[string]resource.Resource{} + objs := map[string]KubeManifest{} charts, err := newChartTracker(base) if err != nil { return nil, errors.Wrapf(err, "walking %q for chartdirs", base) @@ -50,7 +46,7 @@ func Load(base, fallbackNamespace string, paths []string) (map[string]resource.R if err != nil { return errors.Wrapf(err, "path to scan %q is not under base %q", path, base) } - docsInFile, err := ParseMultidoc(bytes, source, fallbackNamespace) + docsInFile, err := ParseMultidoc(bytes, source) if err != nil { return err } @@ -130,14 +126,14 @@ func looksLikeChart(dir string) bool { // ParseMultidoc takes a dump of config (a multidoc YAML) and // constructs an object set from the resources represented therein. -func ParseMultidoc(multidoc []byte, source, fallbackNamespace string) (map[string]resource.Resource, error) { - objs := map[string]resource.Resource{} +func ParseMultidoc(multidoc []byte, source string) (map[string]KubeManifest, error) { + objs := map[string]KubeManifest{} chunks := bufio.NewScanner(bytes.NewReader(multidoc)) initialBuffer := make([]byte, 4096) // Matches startBufSize in bufio/scan.go chunks.Buffer(initialBuffer, 1024*1024) // Allow growth to 1MB chunks.Split(splitYAMLDocument) - var obj resource.Resource + var obj KubeManifest var err error for chunks.Scan() { // It's not guaranteed that the return value of Bytes() will not be mutated later: @@ -146,7 +142,7 @@ func ParseMultidoc(multidoc []byte, source, fallbackNamespace string) (map[strin bytes := chunks.Bytes() bytes2 := make([]byte, len(bytes), cap(bytes)) copy(bytes2, bytes) - if obj, err = unmarshalObject(source, fallbackNamespace, bytes2); err != nil { + if obj, err = unmarshalObject(source, bytes2); err != nil { return nil, errors.Wrapf(err, "parsing YAML doc from %q", source) } if obj == nil { diff --git a/cluster/kubernetes/resource/load_test.go b/cluster/kubernetes/resource/load_test.go index c06be8e8f..1b99c31a6 100644 --- a/cluster/kubernetes/resource/load_test.go +++ b/cluster/kubernetes/resource/load_test.go @@ -13,13 +13,9 @@ import ( "github.com/weaveworks/flux/resource" ) -const ( - fallbackNS = "fallback" -) - // for convenience func base(source, kind, namespace, name string) baseObject { - b := baseObject{source: source, Kind: kind, fallbackNamespace: fallbackNS} + b := baseObject{source: source, Kind: kind} b.Meta.Namespace = namespace b.Meta.Name = name return b @@ -28,7 +24,7 @@ func base(source, kind, namespace, name string) baseObject { func TestParseEmpty(t *testing.T) { doc := `` - objs, err := ParseMultidoc([]byte(doc), "", "test") + objs, err := ParseMultidoc([]byte(doc), "test") if err != nil { t.Error(err) } @@ -48,7 +44,7 @@ kind: Deployment metadata: name: a-deployment ` - objs, err := ParseMultidoc([]byte(docs), "test", fallbackNS) + objs, err := ParseMultidoc([]byte(docs), "test") if err != nil { t.Error(err) } @@ -80,7 +76,7 @@ kind: Deployment metadata: name: a-deployment ` - objs, err := ParseMultidoc([]byte(docs), "test", fallbackNS) + objs, err := ParseMultidoc([]byte(docs), "test") if err != nil { t.Error(err) } @@ -119,7 +115,7 @@ data: buffer.WriteString(line) } - _, err := ParseMultidoc(buffer.Bytes(), "test", fallbackNS) + _, err := ParseMultidoc(buffer.Bytes(), "test") if err != nil { t.Error(err) } @@ -141,7 +137,7 @@ spec: - name: weekly-curl-homepage image: centos:7 # Has curl installed by default ` - objs, err := ParseMultidoc([]byte(doc), "test", fallbackNS) + objs, err := ParseMultidoc([]byte(doc), "test") assert.NoError(t, err) obj, ok := objs["default:cronjob/weekly-curl-homepage"] @@ -171,7 +167,7 @@ items: name: bar namespace: ns ` - res, err := unmarshalObject("", fallbackNS, []byte(doc)) + res, err := unmarshalObject("", []byte(doc)) if err != nil { t.Fatal(err) } @@ -206,7 +202,7 @@ func TestLoadSome(t *testing.T) { if err := testfiles.WriteTestFiles(dir); err != nil { t.Fatal(err) } - objs, err := Load(dir, fallbackNS, []string{dir}) + objs, err := Load(dir, []string{dir}) if err != nil { t.Error(err) } @@ -237,7 +233,7 @@ func TestChartTracker(t *testing.T) { if f == "garbage" { continue } - if m, err := Load(dir, fallbackNS, []string{fq}); err != nil || len(m) == 0 { + if m, err := Load(dir, []string{fq}); err != nil || len(m) == 0 { t.Errorf("Load returned 0 objs, err=%v", err) } } @@ -256,7 +252,7 @@ func TestChartTracker(t *testing.T) { } for _, f := range chartfiles { fq := filepath.Join(dir, f) - if m, err := Load(dir, fallbackNS, []string{fq}); err != nil || len(m) != 0 { + if m, err := Load(dir, []string{fq}); err != nil || len(m) != 0 { t.Errorf("%q not ignored as a chart should be", f) } } diff --git a/cluster/kubernetes/resource/resource.go b/cluster/kubernetes/resource/resource.go index ded2c451d..67701c3c7 100644 --- a/cluster/kubernetes/resource/resource.go +++ b/cluster/kubernetes/resource/resource.go @@ -13,39 +13,66 @@ import ( const ( PolicyPrefix = "flux.weave.works/" - // The namespace to presume if something doesn't have one, and we - // haven't been told what to use as a fallback - FallbackFallbackNamespace = "default" + ClusterScope = "" ) +// KubeManifest represents a manifest for a Kubernetes resource. For +// some Kubernetes-specific purposes we need more information that can +// be obtained from `resource.Resource`. +type KubeManifest interface { + resource.Resource + GroupVersion() string + GetKind() string + GetNamespace() string + SetNamespace(string) +} + // -- unmarshaling code for specific object and field types // struct to embed in objects, to provide default implementation type baseObject struct { - source string - fallbackNamespace string + source string + bytes []byte - bytes []byte - Kind string `yaml:"kind"` - Meta struct { + // these are present for unmarshalling into the struct + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Meta struct { Namespace string `yaml:"namespace"` Name string `yaml:"name"` Annotations map[string]string `yaml:"annotations,omitempty"` } `yaml:"metadata"` } +// GroupVersion implements KubeManifest.GroupVersion, so things with baseObject embedded are < KubeManifest +func (o baseObject) GroupVersion() string { + return o.APIVersion +} + +// GetNamespace implements KubeManifest.GetNamespace, so things embedding baseObject are < KubeManifest +func (o baseObject) GetNamespace() string { + return o.Meta.Namespace +} + +// GetKind implements KubeManifest.GetKind +func (o baseObject) GetKind() string { + return o.Kind +} + func (o baseObject) ResourceID() flux.ResourceID { ns := o.Meta.Namespace if ns == "" { - if o.fallbackNamespace == "" { - ns = FallbackFallbackNamespace - } else { - ns = o.fallbackNamespace - } + ns = ClusterScope } return flux.MakeResourceID(ns, o.Kind, o.Meta.Name) } +// SetNamespace implements KubeManifest.SetNamespace, so things with +// *baseObject embedded are < KubeManifest. NB pointer receiver. +func (o *baseObject) SetNamespace(ns string) { + o.Meta.Namespace = ns +} + // It's useful for comparisons in tests to be able to remove the // record of bytes func (o *baseObject) debyte() { @@ -79,8 +106,8 @@ func (o baseObject) Bytes() []byte { return o.bytes } -func unmarshalObject(source, fallbackNamespace string, bytes []byte) (resource.Resource, error) { - var base = baseObject{source: source, fallbackNamespace: fallbackNamespace, bytes: bytes} +func unmarshalObject(source string, bytes []byte) (KubeManifest, error) { + var base = baseObject{source: source, bytes: bytes} if err := yaml.Unmarshal(bytes, &base); err != nil { return nil, err } @@ -91,7 +118,7 @@ func unmarshalObject(source, fallbackNamespace string, bytes []byte) (resource.R return r, nil } -func unmarshalKind(base baseObject, bytes []byte) (resource.Resource, error) { +func unmarshalKind(base baseObject, bytes []byte) (KubeManifest, error) { switch base.Kind { case "CronJob": var cj = CronJob{baseObject: base} @@ -157,13 +184,13 @@ type rawList struct { func unmarshalList(base baseObject, raw *rawList, list *List) error { list.baseObject = base - list.Items = make([]resource.Resource, len(raw.Items), len(raw.Items)) + list.Items = make([]KubeManifest, len(raw.Items), len(raw.Items)) for i, item := range raw.Items { bytes, err := yaml.Marshal(item) if err != nil { return err } - res, err := unmarshalObject(base.source, base.fallbackNamespace, bytes) + res, err := unmarshalObject(base.source, bytes) if err != nil { return err } diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index 428c612bd..6b7b07303 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -31,7 +31,11 @@ const ( stackLabel = kresource.PolicyPrefix + "stack" checksumAnnotation = kresource.PolicyPrefix + "stack_checksum" - DefaultDefaultNamespace = "default" + // The namespace to presume if something doesn't have one, and we + // haven't been told what to use as a fallback. This is what + // `kubectl` uses when there's no config setting the fallback + // namespace. + DefaultFallbackNamespace = "default" ) // Sync takes a definition of what should be running in the cluster, @@ -42,8 +46,6 @@ const ( func (c *Cluster) Sync(spec cluster.SyncDef) error { logger := log.With(c.logger, "method", "Sync") - fallbackNamespace := c.applier.getDefaultNamespace() - type checksum struct { stack, sum string } @@ -53,7 +55,7 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { // NB we get all resources, since we care about leaving unsynced, // _ignored_ resources alone. - clusterResources, err := c.getResourcesBySelector("", fallbackNamespace) + clusterResources, err := c.getResourcesBySelector("") if err != nil { return errors.Wrap(err, "collating resources in cluster for sync") } @@ -97,7 +99,7 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { if c.GC { orphanedResources := makeChangeSet() - clusterResources, err := c.getResourcesInStack(fallbackNamespace) // <-- FIXME(right now) + clusterResources, err := c.getResourcesInStack() if err != nil { return errors.Wrap(err, "collating resources in cluster for calculating garbage collection") } @@ -140,28 +142,18 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { // --- internals in support of Sync type kuberesource struct { - obj *unstructured.Unstructured + obj *unstructured.Unstructured + namespaced bool } // ResourceID returns the ResourceID for this resource loaded from the // cluster. func (r *kuberesource) ResourceID() flux.ResourceID { ns, kind, name := r.obj.GetNamespace(), r.obj.GetKind(), r.obj.GetName() - return flux.MakeResourceID(ns, kind, name) -} - -// AssumedNamespaceResourceID returns a ResourceID which assumes the -// namespace is fallbackNamespace if it is missing. Resources returned -// from the cluster that are not namespaced (e.g., ClusterRoles) will -// have an empty string in the namespace field. To be able to compare -// them to resources we load from files, we must assume a namespace if -// none is given. -func (r *kuberesource) AssumedNamespaceResourceID(fallbackNamespace string) flux.ResourceID { - ns := r.obj.GetNamespace() - if ns == "" { - ns = fallbackNamespace + if !r.namespaced { + ns = kresource.ClusterScope } - return flux.MakeResourceID(ns, r.obj.GetKind(), r.obj.GetName()) + return flux.MakeResourceID(ns, kind, name) } // Bytes returns a byte slice description @@ -191,7 +183,7 @@ func (r *kuberesource) GetStack() string { return r.obj.GetLabels()[stackLabel] } -func (c *Cluster) getResourcesBySelector(selector, assumedNamespace string) (map[string]*kuberesource, error) { +func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesource, error) { listOptions := meta_v1.ListOptions{} if selector != "" { listOptions.LabelSelector = selector @@ -220,6 +212,8 @@ func (c *Cluster) getResourcesBySelector(selector, assumedNamespace string) (map continue } + namespaced := apiResource.Namespaced + // get group and version var group, version string groupVersion := resource.GroupVersion @@ -253,8 +247,8 @@ func (c *Cluster) getResourcesBySelector(selector, assumedNamespace string) (map } // TODO(michael) also exclude anything that has an ownerReference (that isn't "standard"?) - res := &kuberesource{obj: &data.Items[i]} - result[res.AssumedNamespaceResourceID(assumedNamespace).String()] = res + res := &kuberesource{obj: &data.Items[i], namespaced: namespaced} + result[res.ResourceID().String()] = res } } } @@ -264,8 +258,8 @@ func (c *Cluster) getResourcesBySelector(selector, assumedNamespace string) (map // exportResourcesInStack collates all the resources that belong to a // stack, i.e., were applied by flux. -func (c *Cluster) getResourcesInStack(assumedNamespace string) (map[string]*kuberesource, error) { - return c.getResourcesBySelector(stackLabel, assumedNamespace) // means "has label <>" +func (c *Cluster) getResourcesInStack() (map[string]*kuberesource, error) { + return c.getResourcesBySelector(stackLabel) // means "has label <>" } func applyMetadata(res resource.Resource, stack, checksum string) ([]byte, error) { @@ -322,7 +316,6 @@ func (c *changeSet) stage(cmd string, id flux.ResourceID, source string, bytes [ // Applier is something that will apply a changeset to the cluster. type Applier interface { apply(log.Logger, changeSet, map[flux.ResourceID]error) cluster.SyncError - getDefaultNamespace() string } type Kubectl struct { @@ -363,27 +356,27 @@ func (c *Kubectl) connectArgs() []string { return args } -// getDefaultNamespace returns the fallback namespace used by the +// GetDefaultNamespace returns the fallback namespace used by the // applied when a namespaced resource doesn't have one specified. This // is used when syncing to anticipate the identity of a resource in // the cluster given the manifest from a file (which may be missing // the namespace). -func (k *Kubectl) getDefaultNamespace() string { +func (k *Kubectl) GetDefaultNamespace() (string, error) { cmd := k.kubectlCommand("config", "get-contexts", "--no-headers") out, err := cmd.Output() if err != nil { - return DefaultDefaultNamespace + return "", err } lines := bytes.Split(out, []byte("\n")) for _, line := range lines { words := bytes.Fields(line) if len(words) > 1 && string(words[0]) == "*" { if len(words) == 5 { - return string(words[4]) + return string(words[4]), nil } } } - return DefaultDefaultNamespace + return DefaultFallbackNamespace, nil } // rankOfKind returns an int denoting the position of the given kind diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index 3f8a2b620..b0f26ae47 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -32,7 +32,7 @@ import ( ) const ( - defaultNamespace = "unusual-default" + defaultTestNamespace = "unusual-default" ) func fakeClients() extendedClient { @@ -61,7 +61,7 @@ func fakeClients() extendedClient { }, } - coreClient := corefake.NewSimpleClientset(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: defaultNamespace}}) + coreClient := corefake.NewSimpleClientset(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: defaultTestNamespace}}) fluxClient := fluxfake.NewSimpleClientset() dynamicClient := NewSimpleDynamicClient(scheme) // NB from this package, rather than the official one, since we needed a patched version @@ -100,10 +100,6 @@ type fakeApplier struct { commandRun bool } -func (a fakeApplier) getDefaultNamespace() string { - return defaultNamespace -} - func groupVersionResource(res *unstructured.Unstructured) schema.GroupVersionResource { gvk := res.GetObjectKind().GroupVersionKind() return schema.GroupVersionResource{Group: gvk.Group, Version: gvk.Version, Resource: strings.ToLower(gvk.Kind) + "s"} @@ -205,7 +201,7 @@ func findAPIResource(gvr schema.GroupVersionResource, disco discovery.DiscoveryI func setup(t *testing.T) (*Cluster, *fakeApplier) { clients := fakeClients() - applier := &fakeApplier{client: clients.dynamicClient, discovery: clients.coreClient.Discovery(), defaultNS: defaultNamespace} + applier := &fakeApplier{client: clients.dynamicClient, discovery: clients.coreClient.Discovery(), defaultNS: defaultTestNamespace} kube := &Cluster{ applier: applier, client: clients, @@ -265,7 +261,18 @@ metadata: } test := func(t *testing.T, kube *Cluster, defs, expectedAfterSync string, expectErrors bool) { - resources, err := kresource.ParseMultidoc([]byte(defs), "before", defaultNamespace) + namespacer, err := NewNamespacer(namespaceDefaulterFake(defaultTestNamespace), kube.client.coreClient.Discovery()) + if err != nil { + t.Fatal(err) + } + + resources0, err := kresource.ParseMultidoc([]byte(defs), "before") + if err != nil { + t.Fatal(err) + } + + // Needed to get from KubeManifest to resource.Resource + resources, err := postProcess(resources0, namespacer) if err != nil { t.Fatal(err) } @@ -274,13 +281,13 @@ metadata: if !expectErrors && err != nil { t.Error(err) } - expected, err := kresource.ParseMultidoc([]byte(expectedAfterSync), "after", defaultNamespace) + expected, err := kresource.ParseMultidoc([]byte(expectedAfterSync), "after") if err != nil { panic(err) } // Now check that the resources were created - actual, err := kube.getResourcesInStack(defaultNamespace) + actual, err := kube.getResourcesInStack() if err != nil { t.Fatal(err) } @@ -354,7 +361,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: depFallbackNS - namespace: ` + defaultNamespace + ` + namespace: ` + defaultTestNamespace + ` ` test(t, kube, withoutNS, withNS, false) }) @@ -495,7 +502,7 @@ spec: assert.NoError(t, err) // Check that our resource-getting also sees the pre-existing resource - resources, err := kube.getResourcesBySelector("", defaultNamespace) + resources, err := kube.getResourcesBySelector("") assert.NoError(t, err) assert.Contains(t, resources, "foobar:deployment/dep1") diff --git a/cluster/mock.go b/cluster/mock.go index 629776238..d8c51457d 100644 --- a/cluster/mock.go +++ b/cluster/mock.go @@ -18,7 +18,6 @@ type Mock struct { PublicSSHKeyFunc func(regenerate bool) (ssh.PublicKey, error) UpdateImageFunc func(def []byte, id flux.ResourceID, container string, newImageID image.Ref) ([]byte, error) LoadManifestsFunc func(base string, paths []string) (map[string]resource.Resource, error) - UpdateManifestFunc func(path, resourceID string, f func(def []byte) ([]byte, error)) error UpdatePoliciesFunc func([]byte, flux.ResourceID, policy.Update) ([]byte, error) } @@ -54,10 +53,6 @@ func (m *Mock) LoadManifests(base string, paths []string) (map[string]resource.R return m.LoadManifestsFunc(base, paths) } -func (m *Mock) UpdateManifest(path string, resourceID string, f func(def []byte) ([]byte, error)) error { - return m.UpdateManifestFunc(path, resourceID, f) -} - func (m *Mock) UpdatePolicies(def []byte, id flux.ResourceID, p policy.Update) ([]byte, error) { return m.UpdatePoliciesFunc(def, id, p) } diff --git a/cmd/fluxd/main.go b/cmd/fluxd/main.go index 15e9370bf..92e0497cc 100644 --- a/cmd/fluxd/main.go +++ b/cmd/fluxd/main.go @@ -193,7 +193,7 @@ func main() { var clusterVersion string var sshKeyRing ssh.KeyRing var k8s cluster.Cluster - var k8sManifests cluster.Manifests + var k8sManifests *kubernetes.Manifests var imageCreds func() registry.ImageCreds { restClientConfig, err := rest.InClusterConfig() @@ -283,6 +283,11 @@ func main() { // There is only one way we currently interpret a repo of // files as manifests, and that's as Kubernetes yamels. k8sManifests = &kubernetes.Manifests{} + k8sManifests.Namespacer, err = kubernetes.NewNamespacer(kubectlApplier, clientset.Discovery()) + if err != nil { + logger.Log("err", err) + os.Exit(1) + } } // Wrap the procedure for collecting images to scan diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go index 13328ca9b..3deb4531d 100644 --- a/daemon/daemon_test.go +++ b/daemon/daemon_test.go @@ -20,6 +20,7 @@ import ( "github.com/weaveworks/flux/api/v9" "github.com/weaveworks/flux/cluster" "github.com/weaveworks/flux/cluster/kubernetes" + kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" "github.com/weaveworks/flux/event" "github.com/weaveworks/flux/git" "github.com/weaveworks/flux/git/gittest" @@ -594,6 +595,16 @@ func mustParseImageRef(ref string) image.Ref { return r } +type anonNamespacer func(kresource.KubeManifest) string + +func (fn anonNamespacer) EffectiveNamespace(m kresource.KubeManifest) (string, error) { + return fn(m), nil +} + +var alwaysDefault anonNamespacer = func(kresource.KubeManifest) string { + return "default" +} + func mockDaemon(t *testing.T) (*Daemon, func(), func(), *cluster.Mock, *mockEventWriter, func(func())) { logger := log.NewNopLogger() @@ -646,7 +657,6 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *cluster.Mock, *mockEven return []cluster.Controller{}, nil } k8s.ExportFunc = func() ([]byte, error) { return testBytes, nil } - k8s.LoadManifestsFunc = (&kubernetes.Manifests{}).LoadManifests k8s.PingFunc = func() error { return nil } k8s.SomeServicesFunc = func([]flux.ResourceID) ([]cluster.Controller, error) { return []cluster.Controller{ @@ -654,8 +664,6 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *cluster.Mock, *mockEven }, nil } k8s.SyncFunc = func(def cluster.SyncDef) error { return nil } - k8s.UpdatePoliciesFunc = (&kubernetes.Manifests{}).UpdatePolicies - k8s.UpdateImageFunc = (&kubernetes.Manifests{}).UpdateImage } var imageRegistry registry.Registry @@ -690,7 +698,7 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *cluster.Mock, *mockEven Repo: repo, GitConfig: params, Cluster: k8s, - Manifests: &kubernetes.Manifests{}, + Manifests: &kubernetes.Manifests{Namespacer: alwaysDefault}, Registry: imageRegistry, V: testVersion, Jobs: jobs, diff --git a/daemon/loop_test.go b/daemon/loop_test.go index 6a426eb3f..5088774fc 100644 --- a/daemon/loop_test.go +++ b/daemon/loop_test.go @@ -41,7 +41,6 @@ func daemon(t *testing.T) (*Daemon, func()) { repo, repoCleanup := gittest.Repo(t) k8s = &cluster.Mock{} - k8s.LoadManifestsFunc = (&kubernetes.Manifests{}).LoadManifests k8s.ExportFunc = func() ([]byte, error) { return nil, nil } events = &mockEventWriter{} @@ -64,7 +63,7 @@ func daemon(t *testing.T) (*Daemon, func()) { jobs := job.NewQueue(shutdown, wg) d := &Daemon{ Cluster: k8s, - Manifests: k8s, + Manifests: &kubernetes.Manifests{Namespacer: alwaysDefault}, Registry: ®istryMock.Registry{}, Repo: repo, GitConfig: gitConfig, @@ -231,7 +230,7 @@ func TestDoSync_WithNewCommit(t *testing.T) { } // Push some new changes dirs := checkout.ManifestDirs() - err = cluster.UpdateManifest(k8s, checkout.Dir(), dirs, flux.MustParseResourceID("default:deployment/helloworld"), func(def []byte) ([]byte, error) { + err = cluster.UpdateManifest(d.Manifests, checkout.Dir(), dirs, flux.MustParseResourceID("default:deployment/helloworld"), func(def []byte) ([]byte, error) { // A simple modification so we have changes to push return []byte(strings.Replace(string(def), "replicas: 5", "replicas: 4", -1)), nil }) diff --git a/release/releaser_test.go b/release/releaser_test.go index faa9e7520..ac001f1a9 100644 --- a/release/releaser_test.go +++ b/release/releaser_test.go @@ -13,6 +13,7 @@ import ( "github.com/weaveworks/flux" "github.com/weaveworks/flux/cluster" "github.com/weaveworks/flux/cluster/kubernetes" + kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" "github.com/weaveworks/flux/git" "github.com/weaveworks/flux/git/gittest" "github.com/weaveworks/flux/image" @@ -21,6 +22,12 @@ import ( "github.com/weaveworks/flux/update" ) +type constNamespacer string + +func (ns constNamespacer) EffectiveNamespace(kresource.KubeManifest) (string, error) { + return string(ns), nil +} + var ( // This must match the value in cluster/kubernetes/testfiles/data.go helloContainer = "greeter" @@ -135,7 +142,7 @@ var ( }, }, } - mockManifests = &kubernetes.Manifests{} + mockManifests = &kubernetes.Manifests{Namespacer: constNamespacer("default")} ) func mockCluster(running ...cluster.Controller) *cluster.Mock { @@ -1084,7 +1091,7 @@ func Test_BadRelease(t *testing.T) { ctx = &ReleaseContext{ cluster: cluster, - manifests: &badManifests{Manifests: kubernetes.Manifests{}}, + manifests: &badManifests{Manifests: kubernetes.Manifests{constNamespacer("default")}}, repo: checkout2, registry: mockRegistry, } From ce3699838c4739664f7beaa088a3d5e5599728be Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 7 Feb 2019 17:39:18 +0000 Subject: [PATCH 15/24] Invalidate discovery cache when custom resources change The idea is that only custom resource definitions will change in a running cluster, so just refresh our idea of namespacedness when that happens. --- Gopkg.lock | 22 ++++- Gopkg.toml | 4 + cluster/kubernetes/namespacer.go | 125 +++++++++++++++++++++++--- cluster/kubernetes/namespacer_test.go | 111 ++++++++++++++++++++++- cluster/kubernetes/sync_test.go | 2 +- cmd/fluxd/main.go | 60 +++++++------ 6 files changed, 278 insertions(+), 46 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 13a20ad06..7deddeb6b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -830,12 +830,21 @@ version = "kubernetes-1.11.0" [[projects]] - branch = "master" - digest = "1:0a865e7f317907161f1c2df5f270595b9d7ba09850b7eb39a2b11cc8f9a50d28" + digest = "1:26842d8ec9a7f675635f1c2248991f190610109c446f8b8f405be916d8241f12" name = "k8s.io/apiextensions-apiserver" - packages = ["pkg/features"] + packages = [ + "pkg/apis/apiextensions", + "pkg/apis/apiextensions/v1beta1", + "pkg/client/clientset/clientset", + "pkg/client/clientset/clientset/fake", + "pkg/client/clientset/clientset/scheme", + "pkg/client/clientset/clientset/typed/apiextensions/v1beta1", + "pkg/client/clientset/clientset/typed/apiextensions/v1beta1/fake", + "pkg/features", + ] pruneopts = "" - revision = "84f7c7786e298ad1479d00ff314a2cfa0006cd0c" + revision = "3de98c57bc05a81cf463e0ad7a0af4cec8a5b510" + version = "kubernetes-1.11.0" [[projects]] digest = "1:b6b2fb7b4da1ac973b64534ace2299a02504f16bc7820cb48edb8ca4077183e1" @@ -906,6 +915,7 @@ name = "k8s.io/client-go" packages = [ "discovery", + "discovery/cached", "discovery/fake", "dynamic", "kubernetes", @@ -1206,6 +1216,9 @@ "k8s.io/api/apps/v1", "k8s.io/api/batch/v1beta1", "k8s.io/api/core/v1", + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1", + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset", + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake", "k8s.io/apimachinery/pkg/api/errors", "k8s.io/apimachinery/pkg/api/meta", "k8s.io/apimachinery/pkg/apis/meta/v1", @@ -1219,6 +1232,7 @@ "k8s.io/apimachinery/pkg/util/wait", "k8s.io/apimachinery/pkg/watch", "k8s.io/client-go/discovery", + "k8s.io/client-go/discovery/cached", "k8s.io/client-go/discovery/fake", "k8s.io/client-go/dynamic", "k8s.io/client-go/kubernetes", diff --git a/Gopkg.toml b/Gopkg.toml index 2479e105b..4d09c1c77 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -26,6 +26,10 @@ required = ["k8s.io/code-generator/cmd/client-gen"] name = "k8s.io/apimachinery" version = "kubernetes-1.11.0" +[[constraint]] + name = "k8s.io/apiextensions-apiserver" + version = "kubernetes-1.11.0" + [[constraint]] name = "k8s.io/client-go" version = "8.0.0" diff --git a/cluster/kubernetes/namespacer.go b/cluster/kubernetes/namespacer.go index 07b2397f3..51074a4ab 100644 --- a/cluster/kubernetes/namespacer.go +++ b/cluster/kubernetes/namespacer.go @@ -2,9 +2,17 @@ package kubernetes import ( "fmt" + "sync" + "time" + crdv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + crd "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" - "k8s.io/client-go/discovery/cached" + discocache "k8s.io/client-go/discovery/cached" + toolscache "k8s.io/client-go/tools/cache" kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" ) @@ -24,19 +32,14 @@ func NewNamespacer(ns namespaceDefaulter, d discovery.DiscoveryInterface) (*name if err != nil { return nil, err } - cachedDisco := cached.NewMemCacheClient(d) - // in client-go v9, the call of Invalidate is necessary to force - // it to query for initial data; in subsequent versions, it is a - // no-op reset of the cache validity, so safe to call. - cachedDisco.Invalidate() - return &namespaceViaDiscovery{fallbackNamespace: fallback, disco: cachedDisco}, nil + return &namespaceViaDiscovery{fallbackNamespace: fallback, disco: d}, nil } // effectiveNamespace yields the namespace that would be used for this // resource were it applied, taking into account the kind of the // resource, and local configuration. func (n *namespaceViaDiscovery) EffectiveNamespace(m kresource.KubeManifest) (string, error) { - namespaced, err := n.lookupNamespaced(m) + namespaced, err := n.lookupNamespaced(m.GroupVersion(), m.GetKind()) switch { case err != nil: return "", err @@ -48,11 +51,10 @@ func (n *namespaceViaDiscovery) EffectiveNamespace(m kresource.KubeManifest) (st return m.GetNamespace(), nil } -func (n *namespaceViaDiscovery) lookupNamespaced(m kresource.KubeManifest) (bool, error) { - groupVersion, kind := m.GroupVersion(), m.GetKind() +func (n *namespaceViaDiscovery) lookupNamespaced(groupVersion, kind string) (bool, error) { resourceList, err := n.disco.ServerResourcesForGroupVersion(groupVersion) if err != nil { - return false, err + return false, fmt.Errorf("error looking up API resources for %s.%s: %s", kind, groupVersion, err.Error()) } for _, resource := range resourceList.APIResources { if resource.Kind == kind { @@ -61,3 +63,104 @@ func (n *namespaceViaDiscovery) lookupNamespaced(m kresource.KubeManifest) (bool } return false, fmt.Errorf("resource not found for API %s, kind %s", groupVersion, kind) } + +// This mainly exists so I can put the controller and store in here, +// so as to have access to them for testing; and, so that we can do +// our own invalidation. +type cachedDiscovery struct { + discovery.CachedDiscoveryInterface + + store toolscache.Store + controller toolscache.Controller + + invalidMu sync.Mutex + invalid bool +} + +// The older (v8.0.0) implementation of MemCacheDiscovery refreshes +// the cached values, synchronously, when Invalidate is called. Since +// we will invalidate every time something cahnges, but it only +// matters when we want to read the cached values, this method (and +// ServerResourcesForGroupVersion) saves the invalidation for when a +// read is done. +func (d *cachedDiscovery) Invalidate() { + d.invalidMu.Lock() + d.invalid = true + d.invalidMu.Unlock() +} + +// This happens to be the method that we call in the namespacer; so, +// this is the one where we check whether the cache has been +// invalidated. A cachedDiscovery implementation for more general use +// would do this for all methods (that weren't implemented purely in +// terms of other methods). +func (d *cachedDiscovery) ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) { + d.invalidMu.Lock() + invalid := d.invalid + d.invalid = false + d.invalidMu.Unlock() + if invalid { + d.CachedDiscoveryInterface.Invalidate() + } + return d.CachedDiscoveryInterface.ServerResourcesForGroupVersion(groupVersion) +} + +type chainHandler struct { + first toolscache.ResourceEventHandler + next toolscache.ResourceEventHandler +} + +func (h chainHandler) OnAdd(obj interface{}) { + h.first.OnAdd(obj) + h.next.OnAdd(obj) +} + +func (h chainHandler) OnUpdate(old, new interface{}) { + h.first.OnUpdate(old, new) + h.next.OnUpdate(old, new) +} + +func (h chainHandler) OnDelete(old interface{}) { + h.first.OnDelete(old) + h.next.OnDelete(old) +} + +// MakeCachedDiscovery constructs a CachedDicoveryInterface that will +// be invalidated whenever the set of CRDs change. The idea is that +// the only avenue of a change to the API resources in a running +// system is CRDs being added, updated or deleted. The prehandlers are +// there to allow us to put extra synchronisation in, for testing. +func MakeCachedDiscovery(d discovery.DiscoveryInterface, c crd.Interface, shutdown <-chan struct{}, prehandlers ...toolscache.ResourceEventHandler) *cachedDiscovery { + cachedDisco := &cachedDiscovery{CachedDiscoveryInterface: discocache.NewMemCacheClient(d)} + // We have an empty cache, so it's _a priori_ invalid. (Yes, that's the zero value, but better safe than sorry) + cachedDisco.Invalidate() + + crdClient := c.ApiextensionsV1beta1().CustomResourceDefinitions() + + var handler toolscache.ResourceEventHandler = toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(_ interface{}) { + cachedDisco.Invalidate() + }, + UpdateFunc: func(_, _ interface{}) { + cachedDisco.Invalidate() + }, + DeleteFunc: func(_ interface{}) { + cachedDisco.Invalidate() + }, + } + for _, h := range prehandlers { + handler = chainHandler{first: h, next: handler} + } + + lw := &toolscache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return crdClient.List(options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return crdClient.Watch(options) + }, + } + cachedDisco.store, cachedDisco.controller = toolscache.NewInformer(lw, &crdv1beta1.CustomResourceDefinition{}, 5*time.Minute, handler) + go cachedDisco.controller.Run(shutdown) + return cachedDisco +} diff --git a/cluster/kubernetes/namespacer_test.go b/cluster/kubernetes/namespacer_test.go index 237d0465f..4e3d8ce51 100644 --- a/cluster/kubernetes/namespacer_test.go +++ b/cluster/kubernetes/namespacer_test.go @@ -2,9 +2,13 @@ package kubernetes import ( "testing" + "time" + crdv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + crdfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corefake "k8s.io/client-go/kubernetes/fake" + toolscache "k8s.io/client-go/tools/cache" kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" ) @@ -15,9 +19,9 @@ func (ns namespaceDefaulterFake) GetDefaultNamespace() (string, error) { return string(ns), nil } -func TestNamespaceDefaulting(t *testing.T) { +var getAndList = metav1.Verbs([]string{"get", "list"}) - getAndList := metav1.Verbs([]string{"get", "list"}) +func makeFakeClient() *corefake.Clientset { apiResources := []*metav1.APIResourceList{ { GroupVersion: "apps/v1", @@ -35,8 +39,12 @@ func TestNamespaceDefaulting(t *testing.T) { coreClient := corefake.NewSimpleClientset() coreClient.Fake.Resources = apiResources - disco := coreClient.Discovery() - nser, err := NewNamespacer(namespaceDefaulterFake("fallback-ns"), disco) + return coreClient +} + +func TestNamespaceDefaulting(t *testing.T) { + coreClient := makeFakeClient() + nser, err := NewNamespacer(namespaceDefaulterFake("fallback-ns"), coreClient.Discovery()) if err != nil { t.Fatal(err) } @@ -85,3 +93,98 @@ metadata: assertEffectiveNamespace(":deployment/noNamespace", "fallback-ns") assertEffectiveNamespace("spurious:namespace/notNamespaced", "") } + +func TestCachedDiscovery(t *testing.T) { + coreClient := makeFakeClient() + + myCRD := &crdv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom", + }, + } + crdClient := crdfake.NewSimpleClientset(myCRD) + + // Here's my fake API resource + myAPI := &metav1.APIResourceList{ + GroupVersion: "foo/v1", + APIResources: []metav1.APIResource{ + {Name: "customs", SingularName: "custom", Namespaced: true, Kind: "Custom", Verbs: getAndList}, + }, + } + + apiResources := coreClient.Fake.Resources + coreClient.Fake.Resources = append(apiResources, myAPI) + + shutdown := make(chan struct{}) + defer close(shutdown) + + // this extra handler means we can synchronise on the add later + // being processed + allowAdd := make(chan interface{}) + handler := toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + allowAdd <- obj + }, + } + + cachedDisco := MakeCachedDiscovery(coreClient.Discovery(), crdClient, shutdown, handler) + + namespacer, err := NewNamespacer(namespaceDefaulterFake("bar-ns"), cachedDisco) + if err != nil { + t.Fatal(err) + } + + namespaced, err := namespacer.lookupNamespaced("foo/v1", "Custom") + if err != nil { + t.Fatal(err) + } + if !namespaced { + t.Error("got false from lookupNamespaced, expecting true") + } + + // In a cluster, we'd rely on the apiextensions server to reflect + // changes to CRDs to changes in the API resources. Here I will be + // more narrow, and just test that the API resources are reloaded + // when a CRD is updated or deleted. + + // This is delicate: we can't just change the value in-place, + // since that will update everyone's record of it, and the test + // below will trivially succeed. + updatedAPI := &metav1.APIResourceList{ + GroupVersion: "foo/v1", + APIResources: []metav1.APIResource{ + {Name: "customs", SingularName: "custom", Namespaced: false /* <-- changed */, Kind: "Custom", Verbs: getAndList}, + }, + } + coreClient.Fake.Resources = append(apiResources, updatedAPI) + + // Provoke the cached discovery client into invalidating + _, err = crdClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(myCRD) + if err != nil { + t.Fatal(err) + } + + // Wait for the update to "go through" + select { + case <-allowAdd: + break + case <-time.After(time.Second): + t.Fatal("timed out waiting for Add to happen") + } + + _, exists, err := cachedDisco.store.Get(myCRD) + if err != nil { + t.Error(err) + } + if !exists { + t.Error("does not exist") + } + + namespaced, err = namespacer.lookupNamespaced("foo/v1", "Custom") + if err != nil { + t.Fatal(err) + } + if namespaced { + t.Error("got true from lookupNamespaced, expecting false (after changing it)") + } +} diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index b0f26ae47..9bdeef7de 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -40,7 +40,7 @@ func fakeClients() extendedClient { // Set this to `true` to output a trace of the API actions called // while running the tests - const debug = true + const debug = false getAndList := metav1.Verbs([]string{"get", "list"}) // Adding these means the fake dynamic client will find them, and diff --git a/cmd/fluxd/main.go b/cmd/fluxd/main.go index 92e0497cc..58b644c75 100644 --- a/cmd/fluxd/main.go +++ b/cmd/fluxd/main.go @@ -18,6 +18,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/pflag" k8sifclient "github.com/weaveworks/flux/integrations/client/clientset/versioned" + crd "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" k8sclientdynamic "k8s.io/client-go/dynamic" k8sclient "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -189,6 +190,31 @@ func main() { *sshKeygenDir = *k8sSecretVolumeMountPath } + // Mechanical components. + + // When we can receive from this channel, it indicates that we + // are ready to shut down. + errc := make(chan error) + // This signals other routines to shut down; + shutdown := make(chan struct{}) + // .. and this is to wait for other routines to shut down cleanly. + shutdownWg := &sync.WaitGroup{} + + go func() { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) + errc <- fmt.Errorf("%s", <-c) + }() + + // This means we can return, and it will use the shutdown + // protocol. + defer func() { + // wait here until stopping. + logger.Log("exiting", <-errc) + close(shutdown) + shutdownWg.Wait() + }() + // Cluster component. var clusterVersion string var sshKeyRing ssh.KeyRing @@ -283,7 +309,14 @@ func main() { // There is only one way we currently interpret a repo of // files as manifests, and that's as Kubernetes yamels. k8sManifests = &kubernetes.Manifests{} - k8sManifests.Namespacer, err = kubernetes.NewNamespacer(kubectlApplier, clientset.Discovery()) + + crdClient, err := crd.NewForConfig(restClientConfig) + if err == nil { + disco := kubernetes.MakeCachedDiscovery(clientset.Discovery(), crdClient, shutdown) + k8sManifests.Namespacer, err = kubernetes.NewNamespacer(kubectlApplier, disco) + } + if err == nil { + } if err != nil { logger.Log("err", err) os.Exit(1) @@ -368,31 +401,6 @@ func main() { } } - // Mechanical components. - - // When we can receive from this channel, it indicates that we - // are ready to shut down. - errc := make(chan error) - // This signals other routines to shut down; - shutdown := make(chan struct{}) - // .. and this is to wait for other routines to shut down cleanly. - shutdownWg := &sync.WaitGroup{} - - go func() { - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) - errc <- fmt.Errorf("%s", <-c) - }() - - // This means we can return, and it will use the shutdown - // protocol. - defer func() { - // wait here until stopping. - logger.Log("exiting", <-errc) - close(shutdown) - shutdownWg.Wait() - }() - // Checkpoint: we want to include the fact of whether the daemon // was given a Git repo it could clone; but the expected scenario // is that it will have been set up already, and we don't want to From 927f9f3f2da74a4f3cc1592d73bcc793a334d608 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Wed, 20 Feb 2019 18:02:17 +0000 Subject: [PATCH 16/24] Simplify sync structures Sync a single `SyncSet` (used to be `SyncStack`) at a time, since that's all we need for now. This commit renames various things, along the lines of Stack->SyncSet. --- cluster/cluster.go | 2 +- cluster/kubernetes/sync.go | 88 ++++++++++++++++----------------- cluster/kubernetes/sync_test.go | 6 +-- cluster/mock.go | 4 +- cluster/sync.go | 21 +++----- daemon/daemon_test.go | 6 +-- daemon/loop.go | 3 +- daemon/loop_test.go | 12 ++--- sync/sync.go | 22 +++------ sync/sync_test.go | 12 ++--- 10 files changed, 80 insertions(+), 96 deletions(-) diff --git a/cluster/cluster.go b/cluster/cluster.go index 78eae6870..b97278127 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -29,7 +29,7 @@ type Cluster interface { SomeControllers([]flux.ResourceID) ([]Controller, error) Ping() error Export() ([]byte, error) - Sync(SyncDef) error + Sync(SyncSet) error PublicSSHKey(regenerate bool) (ssh.PublicKey, error) } diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index 6b7b07303..9d5ae0b78 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -28,8 +28,8 @@ import ( ) const ( - stackLabel = kresource.PolicyPrefix + "stack" - checksumAnnotation = kresource.PolicyPrefix + "stack_checksum" + syncSetLabel = kresource.PolicyPrefix + "sync-set" + checksumAnnotation = kresource.PolicyPrefix + "sync-checksum" // The namespace to presume if something doesn't have one, and we // haven't been told what to use as a fallback. This is what @@ -43,15 +43,12 @@ const ( // necessarily indicate complete failure; some resources may succeed // in being synced, and some may fail (for example, they may be // malformed). -func (c *Cluster) Sync(spec cluster.SyncDef) error { +func (c *Cluster) Sync(spec cluster.SyncSet) error { logger := log.With(c.logger, "method", "Sync") - type checksum struct { - stack, sum string - } - // Keep track of the checksum of each stack, so we can compare + // Keep track of the checksum of each resource, so we can compare // them during garbage collection. - checksums := map[string]checksum{} + checksums := map[string]string{} // NB we get all resources, since we care about leaving unsynced, // _ignored_ resources alone. @@ -62,29 +59,30 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { cs := makeChangeSet() var errs cluster.SyncError - for _, stack := range spec.Stacks { - for _, res := range stack.Resources { - id := res.ResourceID().String() - // make a record of the checksum, whether we stage it to - // be applied or not, so that we don't delete it later. - csum := sha1.Sum(res.Bytes()) - checkHex := hex.EncodeToString(csum[:]) - checksums[id] = checksum{stack.Name, checkHex} - if res.Policy().Has(policy.Ignore) { - logger.Log("info", "not applying resource; ignore annotation in file", "resource", res.ResourceID(), "source", res.Source()) - continue - } - if cres, ok := clusterResources[id]; ok && cres.Policy().Has(policy.Ignore) { - logger.Log("info", "not applying resource; ignore annotation in cluster resource", "resource", cres.ResourceID()) - continue - } - resBytes, err := applyMetadata(res, stack.Name, checkHex) - if err == nil { - cs.stage("apply", res.ResourceID(), res.Source(), resBytes) - } else { - errs = append(errs, cluster.ResourceError{ResourceID: res.ResourceID(), Source: res.Source(), Error: err}) - break - } + for _, res := range spec.Resources { + id := res.ResourceID().String() + // make a record of the checksum, whether we stage it to + // be applied or not, so that we don't delete it later. + csum := sha1.Sum(res.Bytes()) + checkHex := hex.EncodeToString(csum[:]) + checksums[id] = checkHex + if res.Policy().Has(policy.Ignore) { + logger.Log("info", "not applying resource; ignore annotation in file", "resource", res.ResourceID(), "source", res.Source()) + continue + } + // It's possible to give a cluster resource the "ignore" + // annotation directly -- e.g., with `kubectl annotate` -- so + // we need to examine the cluster resource here too. + if cres, ok := clusterResources[id]; ok && cres.Policy().Has(policy.Ignore) { + logger.Log("info", "not applying resource; ignore annotation in cluster resource", "resource", cres.ResourceID()) + continue + } + resBytes, err := applyMetadata(res, spec.Name, checkHex) + if err == nil { + cs.stage("apply", res.ResourceID(), res.Source(), resBytes) + } else { + errs = append(errs, cluster.ResourceError{ResourceID: res.ResourceID(), Source: res.Source(), Error: err}) + break } } @@ -99,27 +97,25 @@ func (c *Cluster) Sync(spec cluster.SyncDef) error { if c.GC { orphanedResources := makeChangeSet() - clusterResources, err := c.getResourcesInStack() + clusterResources, err := c.getResourcesInSyncSet(spec.Name) if err != nil { return errors.Wrap(err, "collating resources in cluster for calculating garbage collection") } for resourceID, res := range clusterResources { - actual := checksum{res.GetStack(), res.GetChecksum()} + actual := res.GetChecksum() expected, ok := checksums[resourceID] switch { case !ok: // was not recorded as having been staged for application c.logger.Log("info", "cluster resource not in resources to be synced; deleting", "resource", resourceID) orphanedResources.stage("delete", res.ResourceID(), "", res.Bytes()) - case actual.stack == "": // the label has been removed, out of band (or due to a bug). Best to leave it. - c.logger.Log("warning", "cluster resource with empty stack label; skipping", "resource", resourceID) - continue case actual != expected: c.logger.Log("warning", "resource to be synced has not been updated; skipping", "resource", resourceID) continue default: - // The stack and checksum are the same, indicating that it was applied earlier. Leave it alone. + // The checksum is the same, indicating that it was + // applied earlier. Leave it alone. } } @@ -177,10 +173,10 @@ func (r *kuberesource) GetChecksum() string { return r.obj.GetAnnotations()[checksumAnnotation] } -// GetStack returns the stack recorded on the the resource from -// Kubernetes, or an empty string if it's not present. -func (r *kuberesource) GetStack() string { - return r.obj.GetLabels()[stackLabel] +// GetSyncSet returns the sync set name recorded on the the resource +// from Kubernetes, or an empty string if it's not present. +func (r *kuberesource) GetSyncSet() string { + return r.obj.GetLabels()[syncSetLabel] } func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesource, error) { @@ -258,11 +254,11 @@ func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesou // exportResourcesInStack collates all the resources that belong to a // stack, i.e., were applied by flux. -func (c *Cluster) getResourcesInStack() (map[string]*kuberesource, error) { - return c.getResourcesBySelector(stackLabel) // means "has label <>" +func (c *Cluster) getResourcesInSyncSet(name string) (map[string]*kuberesource, error) { + return c.getResourcesBySelector(fmt.Sprintf("%s=%s", syncSetLabel, name)) // means "has label <>" } -func applyMetadata(res resource.Resource, stack, checksum string) ([]byte, error) { +func applyMetadata(res resource.Resource, set, checksum string) ([]byte, error) { definition := map[interface{}]interface{}{} if err := yaml.Unmarshal(res.Bytes(), &definition); err != nil { return nil, errors.Wrap(err, fmt.Sprintf("failed to parse yaml from %s", res.Source())) @@ -270,9 +266,9 @@ func applyMetadata(res resource.Resource, stack, checksum string) ([]byte, error mixin := map[string]interface{}{} - if stack != "" { + if set != "" { mixinLabels := map[string]string{} - mixinLabels[stackLabel] = stack + mixinLabels[syncSetLabel] = set mixin["labels"] = mixinLabels } diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index 9bdeef7de..53282becf 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -212,7 +212,7 @@ func setup(t *testing.T) (*Cluster, *fakeApplier) { func TestSyncNop(t *testing.T) { kube, mock := setup(t) - if err := kube.Sync(cluster.SyncDef{}); err != nil { + if err := kube.Sync(cluster.SyncSet{}); err != nil { t.Errorf("%#v", err) } if mock.commandRun { @@ -277,7 +277,7 @@ metadata: t.Fatal(err) } - err = sync.Sync(resources, kube) + err = sync.Sync("testset", resources, kube) if !expectErrors && err != nil { t.Error(err) } @@ -287,7 +287,7 @@ metadata: } // Now check that the resources were created - actual, err := kube.getResourcesInStack() + actual, err := kube.getResourcesInSyncSet("testset") if err != nil { t.Fatal(err) } diff --git a/cluster/mock.go b/cluster/mock.go index d8c51457d..2035146ff 100644 --- a/cluster/mock.go +++ b/cluster/mock.go @@ -14,7 +14,7 @@ type Mock struct { SomeServicesFunc func([]flux.ResourceID) ([]Controller, error) PingFunc func() error ExportFunc func() ([]byte, error) - SyncFunc func(SyncDef) error + SyncFunc func(SyncSet) error PublicSSHKeyFunc func(regenerate bool) (ssh.PublicKey, error) UpdateImageFunc func(def []byte, id flux.ResourceID, container string, newImageID image.Ref) ([]byte, error) LoadManifestsFunc func(base string, paths []string) (map[string]resource.Resource, error) @@ -37,7 +37,7 @@ func (m *Mock) Export() ([]byte, error) { return m.ExportFunc() } -func (m *Mock) Sync(c SyncDef) error { +func (m *Mock) Sync(c SyncSet) error { return m.SyncFunc(c) } diff --git a/cluster/sync.go b/cluster/sync.go index 6588bf0fe..ebe3aab9e 100644 --- a/cluster/sync.go +++ b/cluster/sync.go @@ -9,23 +9,18 @@ import ( // Definitions for use in synchronising a cluster with a git repo. -// SyncStack groups a set of resources to be updated. The purpose of -// the grouping is to limit the "blast radius" of changes. For -// example, if we calculate a checksum for each stack and annotate the -// resources within it, any single change will affect only the -// resources in the same stack, meaning fewer things to annotate. (So -// why not do these individually? This, too, can be expensive, since -// it involves examining each resource individually). -type SyncStack struct { +// SyncSet groups the set of resources to be updated. Usually this is +// the set of resources found in a git repo; in any case, it must +// represent the complete set of resources, as garbage collection will +// assume missing resources should be deleted. The name is used to +// distinguish the resources from a set from other resources -- e.g., +// cluster resources not marked as belonging to a set will not be +// deleted by garbage collection. +type SyncSet struct { Name string Resources []resource.Resource } -type SyncDef struct { - // The applications to undertake - Stacks []SyncStack -} - type ResourceError struct { ResourceID flux.ResourceID Source string diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go index 3deb4531d..26b2f4e5d 100644 --- a/daemon/daemon_test.go +++ b/daemon/daemon_test.go @@ -356,9 +356,9 @@ func TestDaemon_NotifyChange(t *testing.T) { ctx := context.Background() var syncCalled int - var syncDef *cluster.SyncDef + var syncDef *cluster.SyncSet var syncMu sync.Mutex - mockK8s.SyncFunc = func(def cluster.SyncDef) error { + mockK8s.SyncFunc = func(def cluster.SyncSet) error { syncMu.Lock() syncCalled++ syncDef = &def @@ -663,7 +663,7 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *cluster.Mock, *mockEven singleService, }, nil } - k8s.SyncFunc = func(def cluster.SyncDef) error { return nil } + k8s.SyncFunc = func(def cluster.SyncSet) error { return nil } } var imageRegistry registry.Registry diff --git a/daemon/loop.go b/daemon/loop.go index 229df7350..8d1923d3a 100644 --- a/daemon/loop.go +++ b/daemon/loop.go @@ -23,6 +23,7 @@ import ( const ( // Timeout for git operations we're prepared to abandon gitOpTimeout = 15 * time.Second + syncSetName = "git" ) type LoopVars struct { @@ -205,7 +206,7 @@ func (d *Daemon) doSync(logger log.Logger, lastKnownSyncTagRev *string, warnedAb } var resourceErrors []event.ResourceError - if err := fluxsync.Sync(allResources, d.Cluster); err != nil { + if err := fluxsync.Sync(syncSetName, allResources, d.Cluster); err != nil { logger.Log("err", err) switch syncerr := err.(type) { case cluster.SyncError: diff --git a/daemon/loop_test.go b/daemon/loop_test.go index 5088774fc..2abff845f 100644 --- a/daemon/loop_test.go +++ b/daemon/loop_test.go @@ -89,13 +89,13 @@ func TestPullAndSync_InitialSync(t *testing.T) { defer cleanup() syncCalled := 0 - var syncDef *cluster.SyncDef + var syncDef *cluster.SyncSet expectedResourceIDs := flux.ResourceIDs{} for id, _ := range testfiles.ResourceMap { expectedResourceIDs = append(expectedResourceIDs, id) } expectedResourceIDs.Sort() - k8s.SyncFunc = func(def cluster.SyncDef) error { + k8s.SyncFunc = func(def cluster.SyncSet) error { syncCalled++ syncDef = &def return nil @@ -160,13 +160,13 @@ func TestDoSync_NoNewCommits(t *testing.T) { } syncCalled := 0 - var syncDef *cluster.SyncDef + var syncDef *cluster.SyncSet expectedResourceIDs := flux.ResourceIDs{} for id, _ := range testfiles.ResourceMap { expectedResourceIDs = append(expectedResourceIDs, id) } expectedResourceIDs.Sort() - k8s.SyncFunc = func(def cluster.SyncDef) error { + k8s.SyncFunc = func(def cluster.SyncSet) error { syncCalled++ syncDef = &def return nil @@ -256,13 +256,13 @@ func TestDoSync_WithNewCommit(t *testing.T) { } syncCalled := 0 - var syncDef *cluster.SyncDef + var syncDef *cluster.SyncSet expectedResourceIDs := flux.ResourceIDs{} for id, _ := range testfiles.ResourceMap { expectedResourceIDs = append(expectedResourceIDs, id) } expectedResourceIDs.Sort() - k8s.SyncFunc = func(def cluster.SyncDef) error { + k8s.SyncFunc = func(def cluster.SyncSet) error { syncCalled++ syncDef = &def return nil diff --git a/sync/sync.go b/sync/sync.go index 0c1688238..b617c407f 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -7,30 +7,24 @@ import ( // Syncer has the methods we need to be able to compile and run a sync type Syncer interface { - Sync(cluster.SyncDef) error + Sync(cluster.SyncSet) error } // Sync synchronises the cluster to the files under a directory. -func Sync(repoResources map[string]resource.Resource, clus Syncer) error { - // TODO: multiple stack support. This will involve partitioning - // the resources into disjoint maps, then passing each to - // makeStack. - defaultStack := makeStack("default", repoResources) - - sync := cluster.SyncDef{Stacks: []cluster.SyncStack{defaultStack}} - if err := clus.Sync(sync); err != nil { +func Sync(setName string, repoResources map[string]resource.Resource, clus Syncer) error { + set := makeSet(setName, repoResources) + if err := clus.Sync(set); err != nil { return err } return nil } -func makeStack(name string, repoResources map[string]resource.Resource) cluster.SyncStack { - stack := cluster.SyncStack{Name: name} +func makeSet(name string, repoResources map[string]resource.Resource) cluster.SyncSet { + s := cluster.SyncSet{Name: name} var resources []resource.Resource for _, res := range repoResources { resources = append(resources, res) } - - stack.Resources = resources - return stack + s.Resources = resources + return s } diff --git a/sync/sync_test.go b/sync/sync_test.go index 3821ca303..011bbc3d5 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -28,7 +28,7 @@ func TestSync(t *testing.T) { t.Fatal(err) } - if err := Sync(resources, clus); err != nil { + if err := Sync("synctest", resources, clus); err != nil { t.Fatal(err) } checkClusterMatchesFiles(t, manifests, clus.resources, checkout.Dir(), dirs) @@ -53,13 +53,11 @@ func setup(t *testing.T) (*git.Checkout, func()) { type syncCluster struct{ resources map[string]string } -func (p *syncCluster) Sync(def cluster.SyncDef) error { +func (p *syncCluster) Sync(def cluster.SyncSet) error { println("=== Syncing ===") - for _, stack := range def.Stacks { - for _, resource := range stack.Resources { - println("Applying " + resource.ResourceID().String()) - p.resources[resource.ResourceID().String()] = string(resource.Bytes()) - } + for _, resource := range def.Resources { + println("Applying " + resource.ResourceID().String()) + p.resources[resource.ResourceID().String()] = string(resource.Bytes()) } println("=== Done syncing ===") return nil From 60346ef9cbaebb9b08b7990a5493ec6b3969cd09 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 21 Feb 2019 12:30:28 +0000 Subject: [PATCH 17/24] Use cached discovery client in sync too Construct the client needed by the cluster in main.go, and include the cached discovery client we construct for namespacing. This has a side benefit of cutting down the number of arguments needed by NewCluster. --- cluster/kubernetes/images.go | 2 +- cluster/kubernetes/images_test.go | 4 ++-- cluster/kubernetes/kubernetes.go | 32 +++++++++++++-------------- cluster/kubernetes/kubernetes_test.go | 3 ++- cluster/kubernetes/sync.go | 2 +- cluster/kubernetes/sync_test.go | 11 ++++----- cmd/fluxd/main.go | 22 +++++++++--------- 7 files changed, 40 insertions(+), 36 deletions(-) diff --git a/cluster/kubernetes/images.go b/cluster/kubernetes/images.go index acea81cce..8370ddb88 100644 --- a/cluster/kubernetes/images.go +++ b/cluster/kubernetes/images.go @@ -17,7 +17,7 @@ import ( func mergeCredentials(log func(...interface{}) error, includeImage func(imageName string) bool, - client extendedClient, + client ExtendedClient, namespace string, podTemplate apiv1.PodTemplateSpec, imageCreds registry.ImageCreds, seenCreds map[string]registry.Credentials) { diff --git a/cluster/kubernetes/images_test.go b/cluster/kubernetes/images_test.go index 84d8d8701..a94682c5a 100644 --- a/cluster/kubernetes/images_test.go +++ b/cluster/kubernetes/images_test.go @@ -64,7 +64,7 @@ func TestMergeCredentials(t *testing.T) { makeServiceAccount(ns, saName, []string{secretName2}), makeImagePullSecret(ns, secretName1, "docker.io"), makeImagePullSecret(ns, secretName2, "quay.io")) - client := extendedClient{coreClient: clientset} + client := ExtendedClient{coreClient: clientset} creds := registry.ImageCreds{} @@ -97,7 +97,7 @@ func TestMergeCredentials_ImageExclusion(t *testing.T) { } clientset := fake.NewSimpleClientset() - client := extendedClient{coreClient: clientset} + client := ExtendedClient{coreClient: clientset} var includeImage = func(imageName string) bool { for _, exp := range []string{"k8s.gcr.io/*", "*test*"} { diff --git a/cluster/kubernetes/kubernetes.go b/cluster/kubernetes/kubernetes.go index 0d0b21760..da67544c2 100644 --- a/cluster/kubernetes/kubernetes.go +++ b/cluster/kubernetes/kubernetes.go @@ -11,6 +11,7 @@ import ( apiv1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" k8sclientdynamic "k8s.io/client-go/dynamic" k8sclient "k8s.io/client-go/kubernetes" @@ -23,11 +24,22 @@ import ( type coreClient k8sclient.Interface type dynamicClient k8sclientdynamic.Interface type fluxHelmClient fhrclient.Interface +type discoveryClient discovery.DiscoveryInterface -type extendedClient struct { +type ExtendedClient struct { coreClient dynamicClient fluxHelmClient + discoveryClient +} + +func MakeClusterClientset(core coreClient, dyn dynamicClient, fluxhelm fluxHelmClient, disco discoveryClient) ExtendedClient { + return ExtendedClient{ + coreClient: core, + dynamicClient: dyn, + fluxHelmClient: fluxhelm, + discoveryClient: disco, + } } // --- add-ons @@ -71,7 +83,7 @@ type Cluster struct { // Do garbage collection when syncing resources GC bool - client extendedClient + client ExtendedClient applier Applier version string // string response for the version command. @@ -91,21 +103,9 @@ type Cluster struct { } // NewCluster returns a usable cluster. -func NewCluster(clientset k8sclient.Interface, - dynamicClientset k8sclientdynamic.Interface, - fluxHelmClientset fhrclient.Interface, - applier Applier, - sshKeyRing ssh.KeyRing, - logger log.Logger, - nsWhitelist []string, - imageExcludeList []string) *Cluster { - +func NewCluster(client ExtendedClient, applier Applier, sshKeyRing ssh.KeyRing, logger log.Logger, nsWhitelist []string, imageExcludeList []string) *Cluster { c := &Cluster{ - client: extendedClient{ - clientset, - dynamicClientset, - fluxHelmClientset, - }, + client: client, applier: applier, logger: logger, sshKeyRing: sshKeyRing, diff --git a/cluster/kubernetes/kubernetes_test.go b/cluster/kubernetes/kubernetes_test.go index 9143da643..0d80fd56a 100644 --- a/cluster/kubernetes/kubernetes_test.go +++ b/cluster/kubernetes/kubernetes_test.go @@ -25,7 +25,8 @@ func newNamespace(name string) *apiv1.Namespace { func testGetAllowedNamespaces(t *testing.T, namespace []string, expected []string) { clientset := fakekubernetes.NewSimpleClientset(newNamespace("default"), newNamespace("kube-system")) - c := NewCluster(clientset, nil, nil, nil, nil, log.NewNopLogger(), namespace, []string{}) + client := ExtendedClient{coreClient: clientset} + c := NewCluster(client, nil, nil, log.NewNopLogger(), namespace, []string{}) namespaces, err := c.getAllowedNamespaces() if err != nil { diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index 9d5ae0b78..8811aa8d1 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -185,7 +185,7 @@ func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesou listOptions.LabelSelector = selector } - resources, err := c.client.coreClient.Discovery().ServerResources() + resources, err := c.client.discoveryClient.ServerResources() if err != nil { return nil, err } diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index 53282becf..5520ac3b2 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -35,7 +35,7 @@ const ( defaultTestNamespace = "unusual-default" ) -func fakeClients() extendedClient { +func fakeClients() ExtendedClient { scheme := runtime.NewScheme() // Set this to `true` to output a trace of the API actions called @@ -81,10 +81,11 @@ func fakeClients() extendedClient { } } - return extendedClient{ - coreClient: coreClient, - fluxHelmClient: fluxClient, - dynamicClient: dynamicClient, + return ExtendedClient{ + coreClient: coreClient, + fluxHelmClient: fluxClient, + dynamicClient: dynamicClient, + discoveryClient: coreClient.Discovery(), } } diff --git a/cmd/fluxd/main.go b/cmd/fluxd/main.go index 58b644c75..f9af51dc8 100644 --- a/cmd/fluxd/main.go +++ b/cmd/fluxd/main.go @@ -17,7 +17,7 @@ import ( "github.com/go-kit/kit/log" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/pflag" - k8sifclient "github.com/weaveworks/flux/integrations/client/clientset/versioned" + integrations "github.com/weaveworks/flux/integrations/client/clientset/versioned" crd "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" k8sclientdynamic "k8s.io/client-go/dynamic" k8sclient "k8s.io/client-go/kubernetes" @@ -242,12 +242,19 @@ func main() { os.Exit(1) } - ifclientset, err := k8sifclient.NewForConfig(restClientConfig) + integrationsClientset, err := integrations.NewForConfig(restClientConfig) if err != nil { logger.Log("error", fmt.Sprintf("Error building integrations clientset: %v", err)) os.Exit(1) } + crdClient, err := crd.NewForConfig(restClientConfig) + if err != nil { + logger.Log("error", fmt.Sprintf("Error building API extensions (CRD) clientset: %v", err)) + os.Exit(1) + } + discoClientset := kubernetes.MakeCachedDiscovery(clientset.Discovery(), crdClient, shutdown) + serverVersion, err := clientset.ServerVersion() if err != nil { logger.Log("err", err) @@ -295,7 +302,8 @@ func main() { logger.Log("kubectl", kubectl) kubectlApplier := kubernetes.NewKubectl(kubectl, restClientConfig) - k8sInst := kubernetes.NewCluster(clientset, dynamicClientset, ifclientset, kubectlApplier, sshKeyRing, logger, *k8sNamespaceWhitelist, *registryExcludeImage) + client := kubernetes.MakeClusterClientset(clientset, dynamicClientset, integrationsClientset, discoClientset) + k8sInst := kubernetes.NewCluster(client, kubectlApplier, sshKeyRing, logger, *k8sNamespaceWhitelist, *registryExcludeImage) k8sInst.GC = *syncGC if err := k8sInst.Ping(); err != nil { @@ -309,14 +317,8 @@ func main() { // There is only one way we currently interpret a repo of // files as manifests, and that's as Kubernetes yamels. k8sManifests = &kubernetes.Manifests{} + k8sManifests.Namespacer, err = kubernetes.NewNamespacer(kubectlApplier, discoClientset) - crdClient, err := crd.NewForConfig(restClientConfig) - if err == nil { - disco := kubernetes.MakeCachedDiscovery(clientset.Discovery(), crdClient, shutdown) - k8sManifests.Namespacer, err = kubernetes.NewNamespacer(kubectlApplier, disco) - } - if err == nil { - } if err != nil { logger.Log("err", err) os.Exit(1) From 692a96ba6cd0707ed6b1a74aa43f38c20631145c Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 21 Feb 2019 15:39:41 +0000 Subject: [PATCH 18/24] Separate cached discovery code (and testing) A small reorganisation to separate - cachedDiscovery from namespacer, since it's used independently - things needed for testing the above, from non-testing code --- cluster/kubernetes/cached_disco.go | 102 ++++++++++++++++++ cluster/kubernetes/cached_disco_test.go | 131 ++++++++++++++++++++++++ cluster/kubernetes/namespacer.go | 110 -------------------- cluster/kubernetes/namespacer_test.go | 99 ------------------ 4 files changed, 233 insertions(+), 209 deletions(-) create mode 100644 cluster/kubernetes/cached_disco.go create mode 100644 cluster/kubernetes/cached_disco_test.go diff --git a/cluster/kubernetes/cached_disco.go b/cluster/kubernetes/cached_disco.go new file mode 100644 index 000000000..5f0a7e69a --- /dev/null +++ b/cluster/kubernetes/cached_disco.go @@ -0,0 +1,102 @@ +package kubernetes + +import ( + "sync" + "time" + + crdv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + crd "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + discocache "k8s.io/client-go/discovery/cached" + toolscache "k8s.io/client-go/tools/cache" +) + +// This exists so that we can do our own invalidation. +type cachedDiscovery struct { + discovery.CachedDiscoveryInterface + + invalidMu sync.Mutex + invalid bool +} + +// The k8s.io/client-go v8.0.0 implementation of MemCacheDiscovery +// refreshes the cached values, synchronously, when Invalidate() is +// called. Since we want to invalidate every time a CRD changes, but +// only refresh values when we need to read the cached values, this +// method defers the invalidation until a read is done. +func (d *cachedDiscovery) Invalidate() { + d.invalidMu.Lock() + d.invalid = true + d.invalidMu.Unlock() +} + +// ServerResourcesForGroupVersion is the method used by the +// namespacer; so, this is the one where we check whether the cache +// has been invalidated. A cachedDiscovery implementation for more +// general use would do this for all methods (that weren't implemented +// purely in terms of other methods). +func (d *cachedDiscovery) ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) { + d.invalidMu.Lock() + invalid := d.invalid + d.invalid = false + d.invalidMu.Unlock() + if invalid { + d.CachedDiscoveryInterface.Invalidate() + } + return d.CachedDiscoveryInterface.ServerResourcesForGroupVersion(groupVersion) +} + +// MakeCachedDiscovery constructs a CachedDicoveryInterface that will +// be invalidated whenever the set of CRDs change. The idea is that +// the only avenue of a change to the API resources in a running +// system is CRDs being added, updated or deleted. +func MakeCachedDiscovery(d discovery.DiscoveryInterface, c crd.Interface, shutdown <-chan struct{}) discovery.CachedDiscoveryInterface { + result, _, _ := makeCachedDiscovery(d, c, shutdown, makeInvalidatingHandler) + return result +} + +// --- + +func makeInvalidatingHandler(cached discovery.CachedDiscoveryInterface) toolscache.ResourceEventHandler { + var handler toolscache.ResourceEventHandler = toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(_ interface{}) { + cached.Invalidate() + }, + UpdateFunc: func(_, _ interface{}) { + cached.Invalidate() + }, + DeleteFunc: func(_ interface{}) { + cached.Invalidate() + }, + } + return handler +} + +type makeHandle func(discovery.CachedDiscoveryInterface) toolscache.ResourceEventHandler + +// makeCachedDiscovery constructs a cached discovery client, with more +// flexibility than MakeCachedDiscovery; e.g., with extra handlers for +// testing. +func makeCachedDiscovery(d discovery.DiscoveryInterface, c crd.Interface, shutdown <-chan struct{}, handlerFn makeHandle) (*cachedDiscovery, toolscache.Store, toolscache.Controller) { + cachedDisco := &cachedDiscovery{CachedDiscoveryInterface: discocache.NewMemCacheClient(d)} + // We have an empty cache, so it's _a priori_ invalid. (Yes, that's the zero value, but better safe than sorry) + cachedDisco.Invalidate() + + crdClient := c.ApiextensionsV1beta1().CustomResourceDefinitions() + lw := &toolscache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return crdClient.List(options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return crdClient.Watch(options) + }, + } + + handler := handlerFn(cachedDisco) + store, controller := toolscache.NewInformer(lw, &crdv1beta1.CustomResourceDefinition{}, 5*time.Minute, handler) + go controller.Run(shutdown) + return cachedDisco, store, controller +} diff --git a/cluster/kubernetes/cached_disco_test.go b/cluster/kubernetes/cached_disco_test.go new file mode 100644 index 000000000..0efed8bd7 --- /dev/null +++ b/cluster/kubernetes/cached_disco_test.go @@ -0,0 +1,131 @@ +package kubernetes + +import ( + "testing" + "time" + + crdv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + crdfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" + toolscache "k8s.io/client-go/tools/cache" +) + +type chainHandler struct { + first toolscache.ResourceEventHandler + next toolscache.ResourceEventHandler +} + +func (h chainHandler) OnAdd(obj interface{}) { + h.first.OnAdd(obj) + h.next.OnAdd(obj) +} + +func (h chainHandler) OnUpdate(old, new interface{}) { + h.first.OnUpdate(old, new) + h.next.OnUpdate(old, new) +} + +func (h chainHandler) OnDelete(old interface{}) { + h.first.OnDelete(old) + h.next.OnDelete(old) +} + +func TestCachedDiscovery(t *testing.T) { + coreClient := makeFakeClient() + + myCRD := &crdv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom", + }, + } + crdClient := crdfake.NewSimpleClientset(myCRD) + + // Here's my fake API resource + myAPI := &metav1.APIResourceList{ + GroupVersion: "foo/v1", + APIResources: []metav1.APIResource{ + {Name: "customs", SingularName: "custom", Namespaced: true, Kind: "Custom", Verbs: getAndList}, + }, + } + + apiResources := coreClient.Fake.Resources + coreClient.Fake.Resources = append(apiResources, myAPI) + + shutdown := make(chan struct{}) + defer close(shutdown) + + // this extra handler means we can synchronise on the add later + // being processed + allowAdd := make(chan interface{}) + + addHandler := toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + allowAdd <- obj + }, + } + makeHandler := func(d discovery.CachedDiscoveryInterface) toolscache.ResourceEventHandler { + return chainHandler{first: addHandler, next: makeInvalidatingHandler(d)} + } + + cachedDisco, store, _ := makeCachedDiscovery(coreClient.Discovery(), crdClient, shutdown, makeHandler) + + namespacer, err := NewNamespacer(namespaceDefaulterFake("bar-ns"), cachedDisco) + if err != nil { + t.Fatal(err) + } + + namespaced, err := namespacer.lookupNamespaced("foo/v1", "Custom") + if err != nil { + t.Fatal(err) + } + if !namespaced { + t.Error("got false from lookupNamespaced, expecting true") + } + + // In a cluster, we'd rely on the apiextensions server to reflect + // changes to CRDs to changes in the API resources. Here I will be + // more narrow, and just test that the API resources are reloaded + // when a CRD is updated or deleted. + + // This is delicate: we can't just change the value in-place, + // since that will update everyone's record of it, and the test + // below will trivially succeed. + updatedAPI := &metav1.APIResourceList{ + GroupVersion: "foo/v1", + APIResources: []metav1.APIResource{ + {Name: "customs", SingularName: "custom", Namespaced: false /* <-- changed */, Kind: "Custom", Verbs: getAndList}, + }, + } + coreClient.Fake.Resources = append(apiResources, updatedAPI) + + // Provoke the cached discovery client into invalidating + _, err = crdClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(myCRD) + if err != nil { + t.Fatal(err) + } + + // Wait for the update to "go through" + select { + case <-allowAdd: + break + case <-time.After(time.Second): + t.Fatal("timed out waiting for Add to happen") + } + + _, exists, err := store.Get(myCRD) + if err != nil { + t.Error(err) + } + if !exists { + t.Error("does not exist") + } + + namespaced, err = namespacer.lookupNamespaced("foo/v1", "Custom") + if err != nil { + t.Fatal(err) + } + if namespaced { + t.Error("got true from lookupNamespaced, expecting false (after changing it)") + } +} diff --git a/cluster/kubernetes/namespacer.go b/cluster/kubernetes/namespacer.go index 51074a4ab..4f2d4c41e 100644 --- a/cluster/kubernetes/namespacer.go +++ b/cluster/kubernetes/namespacer.go @@ -2,17 +2,8 @@ package kubernetes import ( "fmt" - "sync" - "time" - crdv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - crd "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" - discocache "k8s.io/client-go/discovery/cached" - toolscache "k8s.io/client-go/tools/cache" kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" ) @@ -63,104 +54,3 @@ func (n *namespaceViaDiscovery) lookupNamespaced(groupVersion, kind string) (boo } return false, fmt.Errorf("resource not found for API %s, kind %s", groupVersion, kind) } - -// This mainly exists so I can put the controller and store in here, -// so as to have access to them for testing; and, so that we can do -// our own invalidation. -type cachedDiscovery struct { - discovery.CachedDiscoveryInterface - - store toolscache.Store - controller toolscache.Controller - - invalidMu sync.Mutex - invalid bool -} - -// The older (v8.0.0) implementation of MemCacheDiscovery refreshes -// the cached values, synchronously, when Invalidate is called. Since -// we will invalidate every time something cahnges, but it only -// matters when we want to read the cached values, this method (and -// ServerResourcesForGroupVersion) saves the invalidation for when a -// read is done. -func (d *cachedDiscovery) Invalidate() { - d.invalidMu.Lock() - d.invalid = true - d.invalidMu.Unlock() -} - -// This happens to be the method that we call in the namespacer; so, -// this is the one where we check whether the cache has been -// invalidated. A cachedDiscovery implementation for more general use -// would do this for all methods (that weren't implemented purely in -// terms of other methods). -func (d *cachedDiscovery) ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) { - d.invalidMu.Lock() - invalid := d.invalid - d.invalid = false - d.invalidMu.Unlock() - if invalid { - d.CachedDiscoveryInterface.Invalidate() - } - return d.CachedDiscoveryInterface.ServerResourcesForGroupVersion(groupVersion) -} - -type chainHandler struct { - first toolscache.ResourceEventHandler - next toolscache.ResourceEventHandler -} - -func (h chainHandler) OnAdd(obj interface{}) { - h.first.OnAdd(obj) - h.next.OnAdd(obj) -} - -func (h chainHandler) OnUpdate(old, new interface{}) { - h.first.OnUpdate(old, new) - h.next.OnUpdate(old, new) -} - -func (h chainHandler) OnDelete(old interface{}) { - h.first.OnDelete(old) - h.next.OnDelete(old) -} - -// MakeCachedDiscovery constructs a CachedDicoveryInterface that will -// be invalidated whenever the set of CRDs change. The idea is that -// the only avenue of a change to the API resources in a running -// system is CRDs being added, updated or deleted. The prehandlers are -// there to allow us to put extra synchronisation in, for testing. -func MakeCachedDiscovery(d discovery.DiscoveryInterface, c crd.Interface, shutdown <-chan struct{}, prehandlers ...toolscache.ResourceEventHandler) *cachedDiscovery { - cachedDisco := &cachedDiscovery{CachedDiscoveryInterface: discocache.NewMemCacheClient(d)} - // We have an empty cache, so it's _a priori_ invalid. (Yes, that's the zero value, but better safe than sorry) - cachedDisco.Invalidate() - - crdClient := c.ApiextensionsV1beta1().CustomResourceDefinitions() - - var handler toolscache.ResourceEventHandler = toolscache.ResourceEventHandlerFuncs{ - AddFunc: func(_ interface{}) { - cachedDisco.Invalidate() - }, - UpdateFunc: func(_, _ interface{}) { - cachedDisco.Invalidate() - }, - DeleteFunc: func(_ interface{}) { - cachedDisco.Invalidate() - }, - } - for _, h := range prehandlers { - handler = chainHandler{first: h, next: handler} - } - - lw := &toolscache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return crdClient.List(options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return crdClient.Watch(options) - }, - } - cachedDisco.store, cachedDisco.controller = toolscache.NewInformer(lw, &crdv1beta1.CustomResourceDefinition{}, 5*time.Minute, handler) - go cachedDisco.controller.Run(shutdown) - return cachedDisco -} diff --git a/cluster/kubernetes/namespacer_test.go b/cluster/kubernetes/namespacer_test.go index 4e3d8ce51..b0ec71b68 100644 --- a/cluster/kubernetes/namespacer_test.go +++ b/cluster/kubernetes/namespacer_test.go @@ -2,13 +2,9 @@ package kubernetes import ( "testing" - "time" - crdv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - crdfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corefake "k8s.io/client-go/kubernetes/fake" - toolscache "k8s.io/client-go/tools/cache" kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" ) @@ -93,98 +89,3 @@ metadata: assertEffectiveNamespace(":deployment/noNamespace", "fallback-ns") assertEffectiveNamespace("spurious:namespace/notNamespaced", "") } - -func TestCachedDiscovery(t *testing.T) { - coreClient := makeFakeClient() - - myCRD := &crdv1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: "custom", - }, - } - crdClient := crdfake.NewSimpleClientset(myCRD) - - // Here's my fake API resource - myAPI := &metav1.APIResourceList{ - GroupVersion: "foo/v1", - APIResources: []metav1.APIResource{ - {Name: "customs", SingularName: "custom", Namespaced: true, Kind: "Custom", Verbs: getAndList}, - }, - } - - apiResources := coreClient.Fake.Resources - coreClient.Fake.Resources = append(apiResources, myAPI) - - shutdown := make(chan struct{}) - defer close(shutdown) - - // this extra handler means we can synchronise on the add later - // being processed - allowAdd := make(chan interface{}) - handler := toolscache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - allowAdd <- obj - }, - } - - cachedDisco := MakeCachedDiscovery(coreClient.Discovery(), crdClient, shutdown, handler) - - namespacer, err := NewNamespacer(namespaceDefaulterFake("bar-ns"), cachedDisco) - if err != nil { - t.Fatal(err) - } - - namespaced, err := namespacer.lookupNamespaced("foo/v1", "Custom") - if err != nil { - t.Fatal(err) - } - if !namespaced { - t.Error("got false from lookupNamespaced, expecting true") - } - - // In a cluster, we'd rely on the apiextensions server to reflect - // changes to CRDs to changes in the API resources. Here I will be - // more narrow, and just test that the API resources are reloaded - // when a CRD is updated or deleted. - - // This is delicate: we can't just change the value in-place, - // since that will update everyone's record of it, and the test - // below will trivially succeed. - updatedAPI := &metav1.APIResourceList{ - GroupVersion: "foo/v1", - APIResources: []metav1.APIResource{ - {Name: "customs", SingularName: "custom", Namespaced: false /* <-- changed */, Kind: "Custom", Verbs: getAndList}, - }, - } - coreClient.Fake.Resources = append(apiResources, updatedAPI) - - // Provoke the cached discovery client into invalidating - _, err = crdClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(myCRD) - if err != nil { - t.Fatal(err) - } - - // Wait for the update to "go through" - select { - case <-allowAdd: - break - case <-time.After(time.Second): - t.Fatal("timed out waiting for Add to happen") - } - - _, exists, err := cachedDisco.store.Get(myCRD) - if err != nil { - t.Error(err) - } - if !exists { - t.Error("does not exist") - } - - namespaced, err = namespacer.lookupNamespaced("foo/v1", "Custom") - if err != nil { - t.Fatal(err) - } - if namespaced { - t.Error("got true from lookupNamespaced, expecting false (after changing it)") - } -} From 145fabed405575ed4265b4708e2726c9e61f2d05 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 21 Feb 2019 15:45:42 +0000 Subject: [PATCH 19/24] Tidy kuberesource methods - remove GetSyncSet, which is not used (since we use a selector with the set name already, we don't need to get it for comparison) - rename Bytes() so that it's clear what its purpose is, and that it's not an implementation of resource.Resource --- cluster/kubernetes/sync.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index 8811aa8d1..df03b0dfe 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -109,7 +109,7 @@ func (c *Cluster) Sync(spec cluster.SyncSet) error { switch { case !ok: // was not recorded as having been staged for application c.logger.Log("info", "cluster resource not in resources to be synced; deleting", "resource", resourceID) - orphanedResources.stage("delete", res.ResourceID(), "", res.Bytes()) + orphanedResources.stage("delete", res.ResourceID(), "", res.IdentifyingBytes()) case actual != expected: c.logger.Log("warning", "resource to be synced has not been updated; skipping", "resource", resourceID) continue @@ -152,8 +152,9 @@ func (r *kuberesource) ResourceID() flux.ResourceID { return flux.MakeResourceID(ns, kind, name) } -// Bytes returns a byte slice description -func (r *kuberesource) Bytes() []byte { +// Bytes returns a byte slice description, including enough info to +// identify the resource (but not momre) +func (r *kuberesource) IdentifyingBytes() []byte { return []byte(fmt.Sprintf(` apiVersion: %s kind: %s @@ -173,12 +174,6 @@ func (r *kuberesource) GetChecksum() string { return r.obj.GetAnnotations()[checksumAnnotation] } -// GetSyncSet returns the sync set name recorded on the the resource -// from Kubernetes, or an empty string if it's not present. -func (r *kuberesource) GetSyncSet() string { - return r.obj.GetLabels()[syncSetLabel] -} - func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesource, error) { listOptions := meta_v1.ListOptions{} if selector != "" { From 93bccbf7b6b44e2ff04fffe920a20801a6012f1e Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Mon, 25 Feb 2019 17:14:06 +0000 Subject: [PATCH 20/24] Use git repo config to identify SyncSet Using a constant for identifying sync sets means that if more than one fluxd is running in a cluster, they will garbage collect each other's resources. We want to be able to delimit which resources are _this_ fluxd's responsibility. Using the git config, including the branch and paths, means fluxd will only garbage collect files that are in the bit of the repo it is applying. That assumes no other fluxd is looking at (exactly) the same directories in the same git URL -- which is a fair assumption, since that would obviously be a bad config and cause problems, even without considering garbage collection. --- Gopkg.lock | 9 +++++++++ daemon/loop.go | 19 ++++++++++++++++++- git/repo.go | 5 ----- git/url.go | 24 ++++++++++++++++++++++++ git/url_test.go | 20 ++++++++++++++++++++ 5 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 git/url.go create mode 100644 git/url_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 7deddeb6b..c1c4f6a82 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -622,6 +622,14 @@ revision = "0599d764e054d4e983bb120e30759179fafe3942" version = "v1.2.0" +[[projects]] + branch = "master" + digest = "1:2e43242796ee48ff0256eaf784ffaca015614ea5cb284cbc7b6e0fb65c219887" + name = "github.com/whilp/git-urls" + packages = ["."] + pruneopts = "" + revision = "31bac0d230fa29f36ed1b3279c2343752e7196c0" + [[projects]] branch = "master" digest = "1:2ea6df0f542cc95a5e374e9cdd81eaa599ed0d55366eef92d2f6b9efa2795c07" @@ -1210,6 +1218,7 @@ "github.com/stretchr/testify/assert", "github.com/weaveworks/common/middleware", "github.com/weaveworks/go-checkpoint", + "github.com/whilp/git-urls", "golang.org/x/sys/unix", "golang.org/x/time/rate", "gopkg.in/yaml.v2", diff --git a/daemon/loop.go b/daemon/loop.go index 8d1923d3a..2ab75e2b9 100644 --- a/daemon/loop.go +++ b/daemon/loop.go @@ -2,6 +2,8 @@ package daemon import ( "context" + "crypto/sha256" + "encoding/base64" "fmt" "strings" "sync" @@ -23,7 +25,6 @@ import ( const ( // Timeout for git operations we're prepared to abandon gitOpTimeout = 15 * time.Second - syncSetName = "git" ) type LoopVars struct { @@ -162,6 +163,9 @@ func (d *Daemon) doSync(logger log.Logger, lastKnownSyncTagRev *string, warnedAb fluxmetrics.LabelSuccess, fmt.Sprint(retErr == nil), ).Observe(time.Since(started).Seconds()) }() + + syncSetName := makeSyncLabel(d.Repo.Origin(), d.GitConfig) + // We don't care how long this takes overall, only about not // getting bogged down in certain operations, so use an // undeadlined context in general. @@ -450,3 +454,16 @@ func isUnknownRevision(err error) bool { (strings.Contains(err.Error(), "unknown revision or path not in the working tree.") || strings.Contains(err.Error(), "bad revision")) } + +func makeSyncLabel(remote git.Remote, conf git.Config) string { + urlbit := remote.SafeURL() + pathshash := sha256.New() + pathshash.Write([]byte(urlbit)) + pathshash.Write([]byte(conf.Branch)) + for _, path := range conf.Paths { + pathshash.Write([]byte(path)) + } + // the prefix is in part to make sure it's a valid (Kubernetes) + // label value -- a modest abstraction leak + return "git-" + base64.RawURLEncoding.EncodeToString(pathshash.Sum(nil)) +} diff --git a/git/repo.go b/git/repo.go index f81fe47ba..5f104521d 100644 --- a/git/repo.go +++ b/git/repo.go @@ -44,11 +44,6 @@ const ( RepoReady GitRepoStatus = "ready" // has been written to, so ready to sync ) -// Remote points at a git repo somewhere. -type Remote struct { - URL string // clone from here -} - type Repo struct { // As supplied to constructor origin Remote diff --git a/git/url.go b/git/url.go new file mode 100644 index 000000000..d59cafd4f --- /dev/null +++ b/git/url.go @@ -0,0 +1,24 @@ +package git + +import ( + "fmt" + "net/url" + + "github.com/whilp/git-urls" +) + +// Remote points at a git repo somewhere. +type Remote struct { + URL string // clone from here +} + +func (r Remote) SafeURL() string { + u, err := giturls.Parse(r.URL) + if err != nil { + return fmt.Sprintf("", r.URL) + } + if u.User != nil { + u.User = url.User(u.User.Username()) + } + return u.String() +} diff --git a/git/url_test.go b/git/url_test.go new file mode 100644 index 000000000..b119010f3 --- /dev/null +++ b/git/url_test.go @@ -0,0 +1,20 @@ +package git + +import ( + "strings" + "testing" +) + +func TestSafeURL(t *testing.T) { + const password = "abc123" + for _, url := range []string{ + "git@github.com:weaveworks/flux", + "https://user@example.com:5050/repo.git", + "https://user:" + password + "@example.com:5050/repo.git", + } { + u := Remote{url} + if strings.Contains(u.SafeURL(), password) { + t.Errorf("Safe URL for %s contains password %q", url, password) + } + } +} From 1d86d57caaa15d841b2d6b1ddf08611529203958 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Tue, 26 Feb 2019 00:21:01 +0000 Subject: [PATCH 21/24] Delegate GroupVersion parsing to schema.ParseGroupVersion --- cluster/kubernetes/sync.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index df03b0dfe..576529888 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -203,25 +203,12 @@ func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesou continue } - namespaced := apiResource.Namespaced - - // get group and version - var group, version string - groupVersion := resource.GroupVersion - if strings.Contains(groupVersion, "/") { - a := strings.SplitN(groupVersion, "/", 2) - group = a[0] - version = a[1] - } else { - group = "" - version = groupVersion + groupVersion, err := schema.ParseGroupVersion(resource.GroupVersion) + if err != nil { + return nil, err } - resourceClient := c.client.dynamicClient.Resource(schema.GroupVersionResource{ - Group: group, - Version: version, - Resource: apiResource.Name, - }) + resourceClient := c.client.dynamicClient.Resource(groupVersion.WithResource(apiResource.Name)) data, err := resourceClient.List(listOptions) if err != nil { return nil, err @@ -238,7 +225,7 @@ func (c *Cluster) getResourcesBySelector(selector string) (map[string]*kuberesou } // TODO(michael) also exclude anything that has an ownerReference (that isn't "standard"?) - res := &kuberesource{obj: &data.Items[i], namespaced: namespaced} + res := &kuberesource{obj: &data.Items[i], namespaced: apiResource.Namespaced} result[res.ResourceID().String()] = res } } From aeef2c6a3bea1804180c4917fba3e1f941987ad4 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Tue, 26 Feb 2019 16:26:31 +0000 Subject: [PATCH 22/24] Obtain the default namespace directly from kubeconfig --- cluster/kubernetes/cached_disco_test.go | 5 ++- cluster/kubernetes/namespacer.go | 38 ++++++++++++++++++---- cluster/kubernetes/namespacer_test.go | 42 ++++++++++++++++++++----- cluster/kubernetes/sync.go | 29 ----------------- cluster/kubernetes/sync_test.go | 5 ++- cmd/fluxd/main.go | 2 +- 6 files changed, 75 insertions(+), 46 deletions(-) diff --git a/cluster/kubernetes/cached_disco_test.go b/cluster/kubernetes/cached_disco_test.go index 0efed8bd7..82c5dfc6c 100644 --- a/cluster/kubernetes/cached_disco_test.go +++ b/cluster/kubernetes/cached_disco_test.go @@ -70,7 +70,10 @@ func TestCachedDiscovery(t *testing.T) { cachedDisco, store, _ := makeCachedDiscovery(coreClient.Discovery(), crdClient, shutdown, makeHandler) - namespacer, err := NewNamespacer(namespaceDefaulterFake("bar-ns"), cachedDisco) + saved := getDefaultNamespace + getDefaultNamespace = func() (string, error) { return "bar-ns", nil } + defer func() { getDefaultNamespace = saved }() + namespacer, err := NewNamespacer(cachedDisco) if err != nil { t.Fatal(err) } diff --git a/cluster/kubernetes/namespacer.go b/cluster/kubernetes/namespacer.go index 4f2d4c41e..8386e7425 100644 --- a/cluster/kubernetes/namespacer.go +++ b/cluster/kubernetes/namespacer.go @@ -4,28 +4,54 @@ import ( "fmt" "k8s.io/client-go/discovery" + "k8s.io/client-go/tools/clientcmd" kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" ) +// The namespace to presume if something doesn't have one, and we +// haven't been told what to use as a fallback. This is what +// `kubectl` uses when there's no config setting the fallback +// namespace. +const defaultFallbackNamespace = "default" + type namespaceViaDiscovery struct { fallbackNamespace string disco discovery.DiscoveryInterface } -type namespaceDefaulter interface { - GetDefaultNamespace() (string, error) -} - // NewNamespacer creates an implementation of Namespacer -func NewNamespacer(ns namespaceDefaulter, d discovery.DiscoveryInterface) (*namespaceViaDiscovery, error) { - fallback, err := ns.GetDefaultNamespace() +func NewNamespacer(d discovery.DiscoveryInterface) (*namespaceViaDiscovery, error) { + fallback, err := getDefaultNamespace() if err != nil { return nil, err } return &namespaceViaDiscovery{fallbackNamespace: fallback, disco: d}, nil } +// getDefaultNamespace returns the fallback namespace used by the +// when a namespaced resource doesn't have one specified. This is +// used when syncing to anticipate the identity of a resource in the +// cluster given the manifest from a file (which may be missing the +// namespace). +// A variable is used for mocking in tests. +var getDefaultNamespace = func() (string, error) { + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), + &clientcmd.ConfigOverrides{}, + ).RawConfig() + if err != nil { + return "", err + } + + cc := config.CurrentContext + if c, ok := config.Contexts[cc]; ok && c.Namespace != "" { + return c.Namespace, nil + } + + return defaultFallbackNamespace, nil +} + // effectiveNamespace yields the namespace that would be used for this // resource were it applied, taking into account the kind of the // resource, and local configuration. diff --git a/cluster/kubernetes/namespacer_test.go b/cluster/kubernetes/namespacer_test.go index b0ec71b68..6a1a97476 100644 --- a/cluster/kubernetes/namespacer_test.go +++ b/cluster/kubernetes/namespacer_test.go @@ -1,6 +1,8 @@ package kubernetes import ( + "io/ioutil" + "os" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -9,12 +11,6 @@ import ( kresource "github.com/weaveworks/flux/cluster/kubernetes/resource" ) -type namespaceDefaulterFake string - -func (ns namespaceDefaulterFake) GetDefaultNamespace() (string, error) { - return string(ns), nil -} - var getAndList = metav1.Verbs([]string{"get", "list"}) func makeFakeClient() *corefake.Clientset { @@ -39,8 +35,38 @@ func makeFakeClient() *corefake.Clientset { } func TestNamespaceDefaulting(t *testing.T) { + testKubeconfig := `apiVersion: v1 +clusters: [] +contexts: +- context: + cluster: cluster + namespace: namespace + user: user + name: context +current-context: context +kind: Config +preferences: {} +users: [] +` + err := ioutil.WriteFile("testkubeconfig", []byte(testKubeconfig), 0600) + if err != nil { + t.Fatal("cannot create test kubeconfig file") + } + defer os.Remove("testkubeconfig") + + os.Setenv("KUBECONFIG", "testkubeconfig") + defer os.Unsetenv("KUBECONFIG") coreClient := makeFakeClient() - nser, err := NewNamespacer(namespaceDefaulterFake("fallback-ns"), coreClient.Discovery()) + + ns, err := getDefaultNamespace() + if err != nil { + t.Fatal("cannot get default namespace") + } + if ns != "namespace" { + t.Fatal("unexpected default namespace", ns) + } + + nser, err := NewNamespacer(coreClient.Discovery()) if err != nil { t.Fatal(err) } @@ -86,6 +112,6 @@ metadata: } assertEffectiveNamespace("foo-ns:deployment/hasNamespace", "foo-ns") - assertEffectiveNamespace(":deployment/noNamespace", "fallback-ns") + assertEffectiveNamespace(":deployment/noNamespace", "namespace") assertEffectiveNamespace("spurious:namespace/notNamespaced", "") } diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index 576529888..fb8930050 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -30,12 +30,6 @@ import ( const ( syncSetLabel = kresource.PolicyPrefix + "sync-set" checksumAnnotation = kresource.PolicyPrefix + "sync-checksum" - - // The namespace to presume if something doesn't have one, and we - // haven't been told what to use as a fallback. This is what - // `kubectl` uses when there's no config setting the fallback - // namespace. - DefaultFallbackNamespace = "default" ) // Sync takes a definition of what should be running in the cluster, @@ -334,29 +328,6 @@ func (c *Kubectl) connectArgs() []string { return args } -// GetDefaultNamespace returns the fallback namespace used by the -// applied when a namespaced resource doesn't have one specified. This -// is used when syncing to anticipate the identity of a resource in -// the cluster given the manifest from a file (which may be missing -// the namespace). -func (k *Kubectl) GetDefaultNamespace() (string, error) { - cmd := k.kubectlCommand("config", "get-contexts", "--no-headers") - out, err := cmd.Output() - if err != nil { - return "", err - } - lines := bytes.Split(out, []byte("\n")) - for _, line := range lines { - words := bytes.Fields(line) - if len(words) > 1 && string(words[0]) == "*" { - if len(words) == 5 { - return string(words[4]), nil - } - } - } - return DefaultFallbackNamespace, nil -} - // rankOfKind returns an int denoting the position of the given kind // in the partial ordering of Kubernetes resources, according to which // kinds depend on which (derived by hand). diff --git a/cluster/kubernetes/sync_test.go b/cluster/kubernetes/sync_test.go index 5520ac3b2..d1de2d7a2 100644 --- a/cluster/kubernetes/sync_test.go +++ b/cluster/kubernetes/sync_test.go @@ -262,7 +262,10 @@ metadata: } test := func(t *testing.T, kube *Cluster, defs, expectedAfterSync string, expectErrors bool) { - namespacer, err := NewNamespacer(namespaceDefaulterFake(defaultTestNamespace), kube.client.coreClient.Discovery()) + saved := getDefaultNamespace + getDefaultNamespace = func() (string, error) { return defaultTestNamespace, nil } + defer func() { getDefaultNamespace = saved }() + namespacer, err := NewNamespacer(kube.client.coreClient.Discovery()) if err != nil { t.Fatal(err) } diff --git a/cmd/fluxd/main.go b/cmd/fluxd/main.go index f9af51dc8..eef6eac63 100644 --- a/cmd/fluxd/main.go +++ b/cmd/fluxd/main.go @@ -317,7 +317,7 @@ func main() { // There is only one way we currently interpret a repo of // files as manifests, and that's as Kubernetes yamels. k8sManifests = &kubernetes.Manifests{} - k8sManifests.Namespacer, err = kubernetes.NewNamespacer(kubectlApplier, discoClientset) + k8sManifests.Namespacer, err = kubernetes.NewNamespacer(discoClientset) if err != nil { logger.Log("err", err) From c6e2c4e877d6b00016eee94e26dc552c97050a0c Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Tue, 26 Feb 2019 17:04:38 +0000 Subject: [PATCH 23/24] Factor out garbage-collection code --- cluster/kubernetes/sync.go | 64 ++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index fb8930050..2dfba3b1d 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -89,33 +89,11 @@ func (c *Cluster) Sync(spec cluster.SyncSet) error { c.muSyncErrors.RUnlock() if c.GC { - orphanedResources := makeChangeSet() - - clusterResources, err := c.getResourcesInSyncSet(spec.Name) - if err != nil { - return errors.Wrap(err, "collating resources in cluster for calculating garbage collection") - } - - for resourceID, res := range clusterResources { - actual := res.GetChecksum() - expected, ok := checksums[resourceID] - - switch { - case !ok: // was not recorded as having been staged for application - c.logger.Log("info", "cluster resource not in resources to be synced; deleting", "resource", resourceID) - orphanedResources.stage("delete", res.ResourceID(), "", res.IdentifyingBytes()) - case actual != expected: - c.logger.Log("warning", "resource to be synced has not been updated; skipping", "resource", resourceID) - continue - default: - // The checksum is the same, indicating that it was - // applied earlier. Leave it alone. - } - } - - if deleteErrs := c.applier.apply(logger, orphanedResources, nil); len(deleteErrs) > 0 { - errs = append(errs, deleteErrs...) + deleteErrs, gcFailure := c.collectGarbage(spec, checksums, logger) + if gcFailure != nil { + return gcFailure } + errs = append(errs, deleteErrs...) } // If `nil`, errs is a cluster.SyncError(nil) rather than error(nil), so it cannot be returned directly. @@ -129,6 +107,38 @@ func (c *Cluster) Sync(spec cluster.SyncSet) error { return errs } +func (c *Cluster) collectGarbage( + spec cluster.SyncSet, + checksums map[string]string, + logger log.Logger) (cluster.SyncError, error) { + + orphanedResources := makeChangeSet() + + clusterResources, err := c.getResourcesInSyncSet(spec.Name) + if err != nil { + return nil, errors.Wrap(err, "collating resources in cluster for calculating garbage collection") + } + + for resourceID, res := range clusterResources { + actual := res.GetChecksum() + expected, ok := checksums[resourceID] + + switch { + case !ok: // was not recorded as having been staged for application + c.logger.Log("info", "cluster resource not in resources to be synced; deleting", "resource", resourceID) + orphanedResources.stage("delete", res.ResourceID(), "", res.IdentifyingBytes()) + case actual != expected: + c.logger.Log("warning", "resource to be synced has not been updated; skipping", "resource", resourceID) + continue + default: + // The checksum is the same, indicating that it was + // applied earlier. Leave it alone. + } + } + + return c.applier.apply(logger, orphanedResources, nil), nil +} + // --- internals in support of Sync type kuberesource struct { @@ -414,7 +424,7 @@ func (c *Kubectl) apply(logger log.Logger, cs changeSet, errored map[flux.Resour // When deleting objects, the only real concern is that we don't // try to delete things that have already been deleted by - // Kubernete's GC -- most notably, resources in a namespace which + // Kubernetes' GC -- most notably, resources in a namespace which // is also being deleted. GC does not have the dependency ranking, // but we can use it as a shortcut to avoid the above problem at // least. From e949383181b55bec227afd0f985728fda2526217 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Wed, 27 Feb 2019 11:40:55 +0000 Subject: [PATCH 24/24] Document the experimental garbage collection feature --- cmd/fluxd/main.go | 2 +- site/daemon.md | 3 ++- site/faq.md | 13 +++++++++++ site/garbagecollection.md | 45 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 site/garbagecollection.md diff --git a/cmd/fluxd/main.go b/cmd/fluxd/main.go index eef6eac63..d5b5346b7 100644 --- a/cmd/fluxd/main.go +++ b/cmd/fluxd/main.go @@ -99,7 +99,7 @@ func main() { // syncing syncInterval = fs.Duration("sync-interval", 5*time.Minute, "apply config in git to cluster at least this often, even if there are no new commits") - syncGC = fs.Bool("sync-garbage-collection", false, "experimental; delete resources that are no longer in the git repo") + syncGC = fs.Bool("sync-garbage-collection", false, "experimental; delete resources that were created by fluxd, but are no longer in the git repo") // registry memcachedHostname = fs.String("memcached-hostname", "memcached", "hostname for memcached service.") diff --git a/site/daemon.md b/site/daemon.md index 948e84cae..49bbd490a 100644 --- a/site/daemon.md +++ b/site/daemon.md @@ -62,8 +62,9 @@ fluxd requires setup and offers customization though a multitude of flags. |--git-notes-ref | `flux` | ref to use for keeping commit annotations in git notes| |--git-poll-interval | `5m` | period at which to fetch any new commits from the git repo | |--git-timeout | `20s` | duration after which git operations time out | -|**syncing** | | control over how config is applied to the cluster | +|**syncing** | | control over how config is applied to the cluster | |--sync-interval | `5m` | apply the git config to the cluster at least this often. New commits may provoke more frequent syncs | +|--sync-garbage-collection | `false` | experimental: when set, fluxd will delete resources that it created, but are no longer present in git (see [garbage collection](./garbagecollection.md)) | |**registry cache** | | (none of these need overriding, usually) | |--memcached-hostname | `memcached` | hostname for memcached service to use for caching image metadata| |--memcached-timeout | `1s` | maximum time to wait before giving up on memcached requests| diff --git a/site/faq.md b/site/faq.md index 5fa390b2f..422235ab0 100644 --- a/site/faq.md +++ b/site/faq.md @@ -16,6 +16,7 @@ menu_order: 60 * [Is there any special directory layout I need in my git repo?](#is-there-any-special-directory-layout-i-need-in-my-git-repo) * [Why does Flux need a git ssh key with write access?](#why-does-flux-need-a-git-ssh-key-with-write-access) * [Does Flux automatically sync changes back to git?](#does-flux-automatically-sync-changes-back-to-git) + * [Will Flux delete resources when I remove them from git?](#will-flux-delete-resources-when-i-remove-them-from-git) * [How do I give Flux access to an image registry?](#how-do-i-give-flux-access-to-an-image-registry) * [How often does Flux check for new images?](#how-often-does-flux-check-for-new-images) * [How often does Flux check for new git commits (and can I make it sync faster)?](#how-often-does-flux-check-for-new-git-commits-and-can-i-make-it-sync-faster) @@ -145,6 +146,18 @@ For more information about Flux commands see [the fluxctl docs](./fluxctl.md). No. It applies changes to git only when a Flux command or API call makes them. +### Will Flux delete resources when I remove them from git? + +Flux has an experimental (for now) garbage collection feature, +enabled by passing the command-line flag `--sync-garbage-collection` +to fluxd. + +The garbage collection is conservative: it is designed to not delete +resources that were not created by fluxd. This means it will sometimes +_not_ delete resources that _were_ created by fluxd, when +reconfigured. Read more about garbage collection +[here](./garbagecollection.md). + ### How do I give Flux access to an image registry? Flux transparently looks at the image pull secrets that you attach to diff --git a/site/garbagecollection.md b/site/garbagecollection.md new file mode 100644 index 000000000..d7de28cda --- /dev/null +++ b/site/garbagecollection.md @@ -0,0 +1,45 @@ +## Garbage collection + +Part of syncing a cluster with a git repository is getting rid of +resources in the cluster that have been removed in the repository. You +can tell fluxd to do this "garbage collection" using the command-line +flag `--sync-garbage-collection`. It's important to know how it +operates, and appreciate its limitations, before enabling it. + +### How garbage collection works + +When garbage collection is enabled, syncing is done in two phases: + + 1. Apply all the manifests in the git repo (as delimited by the + branch and path arguments), and give each resource a label marking + it as having been synced from this source. + + 2. Ask the cluster for all the resources marked as being from this + source, and delete those that were not applied in step 1. + +In the above, "source" refers to the particular combination of git +repo URL, branch, and paths that this fluxd has been configured to +use, which is taken as identifying the resources under _this_ fluxd's +control. + +We need to be careful about identifying these accurately, since +getting it wrong could mean _not_ deleting resources that should be +deleted; or (much worse), deleting resources that are under another +fluxd's control. + +The definition of "source" affects how garbage collection behaves when +you reconfigure fluxd. It is intended to be conservative: it ensures +that fluxd will not delete resources that it did not create. + +### Limitations of this approach + +In general, if you change an element of the source (the git repo URL, +branch, and paths), there is a possiblility that resources no longer +present in the new source will be missed (i.e., not deleted) by +garbage collection, and you will need to delete them by hand. + +| Config change | What happens | +|-------------------|--------------| +| git URL or branch | If the manifests at the new git repo are the same, they will all be relabelled, and things will proceed as usual. If they are different, the resources from the old repo will be missed by garbage collection and will need to be deleted by hand | +| path added | Existing resources will be relabelled, and new resources (from manifests in the new path) will be created. Then things will proceed as usual. | +| path removed | The resources from manifests in the removed path will be missed by garbage collection, and will need to be deleted by hand. Other resources will be treated as usual. |