Skip to content

Commit

Permalink
Add watch filtering by applyset label
Browse files Browse the repository at this point in the history
- Add "applyset.kubernetes.io/part-of" label to managed objects
- Add "applyset.kubernetes.io/id" label to RootSyncs/RepoSyncs
- Add "applyset.k8s.io/tooling" annotation to RootSyncs/RepoSyncs
- Error if the RSync already has a tooling annotation for a
  different tool (unlikely, but required by KEP).
- Generate ApplySet IDs using kubectl code for compatibility.
- Update TestKubectlCreatesManagedNamespaceResourceMultiRepo &
  TestAddUpdateDeleteLabels with new labels & annotations
  • Loading branch information
karlkfi committed Jul 30, 2024
1 parent 032136d commit 0d6c58a
Show file tree
Hide file tree
Showing 19 changed files with 751 additions and 249 deletions.
573 changes: 363 additions & 210 deletions e2e/testcases/managed_resources_test.go

Large diffs are not rendered by default.

26 changes: 14 additions & 12 deletions e2e/testcases/preserve_fields_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,11 @@ func TestAddUpdateDeleteLabels(t *testing.T) {
nt.T.Fatal(err)
}

var defaultLabels = []string{metadata.ManagedByKey, metadata.DeclaredVersionLabel}
var defaultLabels = []string{
metadata.ManagedByKey,
metadata.DeclaredVersionLabel,
metadata.ApplySetPartOfLabel,
}

// Checking that the configmap with no labels appears on cluster, and
// that no user labels are specified
Expand All @@ -350,12 +354,13 @@ func TestAddUpdateDeleteLabels(t *testing.T) {
nt.T.Fatal(err)
}

// Checking that label is updated after syncing an update.
err = nt.Validate(cmName, ns, &corev1.ConfigMap{},
testpredicates.HasExactlyLabelKeys(append(defaultLabels, "baz")...))
if err != nil {
nt.T.Fatal(err)
}
var updatedLabels []string
updatedLabels = append(updatedLabels, defaultLabels...)
updatedLabels = append(updatedLabels, "baz")

// Checking that label was added after syncing.
nt.Must(nt.Validate(cmName, ns, &corev1.ConfigMap{},
testpredicates.HasExactlyLabelKeys(updatedLabels...)))

delete(cm.Labels, "baz")
nt.Must(nt.RootRepos[configsync.RootSyncName].Add(cmPath, cm))
Expand All @@ -365,11 +370,8 @@ func TestAddUpdateDeleteLabels(t *testing.T) {
}

// Check that the label is deleted after syncing.
err = nt.Validate(cmName, ns, &corev1.ConfigMap{},
testpredicates.HasExactlyLabelKeys(metadata.ManagedByKey, metadata.DeclaredVersionLabel))
if err != nil {
nt.T.Fatal(err)
}
nt.Must(nt.Validate(cmName, ns, &corev1.ConfigMap{},
testpredicates.HasExactlyLabelKeys(defaultLabels...)))

nt.MetricsExpectations.AddObjectApply(configsync.RootSyncKind, rootSyncNN, nsObj)
nt.MetricsExpectations.AddObjectApply(configsync.RootSyncKind, rootSyncNN, cm)
Expand Down
35 changes: 20 additions & 15 deletions pkg/applier/applier_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,19 +89,21 @@ func TestApply(t *testing.T) {
abandonObj := deploymentObj.DeepCopy()
abandonObj.SetName("abandon-me")
abandonObj.SetAnnotations(map[string]string{
common.LifecycleDeleteAnnotation: common.PreventDeletion,
common.LifecycleDeleteAnnotation: common.PreventDeletion, // not removed
metadata.ResourceManagementKey: metadata.ResourceManagementEnabled,
metadata.ResourceIDKey: core.GKNN(abandonObj),
metadata.ResourceManagerKey: resourceManager,
metadata.OwningInventoryKey: "anything",
metadata.SyncTokenAnnotationKey: "anything",
"example-to-not-delete": "anything",
"example-to-not-delete": "anything", // not removed
})
abandonObj.SetLabels(map[string]string{
metadata.ManagedByKey: metadata.ManagedByValue,
metadata.SystemLabel: "anything",
metadata.ArchLabel: "anything",
"example-to-not-delete": "anything",
metadata.ManagedByKey: metadata.ManagedByValue,
metadata.SystemLabel: "anything",
metadata.ArchLabel: "anything",
// TODO: remove ApplySet metadata on abandon (b/355534413)
metadata.ApplySetPartOfLabel: "anything", // not removed
"example-to-not-delete": "anything", // not removed
})
abandonObjID := core.IDOf(abandonObj)

Expand Down Expand Up @@ -322,15 +324,18 @@ func TestApply(t *testing.T) {
expectedServerObjs: []client.Object{
func() client.Object {
obj := abandonObj.DeepCopy()
obj.SetAnnotations(map[string]string{
common.LifecycleDeleteAnnotation: common.PreventDeletion,
// all configsync annotations removed
"example-to-not-delete": "anything",
})
obj.SetLabels(map[string]string{
// all configsync labels removed
"example-to-not-delete": "anything",
})
// all configsync annotations removed
core.RemoveAnnotations(obj,
metadata.ResourceManagementKey,
metadata.ResourceIDKey,
metadata.ResourceManagerKey,
metadata.OwningInventoryKey,
metadata.SyncTokenAnnotationKey)
// all configsync labels removed
core.RemoveLabels(obj,
metadata.ManagedByKey,
metadata.SystemLabel,
metadata.ArchLabel)
obj.SetUID("1")
obj.SetResourceVersion("2")
obj.SetGeneration(1)
Expand Down
16 changes: 15 additions & 1 deletion pkg/applier/clientset.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,15 @@ import (

"github.com/GoogleContainerTools/kpt/pkg/live"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/klog/v2"
"k8s.io/kubectl/pkg/cmd/util"
"kpt.dev/configsync/pkg/metadata"
"sigs.k8s.io/cli-utils/pkg/apply"
"sigs.k8s.io/cli-utils/pkg/apply/event"
"sigs.k8s.io/cli-utils/pkg/inventory"
"sigs.k8s.io/cli-utils/pkg/kstatus/watcher"
"sigs.k8s.io/cli-utils/pkg/object"
"sigs.k8s.io/controller-runtime/pkg/client"
)
Expand Down Expand Up @@ -53,7 +56,7 @@ type ClientSet struct {
}

// NewClientSet constructs a new ClientSet.
func NewClientSet(c client.Client, configFlags *genericclioptions.ConfigFlags, statusMode string) (*ClientSet, error) {
func NewClientSet(c client.Client, configFlags *genericclioptions.ConfigFlags, statusMode, applySetID string) (*ClientSet, error) {
matchVersionKubeConfigFlags := util.NewMatchVersionFlags(configFlags)
f := util.NewFactory(matchVersionKubeConfigFlags)

Expand All @@ -71,9 +74,19 @@ func NewClientSet(c client.Client, configFlags *genericclioptions.ConfigFlags, s
return nil, err
}

// Only watch objects applied by this reconciler for status updates.
// This reduces both the number of events processed and the memory used by
// the informer cache.
watchFilters := &watcher.Filters{
Labels: labels.Set{
metadata.ApplySetPartOfLabel: applySetID,
}.AsSelector(),
}

applier, err := apply.NewApplierBuilder().
WithInventoryClient(invClient).
WithFactory(f).
WithStatusWatcherFilters(watchFilters).
Build()
if err != nil {
return nil, err
Expand All @@ -82,6 +95,7 @@ func NewClientSet(c client.Client, configFlags *genericclioptions.ConfigFlags, s
destroyer, err := apply.NewDestroyerBuilder().
WithInventoryClient(invClient).
WithFactory(f).
WithStatusWatcherFilters(watchFilters).
Build()
if err != nil {
return nil, err
Expand Down
78 changes: 78 additions & 0 deletions pkg/applyset/applyset.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package applyset

import (
"fmt"
"strings"

"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/cli-runtime/pkg/resource"
kubectlapply "k8s.io/kubectl/pkg/cmd/apply"
"kpt.dev/configsync/pkg/declared"
"kpt.dev/configsync/pkg/kinds"
"kpt.dev/configsync/pkg/metadata"
)

// IDFromSync generates an ApplySet ID for the RootSync or RepoSync as
// an ApplySet parent.
func IDFromSync(syncName string, syncScope declared.Scope) string {
return FromSync(syncName, syncScope, nil, nil).ID()
}

// FromSync constructs a new ApplySet for the specified RootSync or RepoSync.
// The RESTMapper & RESTClient are optional, depending on which methods you plan
// to call.
func FromSync(syncName string, syncScope declared.Scope, mapper meta.RESTMapper, client resource.RESTClient) *kubectlapply.ApplySet {
tooling := kubectlapply.ApplySetTooling{
Name: metadata.ApplySetToolingName,
Version: metadata.ApplySetToolingVersion,
}
parent := &kubectlapply.ApplySetParentRef{
Name: syncName,
Namespace: syncScope.SyncNamespace(),
}
switch syncScope {
case declared.RootScope:
parent.RESTMapping = kinds.RootSyncRESTMapping()
default:
parent.RESTMapping = kinds.RepoSyncRESTMapping()
}
return kubectlapply.NewApplySet(parent, tooling, mapper, client)
}

// ParseTooling parses the tooling value with format NAME/VERSION.
// Returns an error if the input does not contain at least one slash (`/`).
func ParseTooling(toolingValue string) (kubectlapply.ApplySetTooling, error) {
parts := strings.Split(toolingValue, "/")
if len(parts) >= 2 {
return kubectlapply.ApplySetTooling{
Name: strings.Join(parts[:len(parts)-1], "/"),
Version: parts[len(parts)-1],
}, nil
}
// Invalid
return kubectlapply.ApplySetTooling{},
fmt.Errorf("invalid applyset tooling value: expected NAME/VERSION: %s", toolingValue)
}

// FormatTooling returns a formatted value for the ApplySet tooling annotation.
func FormatTooling(name, version string) string {
tooling := kubectlapply.ApplySetTooling{
Name: name,
Version: version,
}
return tooling.String()
}
9 changes: 9 additions & 0 deletions pkg/core/decorate.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,15 @@ func RemoveAnnotations(obj Annotated, annotations ...string) {
obj.SetAnnotations(as)
}

// RemoveLabels removes the passed set of labels from obj.
func RemoveLabels(obj Labeled, labels ...string) {
actual := obj.GetLabels()
for _, label := range labels {
delete(actual, label)
}
obj.SetLabels(actual)
}

// AddAnnotations adds the specified annotations to the object.
func AddAnnotations(obj Annotated, annotations map[string]string) {
existing := obj.GetAnnotations()
Expand Down
19 changes: 19 additions & 0 deletions pkg/kinds/resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package kinds

import (
appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime/schema"
configsyncv1beta1 "kpt.dev/configsync/pkg/api/configsync/v1beta1"
)
Expand All @@ -34,3 +35,21 @@ func RootSyncResource() schema.GroupVersionResource {
func RepoSyncResource() schema.GroupVersionResource {
return configsyncv1beta1.SchemeGroupVersion.WithResource("reposyncs")
}

// RootSyncRESTMapping returns the canonical RootSync RESTMapping.
func RootSyncRESTMapping() *meta.RESTMapping {
return &meta.RESTMapping{
Resource: RootSyncResource(),
GroupVersionKind: RootSyncV1Beta1(),
Scope: meta.RESTScopeNamespace,
}
}

// RepoSyncRESTMapping returns the canonical RepoSync RESTMapping.
func RepoSyncRESTMapping() *meta.RESTMapping {
return &meta.RESTMapping{
Resource: RepoSyncResource(),
GroupVersionKind: RepoSyncV1Beta1(),
Scope: meta.RESTScopeNamespace,
}
}
56 changes: 56 additions & 0 deletions pkg/metadata/applyset.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metadata

import (
kubectlapply "k8s.io/kubectl/pkg/cmd/apply"
"kpt.dev/configsync/pkg/api/configsync"
)

// Labels with the `applyset.kubernetes.io/` prefix.
const (
// ApplySetPartOfLabel is the key of the label which indicates that the
// object is a member of an ApplySet. The value of the label MUST match the
// value of ApplySetParentIDLabel on the parent object.
ApplySetPartOfLabel = kubectlapply.ApplysetPartOfLabel

// ApplySetParentIDLabel is the key of the label that makes object an
// ApplySet parent object. Its value MUST use the format specified in
// k8s.io/kubectl/pkg/cmd/apply.V1ApplySetIdFormat.
ApplySetParentIDLabel = kubectlapply.ApplySetParentIDLabel
)

// Annotations with the `applyset.kubernetes.io/` prefix.
const (
// ApplySetToolingAnnotation is the key of the label that indicates which
// tool is used to manage this ApplySet. Tooling should refuse to mutate
// ApplySets belonging to other tools. The value must be in the format
// <toolname>/<semver>. Example value: "kubectl/v1.27" or "helm/v3" or
// "kpt/v1.0.0"
ApplySetToolingAnnotation = kubectlapply.ApplySetToolingAnnotation

// ApplySetToolingName is the name used to represent Config Sync in the
// ApplySet tooling annotation.
ApplySetToolingName = configsync.GroupName

// ApplySetToolingVersion is the version used to represent Config Sync in
// the ApplySet tooling annotation.
//
// The ApplySetKEP and kubectl require this to be a semantic version,
// implying that it should be the version of the tool. But we're using a
// static version instead, to allow listing all objects managed Config Sync,
// regardless of version.
ApplySetToolingVersion = "v1"
)
3 changes: 3 additions & 0 deletions pkg/parse/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"fmt"

"kpt.dev/configsync/pkg/applier"
"kpt.dev/configsync/pkg/applyset"
"kpt.dev/configsync/pkg/core"
"kpt.dev/configsync/pkg/declared"
"kpt.dev/configsync/pkg/importer/analyzer/ast"
Expand All @@ -37,10 +38,12 @@ func addAnnotationsAndLabels(objs []ast.FileObject, scope declared.Scope, syncNa
if err != nil {
return fmt.Errorf("marshaling sourceContext: %w", err)
}
applySetID := applyset.IDFromSync(syncName, scope)
inventoryID := applier.InventoryID(syncName, scope.SyncNamespace())
manager := declared.ResourceManager(scope, syncName)
for _, obj := range objs {
core.SetLabel(obj, metadata.ManagedByKey, metadata.ManagedByValue)
core.SetLabel(obj, metadata.ApplySetPartOfLabel, applySetID)
core.SetAnnotation(obj, metadata.GitContextKey, string(gcVal))
core.SetAnnotation(obj, metadata.ResourceManagerKey, manager)
core.SetAnnotation(obj, metadata.SyncTokenAnnotationKey, commitHash)
Expand Down
Loading

0 comments on commit 0d6c58a

Please sign in to comment.