Add default Unprotected ResourceMatcher (#5225)

* Add default Unprotected ResourceMatcher

* Add GVR filters for CRDs (#5234)

* Add package to discover API resources.

* Change package name discover -> discovery

* Add Resource Matcher for CRDs

* Add filter support for specs (#5249)

pkg/filter/crd.go

@@ -0,0 +1,32 @@
+package filter
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	crdclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// CRDMatcher returns a ResourceMatcher that matches all CRs in this cluster.
+func CRDMatcher(ctx context.Context, cli crdclient.Interface) (ResourceMatcher, error) {
+	crds, err := cli.ApiextensionsV1beta1().CustomResourceDefinitions().List(metav1.ListOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to query CRDs in cluster")
+	}
+	return crdsToMatcher(crds.Items), nil
+}
+
+func crdsToMatcher(crds []apiextensions.CustomResourceDefinition) ResourceMatcher {
+	gvrs := make(ResourceMatcher, 0, len(crds))
+	for _, crd := range crds {
+		gvr := ResourceRequirement{
+			Group:    crd.Spec.Group,
+			Version:  crd.Spec.Version,
+			Resource: crd.Spec.Names.Plural,
+		}
+		gvrs = append(gvrs, gvr)
+	}
+	return gvrs
+}

pkg/filter/crd_test.go

@@ -0,0 +1,35 @@
+package filter
+
+import (
+	"context"
+
+	. "gopkg.in/check.v1"
+	crdclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+
+	"github.com/kanisterio/kanister/pkg/discovery"
+	"github.com/kanisterio/kanister/pkg/kube"
+)
+
+type CRDSuite struct{}
+
+var _ = Suite(&CRDSuite{})
+
+func (s *CRDSuite) TestCRDMatcher(c *C) {
+	ctx := context.Background()
+	cfg, err := kube.LoadConfig()
+	c.Assert(err, IsNil)
+	cli, err := crdclient.NewForConfig(cfg)
+	c.Assert(err, IsNil)
+
+	g, err := CRDMatcher(ctx, cli)
+	c.Assert(err, IsNil)
+
+	gvrs, err := discovery.NamespacedGVRs(ctx, cli.Discovery())
+	c.Assert(err, IsNil)
+	c.Assert(gvrs, Not(HasLen), 0)
+
+	// We assume there's at least one CRD in the cluster.
+	c.Assert(g.Include(gvrs), Not(HasLen), 0)
+	c.Assert(g.Exclude(gvrs), Not(HasLen), 0)
+	c.Assert(len(g.Exclude(gvrs))+len(g.Include(gvrs)), Equals, len(gvrs))
+}

pkg/filter/default.go

@@ -0,0 +1,28 @@
+package filter
+
+// UnprotectedMatcher will match all the known resources that don't make sense
+// to protect when backing up an application.
+var UnprotectedMatcher ResourceMatcher = ResourceMatcher{
+	ResourceRequirement{Group: "", Version: "v1", Resource: "bindings"},
+	ResourceRequirement{Group: "", Version: "v1", Resource: "componentstatuses"},
+	ResourceRequirement{Group: "", Version: "v1", Resource: "endpoints"},
+	ResourceRequirement{Group: "", Version: "v1", Resource: "events"},
+	ResourceRequirement{Group: "", Version: "v1", Resource: "limitranges"},
+	ResourceRequirement{Group: "", Version: "v1", Resource: "namespaces"},
+	ResourceRequirement{Group: "", Version: "v1", Resource: "nodes"},
+	ResourceRequirement{Group: "", Version: "v1", Resource: "pods"},
+	ResourceRequirement{Group: "", Version: "v1", Resource: "podtemplates"},
+	ResourceRequirement{Group: "", Version: "v1", Resource: "replicationcontrollers"},
+	ResourceRequirement{Group: "", Version: "v1", Resource: "resourcequotas"},
+	ResourceRequirement{Group: "", Version: "v1", Resource: "serviceaccounts"},
+	ResourceRequirement{Group: "extensions", Version: "v1beta1"},
+	ResourceRequirement{Group: "apps", Version: "v1", Resource: "controllerrevisions"},
+	ResourceRequirement{Group: "apps", Version: "v1", Resource: "replicasets"},
+	ResourceRequirement{Group: "events.k8s.io", Version: "v1beta1", Resource: "events"},
+	ResourceRequirement{Group: "authorization.k8s.io"},
+	ResourceRequirement{Group: "autoscaling", Version: "v1", Resource: "horizontalpodautoscalers"},
+	ResourceRequirement{Group: "networking.k8s.io", Version: "v1", Resource: "networkpolicies"},
+	ResourceRequirement{Group: "policy", Version: "v1beta1"},
+	ResourceRequirement{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "roles"},
+	ResourceRequirement{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "rolebindings"},
+}

pkg/filter/default_test.go

@@ -0,0 +1,135 @@
+package filter
+
+import (
+	"context"
+
+	. "gopkg.in/check.v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"github.com/kanisterio/kanister/pkg/discovery"
+	"github.com/kanisterio/kanister/pkg/kube"
+)
+
+type DefaultSuite struct {
+	gvrs []schema.GroupVersionResource
+}
+
+var _ = Suite(&DefaultSuite{})
+
+func (s *DefaultSuite) SetUpSuite(c *C) {
+	ctx := context.Background()
+	cli, err := kube.NewClient()
+	c.Assert(err, IsNil)
+	s.gvrs, err = discovery.AllGVRs(ctx, cli.Discovery())
+	c.Assert(err, IsNil)
+
+}
+
+func gvrSet(gvrs []schema.GroupVersionResource) map[schema.GroupVersionResource]struct{} {
+	s := make(map[schema.GroupVersionResource]struct{}, len(gvrs))
+	for _, gvr := range gvrs {
+		s[gvr] = struct{}{}
+	}
+	return s
+}
+
+// knownCoreGVRs returns specific GVRs that will match the CoreGroups filter.
+// This list may change slightly between different clusters. This list is likely
+// to be common between different clusters.
+func knownCoreGVRs() []schema.GroupVersionResource {
+	return []schema.GroupVersionResource{
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "serviceaccounts"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "persistentvolumes"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "endpoints"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "podtemplates"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "events"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "bindings"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "replicationcontrollers"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "nodes"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "resourcequotas"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "namespaces"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "componentstatuses"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "persistentvolumeclaims"},
+		schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "deployments"},
+		schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "replicasets"},
+		schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "networkpolicies"},
+		schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "daemonsets"},
+		schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "ingresses"},
+		schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "replicationcontrollers"},
+		schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "podsecuritypolicies"},
+		schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"},
+		schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "statefulsets"},
+		schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "replicasets"},
+		schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"},
+		schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "controllerrevisions"},
+		schema.GroupVersionResource{Group: "authorization.k8s.io", Version: "v1", Resource: "selfsubjectaccessreviews"},
+		schema.GroupVersionResource{Group: "authorization.k8s.io", Version: "v1", Resource: "subjectaccessreviews"},
+		schema.GroupVersionResource{Group: "authorization.k8s.io", Version: "v1", Resource: "selfsubjectrulesreviews"},
+		schema.GroupVersionResource{Group: "authorization.k8s.io", Version: "v1", Resource: "localsubjectaccessreviews"},
+		schema.GroupVersionResource{Group: "autoscaling", Version: "v1", Resource: "horizontalpodautoscalers"},
+		schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"},
+		schema.GroupVersionResource{Group: "batch", Version: "v1beta1", Resource: "cronjobs"},
+		schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "networkpolicies"},
+		schema.GroupVersionResource{Group: "policy", Version: "v1beta1", Resource: "poddisruptionbudgets"},
+		schema.GroupVersionResource{Group: "policy", Version: "v1beta1", Resource: "podsecuritypolicies"},
+	}
+}
+
+func knownProtectedGVRs() []schema.GroupVersionResource {
+	return []schema.GroupVersionResource{
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"},
+		schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"},
+		schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "statefulsets"},
+		schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"},
+		schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"},
+		schema.GroupVersionResource{Group: "batch", Version: "v1beta1", Resource: "cronjobs"},
+	}
+}
+
+func knownUnprotectedGVRs() []schema.GroupVersionResource {
+	return []schema.GroupVersionResource{
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "serviceaccounts"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "endpoints"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "podtemplates"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "events"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "bindings"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "replicationcontrollers"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "nodes"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "resourcequotas"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "namespaces"},
+		schema.GroupVersionResource{Group: "", Version: "v1", Resource: "componentstatuses"},
+		schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "replicasets"},
+		schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "networkpolicies"},
+		schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "replicationcontrollers"},
+		schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "podsecuritypolicies"},
+		schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "replicasets"},
+		schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "controllerrevisions"},
+		schema.GroupVersionResource{Group: "authorization.k8s.io", Version: "v1", Resource: "selfsubjectaccessreviews"},
+		schema.GroupVersionResource{Group: "authorization.k8s.io", Version: "v1", Resource: "subjectaccessreviews"},
+		schema.GroupVersionResource{Group: "authorization.k8s.io", Version: "v1", Resource: "selfsubjectrulesreviews"},
+		schema.GroupVersionResource{Group: "authorization.k8s.io", Version: "v1", Resource: "localsubjectaccessreviews"},
+		schema.GroupVersionResource{Group: "autoscaling", Version: "v1", Resource: "horizontalpodautoscalers"},
+		schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "networkpolicies"},
+		schema.GroupVersionResource{Group: "policy", Version: "v1beta1", Resource: "poddisruptionbudgets"},
+		schema.GroupVersionResource{Group: "policy", Version: "v1beta1", Resource: "podsecuritypolicies"},
+	}
+}
+
+func (s *DefaultSuite) TestUnprotectedGVRs(c *C) {
+	protected := gvrSet(UnprotectedMatcher.Exclude(s.gvrs))
+	for _, gvr := range knownProtectedGVRs() {
+		_, ok := protected[gvr]
+		c.Assert(ok, Equals, true, Commentf("GVR should be in protected list: %v", gvr))
+	}
+	for _, gvr := range knownUnprotectedGVRs() {
+		_, ok := protected[gvr]
+		c.Assert(ok, Equals, false, Commentf("GVR should not be in protected list: %v", gvr))
+	}
+}

pkg/filter/unstructured.go

@@ -0,0 +1,34 @@
+package filter
+
+import (
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type Specs map[schema.GroupVersionResource][]unstructured.Unstructured
+
+func (s Specs) keys() []schema.GroupVersionResource {
+	gvrs := make([]schema.GroupVersionResource, 0, len(s))
+	for gvr := range s {
+		gvrs = append(gvrs, gvr)
+	}
+	return gvrs
+}
+
+func (s Specs) Include(g ResourceMatcher) Specs {
+	gvrs := g.Include(s.keys())
+	ret := make(Specs, len(gvrs))
+	for _, gvr := range gvrs {
+		ret[gvr] = s[gvr]
+	}
+	return ret
+}
+
+func (s Specs) Exclude(g ResourceMatcher) Specs {
+	gvrs := g.Exclude(s.keys())
+	ret := make(Specs, len(gvrs))
+	for _, gvr := range gvrs {
+		ret[gvr] = s[gvr]
+	}
+	return ret
+}

pkg/filter/unstructured_test.go

@@ -0,0 +1,68 @@
+package filter
+
+import (
+	. "gopkg.in/check.v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type UnstructuredSuite struct {
+}
+
+var _ = Suite(&UnstructuredSuite{})
+
+func (s *UnstructuredSuite) TestIncludeExclude(c *C) {
+	for _, tc := range []struct {
+		s       Specs
+		gvr     ResourceMatcher
+		include Specs
+		exclude Specs
+	}{
+		{
+			s:       nil,
+			gvr:     nil,
+			include: Specs{},
+			exclude: Specs{},
+		},
+		{
+			s:       Specs{},
+			gvr:     ResourceMatcher{},
+			include: Specs{},
+			exclude: Specs{},
+		},
+		{
+			s: Specs{
+				schema.GroupVersionResource{Group: "mygroup"}: nil,
+			},
+			gvr: ResourceMatcher{},
+			include: Specs{
+				schema.GroupVersionResource{Group: "mygroup"}: nil,
+			},
+			exclude: Specs{
+				schema.GroupVersionResource{Group: "mygroup"}: nil,
+			},
+		},
+		{
+			s: Specs{
+				schema.GroupVersionResource{Group: "mygroup"}: nil,
+			},
+			gvr: ResourceMatcher{ResourceRequirement{Group: "mygroup"}},
+			include: Specs{
+				schema.GroupVersionResource{Group: "mygroup"}: nil,
+			},
+			exclude: Specs{},
+		},
+		{
+			s: Specs{
+				schema.GroupVersionResource{Group: "mygroup"}: nil,
+			},
+			gvr:     ResourceMatcher{ResourceRequirement{Group: "yourgroup"}},
+			include: Specs{},
+			exclude: Specs{
+				schema.GroupVersionResource{Group: "mygroup"}: nil,
+			},
+		},
+	} {
+		c.Check(tc.s.Include(tc.gvr), DeepEquals, tc.include)
+		c.Check(tc.s.Exclude(tc.gvr), DeepEquals, tc.exclude)
+	}
+}