feat: exclude node scanning by node labels (#1239)

* feat: exclude node scanning by node labels Signed-off-by: chenk <hen.keinan@gmail.com>
aquasecurity · May 28, 2023 · ce4ca43 · ce4ca43
1 parent 000c153
commit ce4ca43
Show file tree

Hide file tree

Showing 8 changed files with 58 additions and 7 deletions.
diff --git a/deploy/helm/README.md b/deploy/helm/README.md
@@ -104,8 +104,9 @@ The values of the `OPERATOR_NAMESPACE` and `OPERATOR_TARGET_NAMESPACES` determin
 | `metrics.resourceLabelsPrefix`                 | `k8s_label`                               | Prefix that will be prepended to the labels names indicated in `report.ResourceLabels` when including them in the Prometheus metrics                                                                                                                                                                               |
 | `report.recordFailedChecksOnly`                | `"true"`                                  | this flag is to record only failed checks on misconfiguration reports (config-audit and rbac assessment)                                                                                                                                                                                                           |
 | `skipResourceByLabels`                         | N/A                                       | One-line comma-separated labels keys which trivy-operator will skip scanning on resources with matching labels. Example: `test,transient`                                                                                                                                                                          |
-| `node.collector.imageRef`                      | ghcr.io/aquasecurity/node-collector:0.0.6 | The imageRef use for node-collector job .                                                                                                                                                                                                                                                                          |
-| `node.collector.imagePullSecret`               | N/A                                       | imagePullSecret is the secret name to be used when pulling node-collector image from private registries .                                                                                                                                                                                                          |
+| `node.collector.imageRef`                      | ghcr.io/aquasecurity/node-collector:0.0.6 | The imageRef use for node-collector job .                                                                                                                                                                               |
+| `node.collector.imagePullSecret`               | N/A                                       | imagePullSecret is the secret name to be used when pulling node-collector image from private registries .                                                                                                                                                                         |
+| `nodeCollector.excludeNodes`                        | `""`                      | excludeNodes comma-separated node labels that the node-collector job should exclude from scanning (example kubernetes.io/arch=arm64,team=dev)                                                                                                                                                                                                                                      |
 
 ## Trivy Configuration
 

diff --git a/deploy/helm/templates/config.yaml b/deploy/helm/templates/config.yaml
@@ -22,6 +22,9 @@ data:
   {{- with .Values.trivyOperator.scanJobAutomountServiceAccountToken }}
   scanJob.automountServiceAccountToken: {{ . | quote }}
   {{- end }}
+  {{- with .Values.nodeCollector.excludeNodes }}
+  nodeCollector.excludeNodes: {{ . | quote }}
+  {{- end }}
   {{- with .Values.trivyOperator.scanJobPodTemplateLabels }}
   scanJob.podTemplateLabels: {{ . | quote }}
   {{- end }}

diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml
@@ -478,7 +478,8 @@ nodeCollector:
   # imagePullSecret is the secret name to be used when pulling node-collector image from private registries example : reg-secret
   # It is the user responsibility to create the secret for the private registry in `trivy-operator` namespace
   # imagePullSecret:
-
+  # excludeNodes comma-separated node labels that the node-collector job should exclude from scanning (example kubernetes.io/arch=arm64,team=dev)
+  excludeNodes:
   # node-collector pod volumeMounts definition for collecting config files information
   volumeMounts:
     - name: var-lib-etcd

diff --git a/docs/getting-started/installation/configuration.md b/docs/getting-started/installation/configuration.md
@@ -43,7 +43,6 @@ You can configure Trivy-Operator to control it's behavior and adapt it to your n
 
 The values of the `OPERATOR_NAMESPACE` and `OPERATOR_TARGET_NAMESPACES` determine the install mode, which in turn determines the multitenancy support of the operator.
 
-
 | MODE| OPERATOR_NAMESPACE | OPERATOR_TARGET_NAMESPACES | DESCRIPTION|
 |---|---|---|---|
 | OwnNamespace| `operators`| `operators`| The operator can be configured to watch events in the namespace it is deployed in.                             |
@@ -76,7 +75,8 @@ To change the target namespace from all namespaces to the `default` namespace ed
 |`report.recordFailedChecksOnly`| `"true"`| this flag is to record only failed checks on misconfiguration reports (config-audit and rbac assessment)
 | `skipResourceByLabels`| N/A| One-line comma-separated labels keys which trivy-operator will skip scanning on resources with matching labels. Example: `test,transient`|
 | `node.collector.imageRef`             | ghcr.io/aquasecurity/node-collector:0.0.6                | The imageRef use for node-collector job .                                                                                                                                                                                                                                                                                                               |
-| `node.collector.imagePullSecret`             | N/A                | imagePullSecret is the secret name to be used when pulling node-collector image from private registries .                                                                                                                                                                                                                                                                                                               |
+| `node.collector.imagePullSecret`             | N/A                | imagePullSecret is the secret name to be used when pulling node-collector image from private registries .                                                                                                                                                                                                                                                                 |
+| `nodeCollector.excludeNodes`                        | `""`                      | excludeNodes comma-separated node labels that the node-collector job should exclude from scanning (example kubernetes.io/arch=arm64,team=dev)                                                                                                                                                                                                                                      |
 
 ## Example - patch ConfigMap
 
@@ -124,5 +124,4 @@ kubectl patch cm trivy-operator-trivy-config -n trivy-system \
 
 [tolerations]: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration
 
-
 [prometheus]: https://github.com/prometheus
diff --git a/docs/settings.md b/docs/settings.md
@@ -13,6 +13,7 @@ ConfigMap:
 ```
 TRIVY_OPERATOR_NAMESPACE=<your trivy operator namespace>
 ```
+
 ```
 kubectl patch cm trivy-operator-trivy-config -n $TRIVY_OPERATOR_NAMESPACE \
   --type merge \
@@ -32,6 +33,7 @@ To set the GitHub token used by Trivy add the `trivy.githubToken` value to the `
 TRIVY_OPERATOR_NAMESPACE=<your trivy opersator namespace>
 GITHUB_TOKEN=<your token>
 ```
+
 ```
 kubectl patch secret trivy-operator-trivy-config -n $TRIVY_OPERATOR_NAMESPACE \
   --type merge \
@@ -66,6 +68,7 @@ configuration settings for common use cases. For example, switch Trivy from [Sta
 | `compliance.reportType`               | `summary`                   | this flag control the type of report generated summary or all                |
 | `compliance.cron`                     | `0 */6 * * *`                   | this flag control the cron interval for compliance report generation                |
 | `scanJob.compressLogs`                                         | `"true"`                              | Control whether scanjob output should be compressed                                                                                                                                     |
+| `nodeCollector.excludeNodes`                        | `""`                      | excludeNodes comma-separated node labels that the node-collector job should exclude from scanning (example kubernetes.io/arch=arm64,team=dev)                                                                                                                                                                                                                                      |
 
 !!! tip
     You can delete a configuration key.For example, the following `kubectl patch` command deletes the `trivy.httpProxy` key:

diff --git a/pkg/configauditreport/controller/node.go b/pkg/configauditreport/controller/node.go
@@ -6,6 +6,7 @@ import (
 	"github.com/aquasecurity/trivy-operator/pkg/configauditreport"
 	"github.com/aquasecurity/trivy-operator/pkg/infraassessment"
 	"github.com/aquasecurity/trivy-operator/pkg/operator/jobs"
+	"github.com/aquasecurity/trivy-operator/pkg/operator/predicate"
 	. "github.com/aquasecurity/trivy-operator/pkg/operator/predicate"
 	"github.com/aquasecurity/trivy-operator/pkg/trivyoperator"
 
@@ -46,8 +47,13 @@ type NodeReconciler struct {
 // +kubebuilder:rbac:groups=aquasecurity.github.io,resources=clusterinfraassessmentreports,verbs=get;list;watch;create;update;patch;delete
 
 func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	excludeNodePredicate, err := predicate.ExcludeNode(r.ConfigData)
+	if err != nil {
+		return err
+	}
+
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&corev1.Node{}, builder.WithPredicates(IsLinuxNode)).
+		For(&corev1.Node{}, builder.WithPredicates(IsLinuxNode, predicate.Not((excludeNodePredicate)))).
 		Owns(&v1alpha1.ClusterInfraAssessmentReport{}).
 		Complete(r.reconcileNodes())
 }

diff --git a/pkg/operator/predicate/predicate.go b/pkg/operator/predicate/predicate.go
@@ -125,6 +125,25 @@ var IsLinuxNode = predicate.NewPredicateFuncs(func(obj client.Object) bool {
 	return false
 })
 
+var ExcludeNode = func(config trivyoperator.ConfigData) (predicate.Predicate, error) {
+	excludeNodes, err := config.GetNodeCollectorExcludeNodes()
+	if err != nil {
+		return nil, err
+	}
+	return predicate.NewPredicateFuncs(func(obj client.Object) bool {
+		if len(excludeNodes) == 0 {
+			return false
+		}
+		var matchingLabels int
+		for key, val := range excludeNodes {
+			if lVal, ok := obj.GetLabels()[key]; ok && lVal == val {
+				matchingLabels++
+			}
+		}
+		return matchingLabels == len(excludeNodes)
+	}), nil
+}
+
 // IsLeaderElectionResource returns true for resources used in leader election, means resources
 // annotated with resourcelock.LeaderElectionRecordAnnotationKey.
 var IsLeaderElectionResource = predicate.NewPredicateFuncs(func(obj client.Object) bool {

diff --git a/pkg/trivyoperator/config.go b/pkg/trivyoperator/config.go
@@ -59,6 +59,7 @@ const (
 	keyScanJobTolerations                = "scanJob.tolerations"
 	KeyScanJobcompressLogs               = "scanJob.compressLogs"
 	KeyNodeCollectorVolumes              = "nodeCollector.volumes"
+	KeyNodeCollectorExcludeNodes         = "nodeCollector.excludeNodes"
 	KeyNodeCollectorVolumeMounts         = "nodeCollector.volumeMounts"
 	keyScanJobNodeSelector               = "scanJob.nodeSelector"
 	keyScanJobAnnotations                = "scanJob.annotations"
@@ -253,6 +254,24 @@ func (c ConfigData) GetScanJobAnnotations() (map[string]string, error) {
 	return scanJobAnnotationsMap, nil
 }
 
+func (c ConfigData) GetNodeCollectorExcludeNodes() (map[string]string, error) {
+	nodeCollectorExcludeNodesStr, found := c[KeyNodeCollectorExcludeNodes]
+	if !found || strings.TrimSpace(nodeCollectorExcludeNodesStr) == "" {
+		return map[string]string{}, nil
+	}
+
+	nodeCollectorExcludeNodesMap := map[string]string{}
+	for _, excludeNode := range strings.Split(nodeCollectorExcludeNodesStr, ",") {
+		sepByEqual := strings.Split(excludeNode, "=")
+		if len(sepByEqual) != 2 {
+			return map[string]string{}, fmt.Errorf("failed parsing incorrectly formatted exclude nodes values: %s", nodeCollectorExcludeNodesStr)
+		}
+		key, value := sepByEqual[0], sepByEqual[1]
+		nodeCollectorExcludeNodesMap[key] = value
+	}
+	return nodeCollectorExcludeNodesMap, nil
+}
+
 func (c ConfigData) GetScanJobPodTemplateLabels() (labels.Set, error) {
 	scanJobPodTemplateLabelsStr, found := c[keyScanJobPodTemplateLabels]
 	if !found || strings.TrimSpace(scanJobPodTemplateLabelsStr) == "" {