From 88e9d092e4504824ce02bc0452c80f1c0dabc069 Mon Sep 17 00:00:00 2001 From: Alexander Brand Date: Wed, 1 May 2019 11:41:28 -0400 Subject: [PATCH] Add audit logging support with a default logging policy Enable Kubernetes API audit logging with a default logging policy. When desired, users can provide their custom audit policy rules using an ansible variable. Fixes #138 Signed-off-by: Alexander Brand --- .../roles/kubernetes-common/defaults/main.yml | 16 ++ .../roles/kubernetes-master/defaults/main.yml | 1 + .../roles/kubernetes-master/tasks/main.yml | 5 + .../kubernetes/apiserver-audit-policy.yaml | 162 ++++++++++++++++++ 4 files changed, 184 insertions(+) create mode 100644 ansible/roles/kubernetes-master/templates/etc/kubernetes/apiserver-audit-policy.yaml diff --git a/ansible/roles/kubernetes-common/defaults/main.yml b/ansible/roles/kubernetes-common/defaults/main.yml index 6494130..82fe4c9 100644 --- a/ansible/roles/kubernetes-common/defaults/main.yml +++ b/ansible/roles/kubernetes-common/defaults/main.yml @@ -16,6 +16,22 @@ kubernetes_common_kubeadm_config: controlPlaneEndpoint: "{{ kubernetes_common_api_fqdn }}" apiServerExtraArgs: "endpoint-reconciler-type": "lease" + "audit-policy-file": "/etc/kubernetes/apiserver-audit-policy.yaml" + "audit-log-path": "/var/log/kubernetes/kube-apiserver-audit.log" + "audit-log-maxage": "30" # days + "audit-log-maxsize": "100" # megabytes + "audit-log-maxbackup": "1" + apiServerExtraVolumes: + - name: audit-policy + hostPath: /etc/kubernetes/apiserver-audit-policy.yaml + mountPath: /etc/kubernetes/apiserver-audit-policy.yaml + writable: false + pathType: File + - name: audit-logs + hostPath: /var/log/kubernetes + mountPath: /var/log/kubernetes + writable: true + pathType: DirectoryOrCreate apiServerCertSANs: "{{ kubernetes_common_api_ip | kube_lookup_hostname(kubernetes_common_api_fqdn, True) }}" etcd: external: diff --git a/ansible/roles/kubernetes-master/defaults/main.yml b/ansible/roles/kubernetes-master/defaults/main.yml index ed97d53..a9e8487 100644 --- a/ansible/roles/kubernetes-master/defaults/main.yml +++ b/ansible/roles/kubernetes-master/defaults/main.yml @@ -1 +1,2 @@ --- +kubernetes_common_audit_policy_rules: [] \ No newline at end of file diff --git a/ansible/roles/kubernetes-master/tasks/main.yml b/ansible/roles/kubernetes-master/tasks/main.yml index d0e891d..bb1d8e6 100644 --- a/ansible/roles/kubernetes-master/tasks/main.yml +++ b/ansible/roles/kubernetes-master/tasks/main.yml @@ -18,6 +18,11 @@ immediate: True when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' +- name: drop api-server audit policy template + template: + dest: /etc/kubernetes/apiserver-audit-policy.yaml + src: etc/kubernetes/apiserver-audit-policy.yaml + - include_tasks: install.yml when: wardroom_action == 'install' diff --git a/ansible/roles/kubernetes-master/templates/etc/kubernetes/apiserver-audit-policy.yaml b/ansible/roles/kubernetes-master/templates/etc/kubernetes/apiserver-audit-policy.yaml new file mode 100644 index 0000000..c022491 --- /dev/null +++ b/ansible/roles/kubernetes-master/templates/etc/kubernetes/apiserver-audit-policy.yaml @@ -0,0 +1,162 @@ +apiVersion: audit.k8s.io/v1beta1 +kind: Policy +rules: +{% if kubernetes_common_audit_policy_rules|length > 0 %} +{{ kubernetes_common_audit_policy_rules | indent(2, true) }} +{% else %} + # Default rules obtained from https://github.com/kubernetes/kubernetes/blob/v1.13.5/cluster/gce/gci/configure-helper.sh#L752 + # The following requests were manually identified as high-volume and low-risk, + # so drop them. + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core + resources: ["endpoints", "services", "services/status"] + - level: None + # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. + # TODO(#46983): Change this to the ingress controller service account. + users: ["system:unsecured"] + namespaces: ["kube-system"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["configmaps"] + - level: None + users: ["kubelet"] # legacy kubelet identity + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + userGroups: ["system:nodes"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:serviceaccount:kube-system:endpoint-controller + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["endpoints"] + - level: None + users: ["system:apiserver"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + - level: None + users: ["cluster-autoscaler"] + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["configmaps", "endpoints"] + # Don't log HPA fetching metrics. + - level: None + users: + - system:kube-controller-manager + verbs: ["get", "list"] + resources: + - group: "metrics.k8s.io" + # Don't log these read-only URLs. + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + # Don't log events requests. + - level: None + resources: + - group: "" # core + resources: ["events"] + # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes + - level: Request + users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + - level: Request + userGroups: ["system:nodes"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + # deletecollection calls can be large, don't log responses for expected namespace deletions + - level: Request + users: ["system:serviceaccount:kube-system:namespace-controller"] + verbs: ["deletecollection"] + omitStages: + - "RequestReceived" + # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, + # so only log at the Metadata level. + - level: Metadata + resources: + - group: "" # core + resources: ["secrets", "configmaps"] + - group: authentication.k8s.io + resources: ["tokenreviews"] + omitStages: + - "RequestReceived" + # Get repsonses can be large; skip them. + - level: Request + verbs: ["get", "list", "watch"] + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for known APIs + - level: RequestResponse + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for all other requests. + - level: Metadata + omitStages: + - "RequestReceived" +{% endif %} \ No newline at end of file