Skip to content
This repository has been archived by the owner on Mar 2, 2022. It is now read-only.

Add audit logging support with a default logging policy #163

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions ansible/roles/kubernetes-common/defaults/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,22 @@ kubernetes_common_kubeadm_config:
controlPlaneEndpoint: "{{ kubernetes_common_api_fqdn }}"
apiServerExtraArgs:
"endpoint-reconciler-type": "lease"
"audit-policy-file": "/etc/kubernetes/apiserver-audit-policy.yaml"
"audit-log-path": "/var/log/kubernetes/kube-apiserver-audit.log"
"audit-log-maxage": "30" # days
"audit-log-maxsize": "100" # megabytes
"audit-log-maxbackup": "1"
apiServerExtraVolumes:
- name: audit-policy
hostPath: /etc/kubernetes/apiserver-audit-policy.yaml
mountPath: /etc/kubernetes/apiserver-audit-policy.yaml
writable: false
pathType: File
- name: audit-logs
hostPath: /var/log/kubernetes
mountPath: /var/log/kubernetes
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this directory always exist? Should be sure that we create it/ensure it exists.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Kubelet will create it if it does not exist (Due to DirectoryOrCreate pathType). Do you think it is better to create the dir in advance?

Copy link

@erictcgs erictcgs May 1, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason not to have log destination set to - (stdout) so that standard docker log rotation and any log aggregation would be used? If staying with /var/log/kubernetes, what log rotation is in place, or what's available to prevent filling the partition?

Update - just looked at the full file, hadn't realized there were already rotation parameters in place, that addresses that concern. Only other question would be connecting to centralized logging - if sent to stdout then fluentd would add all pod labels for filtering (e.g. in elastic). If current destination is kept, I'm not quite sure how ansible hash merge strategy works for removing an element from a list, ie if user wants log destination to be '-', volumes list should be only the audit config, not the /var/log/kubernetes.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Logs are sent to /var/log/kubernetes/ based on the default fluentd-kubernetes-daemonset configuration:

https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v1.3/debian-elasticsearch/conf/kubernetes.conf#L171

writable: true
pathType: DirectoryOrCreate
apiServerCertSANs: "{{ kubernetes_common_api_ip | kube_lookup_hostname(kubernetes_common_api_fqdn, True) }}"
etcd:
external:
Expand Down
2 changes: 2 additions & 0 deletions ansible/roles/kubernetes-master/defaults/main.yml
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
---
# This variable can be used to provide a custom list of audit policy rules.
kubernetes_common_audit_policy_rules: []
5 changes: 5 additions & 0 deletions ansible/roles/kubernetes-master/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@
immediate: True
when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux'

- name: drop api-server audit policy template
template:
dest: /etc/kubernetes/apiserver-audit-policy.yaml
src: etc/kubernetes/apiserver-audit-policy.yaml

- include_tasks: install.yml
when: wardroom_action == 'install'

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
{% if kubernetes_common_audit_policy_rules|length > 0 %}
{{ kubernetes_common_audit_policy_rules | indent(2, true) }}
{% else %}
# Default rules obtained from https://github.com/kubernetes/kubernetes/blob/v1.13.5/cluster/gce/gci/configure-helper.sh#L752
# The following requests were manually identified as high-volume and low-risk,
# so drop them.
- level: None
users: ["system:kube-proxy"]
verbs: ["watch"]
resources:
- group: "" # core
resources: ["endpoints", "services", "services/status"]
- level: None
# Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
# TODO(#46983): Change this to the ingress controller service account.
users: ["system:unsecured"]
namespaces: ["kube-system"]
verbs: ["get"]
resources:
- group: "" # core
resources: ["configmaps"]
- level: None
users: ["kubelet"] # legacy kubelet identity
verbs: ["get"]
resources:
- group: "" # core
resources: ["nodes", "nodes/status"]
- level: None
userGroups: ["system:nodes"]
verbs: ["get"]
resources:
- group: "" # core
resources: ["nodes", "nodes/status"]
- level: None
users:
- system:kube-controller-manager
- system:kube-scheduler
- system:serviceaccount:kube-system:endpoint-controller
verbs: ["get", "update"]
namespaces: ["kube-system"]
resources:
- group: "" # core
resources: ["endpoints"]
- level: None
users: ["system:apiserver"]
verbs: ["get"]
resources:
- group: "" # core
resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
- level: None
users: ["cluster-autoscaler"]
verbs: ["get", "update"]
namespaces: ["kube-system"]
resources:
- group: "" # core
resources: ["configmaps", "endpoints"]
# Don't log HPA fetching metrics.
- level: None
users:
- system:kube-controller-manager
verbs: ["get", "list"]
resources:
- group: "metrics.k8s.io"
# Don't log these read-only URLs.
- level: None
nonResourceURLs:
- /healthz*
- /version
- /swagger*
# Don't log events requests.
- level: None
resources:
- group: "" # core
resources: ["events"]
# node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
- level: Request
users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
verbs: ["update","patch"]
resources:
- group: "" # core
resources: ["nodes/status", "pods/status"]
omitStages:
- "RequestReceived"
- level: Request
userGroups: ["system:nodes"]
verbs: ["update","patch"]
resources:
- group: "" # core
resources: ["nodes/status", "pods/status"]
omitStages:
- "RequestReceived"
# deletecollection calls can be large, don't log responses for expected namespace deletions
- level: Request
users: ["system:serviceaccount:kube-system:namespace-controller"]
verbs: ["deletecollection"]
omitStages:
- "RequestReceived"
# Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
# so only log at the Metadata level.
- level: Metadata
resources:
- group: "" # core
resources: ["secrets", "configmaps"]
- group: authentication.k8s.io
resources: ["tokenreviews"]
omitStages:
- "RequestReceived"
# Get repsonses can be large; skip them.
- level: Request
verbs: ["get", "list", "watch"]
resources:
- group: "" # core
- group: "admissionregistration.k8s.io"
- group: "apiextensions.k8s.io"
- group: "apiregistration.k8s.io"
- group: "apps"
- group: "authentication.k8s.io"
- group: "authorization.k8s.io"
- group: "autoscaling"
- group: "batch"
- group: "certificates.k8s.io"
- group: "extensions"
- group: "metrics.k8s.io"
- group: "networking.k8s.io"
- group: "policy"
- group: "rbac.authorization.k8s.io"
- group: "scheduling.k8s.io"
- group: "settings.k8s.io"
- group: "storage.k8s.io"
omitStages:
- "RequestReceived"
# Default level for known APIs
- level: RequestResponse
resources:
- group: "" # core
- group: "admissionregistration.k8s.io"
- group: "apiextensions.k8s.io"
- group: "apiregistration.k8s.io"
- group: "apps"
- group: "authentication.k8s.io"
- group: "authorization.k8s.io"
- group: "autoscaling"
- group: "batch"
- group: "certificates.k8s.io"
- group: "extensions"
- group: "metrics.k8s.io"
- group: "networking.k8s.io"
- group: "policy"
- group: "rbac.authorization.k8s.io"
- group: "scheduling.k8s.io"
- group: "settings.k8s.io"
- group: "storage.k8s.io"
omitStages:
- "RequestReceived"
# Default level for all other requests.
- level: Metadata
omitStages:
- "RequestReceived"
{% endif %}