Skip to content

Commit

Permalink
Refactor mesh-gateway ACL flow (#1085)
Browse files Browse the repository at this point in the history
* Refactor mesh-gateway ACL flow
  • Loading branch information
Ashwin Venkatesh authored and jmurret committed Mar 22, 2022
1 parent 7bdbbb3 commit b3840e7
Show file tree
Hide file tree
Showing 14 changed files with 355 additions and 144 deletions.
14 changes: 14 additions & 0 deletions acceptance/tests/vault/vault_wan_fed_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,12 +200,26 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
primaryConsulCluster := consul.NewHelmCluster(t, primaryConsulHelmValues, primaryCtx, cfg, consulReleaseName)
primaryConsulCluster.Create(t)

var k8sAuthMethodHost string
// When running on kind, the kube API address in kubeconfig will have a localhost address
// which will not work from inside the container. That's why we need to use the endpoints address instead
// which will point the node IP.
if cfg.UseKind {
// The Kubernetes AuthMethod host is read from the endpoints for the Kubernetes service.
kubernetesEndpoint, err := secondaryCtx.KubernetesClient(t).CoreV1().Endpoints("default").Get(context.Background(), "kubernetes", metav1.GetOptions{})
require.NoError(t, err)
k8sAuthMethodHost = fmt.Sprintf("%s:%d", kubernetesEndpoint.Subsets[0].Addresses[0].IP, kubernetesEndpoint.Subsets[0].Ports[0].Port)
} else {
k8sAuthMethodHost = k8s.KubernetesAPIServerHostFromOptions(t, secondaryCtx.KubectlOptions(t))
}

// Get the address of the mesh gateway.
primaryMeshGWAddress := meshGatewayAddress(t, cfg, primaryCtx, consulReleaseName)
secondaryConsulHelmValues := map[string]string{
"global.datacenter": "dc2",

"global.federation.enabled": "true",
"global.federation.k8sAuthMethodHost": k8sAuthMethodHost,
"global.federation.primaryDatacenter": "dc1",
"global.federation.primaryGateways[0]": primaryMeshGWAddress,

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ spec:
{{- if .Values.global.adminPartitions.enabled }}
-partition={{ .Values.global.adminPartitions.name }} \
{{- end }}
-log-level={{ default .Values.global.logLevel .Values.controller.logLevel }} \
-log-level={{ default .Values.global.logLevel .Values.apiGateway.logLevel }} \
-log-json={{ .Values.global.logJSON }}
resources:
requests:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ spec:
{{- if .Values.global.adminPartitions.enabled }}
-partition={{ .Values.global.adminPartitions.name }} \
{{- end }}
-log-level={{ default .Values.global.logLevel .Values.controller.logLevel }} \
-log-level={{ default .Values.global.logLevel }} \
-log-json={{ .Values.global.logJSON }}
resources:
requests:
Expand Down
2 changes: 1 addition & 1 deletion charts/consul/templates/connect-inject-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,7 @@ spec:
{{- if .Values.global.adminPartitions.enabled }}
-partition={{ .Values.global.adminPartitions.name }} \
{{- end }}
-log-level={{ default .Values.global.logLevel .Values.controller.logLevel }} \
-log-level={{ default .Values.global.logLevel .Values.connectInject.logLevel }} \
-log-json={{ .Values.global.logJSON }}
resources:
requests:
Expand Down
9 changes: 0 additions & 9 deletions charts/consul/templates/mesh-gateway-clusterrole.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,6 @@ rules:
verbs:
- use
{{- end }}
{{- if .Values.global.acls.manageSystemACLs }}
- apiGroups: [""]
resources:
- secrets
resourceNames:
- {{ template "consul.fullname" . }}-mesh-gateway-acl-token
verbs:
- get
{{- end }}
{{- if eq .Values.meshGateway.wanAddress.source "Service" }}
- apiGroups: [""]
resources:
Expand Down
62 changes: 36 additions & 26 deletions charts/consul/templates/mesh-gateway-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -121,36 +121,46 @@ spec:
{{- if (and .Values.global.tls.enabled .Values.global.tls.enableAutoEncrypt) }}
{{- include "consul.getAutoEncryptClientCA" . | nindent 8 }}
{{- end }}
# service-init registers the mesh gateway service.
- name: service-init
- name: mesh-gateway-init
image: {{ .Values.global.imageK8S }}
env:
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
{{- if .Values.global.tls.enabled }}
- name: CONSUL_CACERT
value: /consul/tls/ca/tls.crt
{{- end }}
- name: CONSUL_HTTP_ADDR
{{- if .Values.global.tls.enabled }}
- name: CONSUL_HTTP_ADDR
value: https://$(HOST_IP):8501
- name: CONSUL_CACERT
value: /consul/tls/ca/tls.crt
value: https://$(HOST_IP):8501
{{- else }}
- name: CONSUL_HTTP_ADDR
value: http://$(HOST_IP):8500
value: http://$(HOST_IP):8500
{{- end }}
command:
- "/bin/sh"
- "-ec"
- |
{{- if .Values.global.acls.manageSystemACLs }}
consul-k8s-control-plane acl-init \
-secret-name="{{ template "consul.fullname" . }}-mesh-gateway-acl-token" \
-k8s-namespace={{ .Release.Namespace }} \
-token-sink-file=/consul/service/acl-token
-component-name=mesh-gateway \
-token-sink-file=/consul/service/acl-token \
{{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }}
-acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }} \
-primary-datacenter={{ .Values.global.federation.primaryDatacenter }} \
{{- else }}
-acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method \
{{- end }}
{{- if .Values.global.adminPartitions.enabled }}
-partition={{ .Values.global.adminPartitions.name }} \
{{- end }}
-log-level={{ default .Values.global.logLevel }} \
-log-json={{ .Values.global.logJSON }}
{{ end }}
{{- $source := .Values.meshGateway.wanAddress.source }}
Expand Down Expand Up @@ -258,6 +268,9 @@ spec:
{{- end }}
{{- end }}
volumeMounts:
- mountPath: /consul/service
name: consul-service
readOnly: true
- name: consul-bin
mountPath: /consul-bin
{{- if .Values.global.tls.enabled }}
Expand Down Expand Up @@ -285,12 +298,9 @@ spec:
fieldPath: spec.nodeName
{{- end }}
{{- if .Values.global.acls.manageSystemACLs }}
- name: CONSUL_HTTP_TOKEN
valueFrom:
secretKeyRef:
name: "{{ template "consul.fullname" . }}-mesh-gateway-acl-token"
key: "token"
{{- end}}
- name: CONSUL_HTTP_TOKEN_FILE
value: /consul/service/acl-token
{{- end }}
{{- if .Values.global.tls.enabled }}
- name: CONSUL_HTTP_ADDR
value: https://$(HOST_IP):8501
Expand Down Expand Up @@ -337,7 +347,7 @@ spec:
lifecycle:
preStop:
exec:
command: ["/bin/sh", "-ec", "/consul-bin/consul services deregister -id=\"{{ .Values.meshGateway.consulServiceName }}\""]
command: ["/bin/sh", "-ec", "/consul-bin/consul services deregister -id=\"{{ .Values.meshGateway.consulServiceName }}\"", "&&", "/bin/sh", "-ec", "/consul-bin/consul logout"]

# consul-sidecar ensures the mesh gateway is always registered with
# the local Consul agent, even if it loses the initial registration.
Expand Down
2 changes: 1 addition & 1 deletion charts/consul/templates/server-acl-init-job.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ spec:
{{- end }}
{{- if .Values.meshGateway.enabled }}
-create-mesh-gateway-token=true \
-mesh-gateway=true \
{{- end }}
{{- if .Values.ingressGateways.enabled }}
Expand Down
2 changes: 1 addition & 1 deletion charts/consul/templates/sync-catalog-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ spec:
{{- if .Values.global.adminPartitions.enabled }}
-partition={{ .Values.global.adminPartitions.name }} \
{{- end }}
-log-level={{ default .Values.global.logLevel .Values.controller.logLevel }} \
-log-level={{ default .Values.global.logLevel .Values.syncCatalog.logLevel }} \
-log-json={{ .Values.global.logJSON }}
resources:
requests:
Expand Down
14 changes: 1 addition & 13 deletions charts/consul/test/unit/mesh-gateway-clusterrole.bats
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,6 @@ load _helpers
[ "${actual}" = "podsecuritypolicies" ]
}

@test "meshGateway/ClusterRole: rules for global.acls.manageSystemACLs=true" {
cd `chart_dir`
local actual=$(helm template \
-s templates/mesh-gateway-clusterrole.yaml \
--set 'meshGateway.enabled=true' \
--set 'connectInject.enabled=true' \
--set 'global.acls.manageSystemACLs=true' \
. | tee /dev/stderr |
yq -r '.rules[0].resources[0]' | tee /dev/stderr)
[ "${actual}" = "secrets" ]
}

@test "meshGateway/ClusterRole: rules for meshGateway.wanAddress.source=Service" {
cd `chart_dir`
local actual=$(helm template \
Expand Down Expand Up @@ -83,5 +71,5 @@ load _helpers
--set 'meshGateway.wanAddress.source=Service' \
. | tee /dev/stderr |
yq -r '.rules | length' | tee /dev/stderr)
[ "${actual}" = "3" ]
[ "${actual}" = "2" ]
}
Loading

0 comments on commit b3840e7

Please sign in to comment.