From 60460cd5afc2f34e539d93e68d4fefeb0d932853 Mon Sep 17 00:00:00 2001 From: neil85 Date: Tue, 31 Oct 2023 12:52:58 +0800 Subject: [PATCH 1/7] update configuration for plugins and tls --- .../templates/apisix-configmap.yaml | 108 ++++-------------- .../templates/deployment.yaml | 2 +- charts/apisix-ingress-controller/values.yaml | 21 ++++ 3 files changed, 46 insertions(+), 85 deletions(-) diff --git a/charts/apisix-ingress-controller/templates/apisix-configmap.yaml b/charts/apisix-ingress-controller/templates/apisix-configmap.yaml index 73105247..8ab2acec 100644 --- a/charts/apisix-ingress-controller/templates/apisix-configmap.yaml +++ b/charts/apisix-ingress-controller/templates/apisix-configmap.yaml @@ -58,91 +58,31 @@ data: worker_rlimit_nofile: {{ .Values.gateway.nginx.workerRlimitNofile }} # the number of files a worker process can open, should be larger than worker_connections event: worker_connections: {{ .Values.gateway.nginx.workerConnections }} + + ssl: + enable: {{ .Values.gateway.tls.enabled }} + listen: + - port: {{ .Values.gateway.tls.containerPort }} + enable_http2: {{ .Values.gateway.tls.http2.enabled }} + {{- with .Values.gateway.tls.additionalContainerPorts }} + {{- toYaml . | nindent 10}} + {{- end }} + ssl_protocols: {{ .Values.gateway.tls.sslProtocols | quote }} + ssl_ciphers: "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA" + {{- if and .Values.gateway.tls.enabled .Values.gateway.tls.existingCASecret }} + ssl_trusted_certificate: "/usr/local/apisix/conf/ssl/{{ .Values.gateway.tls.certCAFilename }}" + {{- end }} + {{- if and .Values.gateway.tls.enabled .Values.gateway.tls.fallbackSNI }} + fallback_sni: {{ .Values.gateway.tls.fallbackSNI | quote }} + {{- end }} - plugins: # plugin list (sorted by priority) - - real-ip # priority: 23000 - - ai # priority: 22900 - - client-control # priority: 22000 - - proxy-control # priority: 21990 - - request-id # priority: 12015 - - zipkin # priority: 12011 - #- skywalking # priority: 12010 - #- opentelemetry # priority: 12009 - - ext-plugin-pre-req # priority: 12000 - - fault-injection # priority: 11000 - - mocking # priority: 10900 - - serverless-pre-function # priority: 10000 - #- batch-requests # priority: 4010 - - cors # priority: 4000 - - ip-restriction # priority: 3000 - - ua-restriction # priority: 2999 - - referer-restriction # priority: 2990 - - csrf # priority: 2980 - - uri-blocker # priority: 2900 - - request-validation # priority: 2800 - - openid-connect # priority: 2599 - - cas-auth # priority: 2597 - - authz-casbin # priority: 2560 - - authz-casdoor # priority: 2559 - - wolf-rbac # priority: 2555 - - ldap-auth # priority: 2540 - - hmac-auth # priority: 2530 - - basic-auth # priority: 2520 - - jwt-auth # priority: 2510 - - key-auth # priority: 2500 - - consumer-restriction # priority: 2400 - - forward-auth # priority: 2002 - - opa # priority: 2001 - - authz-keycloak # priority: 2000 - #- error-log-logger # priority: 1091 - - proxy-mirror # priority: 1010 - - proxy-cache # priority: 1009 - - proxy-rewrite # priority: 1008 - - workflow # priority: 1006 - - api-breaker # priority: 1005 - - limit-conn # priority: 1003 - - limit-count # priority: 1002 - - limit-req # priority: 1001 - #- node-status # priority: 1000 - - gzip # priority: 995 - - traffic-split # priority: 966 - - redirect # priority: 900 - - response-rewrite # priority: 899 - - kafka-proxy # priority: 508 - #- dubbo-proxy # priority: 507 - - grpc-transcode # priority: 506 - - grpc-web # priority: 505 - - public-api # priority: 501 - - prometheus # priority: 500 - - datadog # priority: 495 - - elasticsearch-logger # priority: 413 - - echo # priority: 412 - - loggly # priority: 411 - - http-logger # priority: 410 - - splunk-hec-logging # priority: 409 - - skywalking-logger # priority: 408 - - google-cloud-logging # priority: 407 - - sls-logger # priority: 406 - - tcp-logger # priority: 405 - - kafka-logger # priority: 403 - - rocketmq-logger # priority: 402 - - syslog # priority: 401 - - udp-logger # priority: 400 - - file-logger # priority: 399 - - clickhouse-logger # priority: 398 - - tencent-cloud-cls # priority: 397 - - inspect # priority: 200 - #- log-rotate # priority: 100 - # <- recommend to use priority (0, 100) for your custom plugins - - example-plugin # priority: 0 - #- gm # priority: -43 - - aws-lambda # priority: -1899 - - azure-functions # priority: -1900 - - openwhisk # priority: -1901 - - openfunction # priority: -1902 - - serverless-post-function # priority: -2000 - - ext-plugin-post-req # priority: -3000 - - ext-plugin-post-resp # priority: -4000 + {{- if .Values.plugins }} + plugins: # plugin list + {{- range $plugin := .Values.plugins }} + {{- if ne $plugin "" }} + - {{ $plugin }} + {{- end }} + {{- end }} plugin_attr: prometheus: diff --git a/charts/apisix-ingress-controller/templates/deployment.yaml b/charts/apisix-ingress-controller/templates/deployment.yaml index efa83616..ccddd9e7 100644 --- a/charts/apisix-ingress-controller/templates/deployment.yaml +++ b/charts/apisix-ingress-controller/templates/deployment.yaml @@ -140,7 +140,7 @@ spec: containerPort: 9180 protocol: TCP - name: https - containerPort: 9443 + containerPort: {{ .Values.gateway.tls.containerPort }} protocol: TCP {{- if .Values.serviceMonitor.enabled }} - containerPort: 9091 diff --git a/charts/apisix-ingress-controller/values.yaml b/charts/apisix-ingress-controller/values.yaml index 6b8d265b..a6f6b2f6 100644 --- a/charts/apisix-ingress-controller/values.yaml +++ b/charts/apisix-ingress-controller/values.yaml @@ -232,3 +232,24 @@ gateway: errorLog: stderr # -- Nginx error logs level errorLogLevel: warn + tls: + enabled: false + servicePort: 443 + containerPort: 9443 + # -- Support multiple https ports, See [Configuration](https://github.com/apache/apisix/blob/0bc65ea9acd726f79f80ae0abd8f50b7eb172e3d/conf/config-default.yaml#L99) + additionalContainerPorts: [] + # - ip: 127.0.0.3 # Specific IP, If not set, the default value is `0.0.0.0`. + # port: 9445 + # enable_http2: true + # -- Specifies the name of Secret contains trusted CA certificates in the PEM format used to verify the certificate when APISIX needs to do SSL/TLS handshaking with external services (e.g. etcd) + existingCASecret: "" + # -- Filename be used in the gateway.tls.existingCASecret + certCAFilename: "" + http2: + enabled: true + # -- TLS protocols allowed to use. + sslProtocols: "TLSv1.2 TLSv1.3" + # -- Define SNI to fallback if none is presented by client + fallbackSNI: "" +# -- Customize the list of APISIX plugins to enable. By default, APISIX's default plugins are automatically used. See [config-default.yaml](https://github.com/apache/apisix/blob/master/conf/config-default.yaml) +plugins: [] From 67f3775e58f9053f8cc0518177a30b2a283cc2a7 Mon Sep 17 00:00:00 2001 From: neil85 Date: Tue, 31 Oct 2023 12:56:31 +0800 Subject: [PATCH 2/7] updating readme --- charts/apisix-ingress-controller/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/charts/apisix-ingress-controller/README.md b/charts/apisix-ingress-controller/README.md index a2af3bee..bebf7d8d 100644 --- a/charts/apisix-ingress-controller/README.md +++ b/charts/apisix-ingress-controller/README.md @@ -151,6 +151,15 @@ The same for container level, you need to set: | gateway.nginx.workerConnections | string | `"10620"` | Nginx worker connections | | gateway.nginx.workerProcesses | string | `"auto"` | Nginx worker processes | | gateway.nginx.workerRlimitNofile | string | `"20480"` | Nginx workerRlimitNoFile | +| gateway.tls.additionalContainerPorts | list | `[]` | Support multiple https ports, See [Configuration](https://github.com/apache/apisix/blob/0bc65ea9acd726f79f80ae0abd8f50b7eb172e3d/conf/config-default.yaml#L99) | +| gateway.tls.certCAFilename | string | `""` | Filename be used in the gateway.tls.existingCASecret | +| gateway.tls.containerPort | int | `9443` | | +| gateway.tls.enabled | bool | `false` | | +| gateway.tls.existingCASecret | string | `""` | Specifies the name of Secret contains trusted CA certificates in the PEM format used to verify the certificate when APISIX needs to do SSL/TLS handshaking with external services (e.g. etcd) | +| gateway.tls.fallbackSNI | string | `""` | Define SNI to fallback if none is presented by client | +| gateway.tls.http2.enabled | bool | `true` | | +| gateway.tls.servicePort | int | `443` | | +| gateway.tls.sslProtocols | string | `"TLSv1.2 TLSv1.3"` | TLS protocols allowed to use. | | gateway.type | string | `"NodePort"` | Apache APISIX service type for user access itself | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"apache/apisix-ingress-controller"` | | @@ -161,6 +170,7 @@ The same for container level, you need to set: | labelsOverride | object | `{}` | Override default labels assigned to Apache APISIX ingress controller resource | | nameOverride | string | `""` | Default values for apisix-ingress-controller. This is a YAML-formatted file. Declare variables to be passed into your templates. | | nodeSelector | object | `{}` | | +| plugins | list | `[]` | Customize the list of APISIX plugins to enable. By default, APISIX's default plugins are automatically used. See [config-default.yaml](https://github.com/apache/apisix/blob/master/conf/config-default.yaml) | | podAnnotations | object | `{}` | | | podSecurityContext | object | `{}` | | | priorityClassName | string | `""` | | From 01448372f832f90734eb152b7ef0cce0f9c88c87 Mon Sep 17 00:00:00 2001 From: neil85 Date: Tue, 31 Oct 2023 12:57:36 +0800 Subject: [PATCH 3/7] update service ports --- .../apisix-ingress-controller/templates/service-apisix.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/apisix-ingress-controller/templates/service-apisix.yaml b/charts/apisix-ingress-controller/templates/service-apisix.yaml index 5125974d..4abf0f3a 100644 --- a/charts/apisix-ingress-controller/templates/service-apisix.yaml +++ b/charts/apisix-ingress-controller/templates/service-apisix.yaml @@ -17,8 +17,8 @@ spec: targetPort: 9080 - name: https protocol: TCP - port: 443 - targetPort: 9443 + port: {{ .Values.gateway.tls.servicePort }} + targetPort: {{ .Values.gateway.tls.containerPort }} selector: {{- include "apisix-ingress-controller.selectorLabels" . | nindent 4 }} type: {{ .Values.gateway.type }} # LoadBalancer or NodePort From e8f6fec6d489ee3be25fc46fe1c5ebb1bafbb6d2 Mon Sep 17 00:00:00 2001 From: neil85 Date: Tue, 31 Oct 2023 13:16:42 +0800 Subject: [PATCH 4/7] fix chart lint --- charts/apisix-ingress-controller/templates/apisix-configmap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/apisix-ingress-controller/templates/apisix-configmap.yaml b/charts/apisix-ingress-controller/templates/apisix-configmap.yaml index 8ab2acec..db872c5b 100644 --- a/charts/apisix-ingress-controller/templates/apisix-configmap.yaml +++ b/charts/apisix-ingress-controller/templates/apisix-configmap.yaml @@ -83,6 +83,7 @@ data: - {{ $plugin }} {{- end }} {{- end }} + {{- end }} plugin_attr: prometheus: From f28aef193fbbe4228c08400510056a110437e3b3 Mon Sep 17 00:00:00 2001 From: neil85 Date: Tue, 31 Oct 2023 14:24:29 +0800 Subject: [PATCH 5/7] revert plugin changes --- .../templates/apisix-configmap.yaml | 92 +++++++++++++++++-- charts/apisix-ingress-controller/values.yaml | 2 - 2 files changed, 84 insertions(+), 10 deletions(-) diff --git a/charts/apisix-ingress-controller/templates/apisix-configmap.yaml b/charts/apisix-ingress-controller/templates/apisix-configmap.yaml index db872c5b..6cdd5a27 100644 --- a/charts/apisix-ingress-controller/templates/apisix-configmap.yaml +++ b/charts/apisix-ingress-controller/templates/apisix-configmap.yaml @@ -76,14 +76,90 @@ data: fallback_sni: {{ .Values.gateway.tls.fallbackSNI | quote }} {{- end }} - {{- if .Values.plugins }} - plugins: # plugin list - {{- range $plugin := .Values.plugins }} - {{- if ne $plugin "" }} - - {{ $plugin }} - {{- end }} - {{- end }} - {{- end }} + plugins: # plugin list (sorted by priority) + - real-ip # priority: 23000 + - ai # priority: 22900 + - client-control # priority: 22000 + - proxy-control # priority: 21990 + - request-id # priority: 12015 + - zipkin # priority: 12011 + #- skywalking # priority: 12010 + #- opentelemetry # priority: 12009 + - ext-plugin-pre-req # priority: 12000 + - fault-injection # priority: 11000 + - mocking # priority: 10900 + - serverless-pre-function # priority: 10000 + #- batch-requests # priority: 4010 + - cors # priority: 4000 + - ip-restriction # priority: 3000 + - ua-restriction # priority: 2999 + - referer-restriction # priority: 2990 + - csrf # priority: 2980 + - uri-blocker # priority: 2900 + - request-validation # priority: 2800 + - openid-connect # priority: 2599 + - cas-auth # priority: 2597 + - authz-casbin # priority: 2560 + - authz-casdoor # priority: 2559 + - wolf-rbac # priority: 2555 + - ldap-auth # priority: 2540 + - hmac-auth # priority: 2530 + - basic-auth # priority: 2520 + - jwt-auth # priority: 2510 + - key-auth # priority: 2500 + - consumer-restriction # priority: 2400 + - forward-auth # priority: 2002 + - opa # priority: 2001 + - authz-keycloak # priority: 2000 + #- error-log-logger # priority: 1091 + - proxy-mirror # priority: 1010 + - proxy-cache # priority: 1009 + - proxy-rewrite # priority: 1008 + - workflow # priority: 1006 + - api-breaker # priority: 1005 + - limit-conn # priority: 1003 + - limit-count # priority: 1002 + - limit-req # priority: 1001 + #- node-status # priority: 1000 + - gzip # priority: 995 + - traffic-split # priority: 966 + - redirect # priority: 900 + - response-rewrite # priority: 899 + - kafka-proxy # priority: 508 + #- dubbo-proxy # priority: 507 + - grpc-transcode # priority: 506 + - grpc-web # priority: 505 + - public-api # priority: 501 + - prometheus # priority: 500 + - datadog # priority: 495 + - elasticsearch-logger # priority: 413 + - echo # priority: 412 + - loggly # priority: 411 + - http-logger # priority: 410 + - splunk-hec-logging # priority: 409 + - skywalking-logger # priority: 408 + - google-cloud-logging # priority: 407 + - sls-logger # priority: 406 + - tcp-logger # priority: 405 + - kafka-logger # priority: 403 + - rocketmq-logger # priority: 402 + - syslog # priority: 401 + - udp-logger # priority: 400 + - file-logger # priority: 399 + - clickhouse-logger # priority: 398 + - tencent-cloud-cls # priority: 397 + - inspect # priority: 200 + #- log-rotate # priority: 100 + # <- recommend to use priority (0, 100) for your custom plugins + - example-plugin # priority: 0 + #- gm # priority: -43 + - aws-lambda # priority: -1899 + - azure-functions # priority: -1900 + - openwhisk # priority: -1901 + - openfunction # priority: -1902 + - serverless-post-function # priority: -2000 + - ext-plugin-post-req # priority: -3000 + - ext-plugin-post-resp # priority: -4000 plugin_attr: prometheus: diff --git a/charts/apisix-ingress-controller/values.yaml b/charts/apisix-ingress-controller/values.yaml index a6f6b2f6..d395ad56 100644 --- a/charts/apisix-ingress-controller/values.yaml +++ b/charts/apisix-ingress-controller/values.yaml @@ -251,5 +251,3 @@ gateway: sslProtocols: "TLSv1.2 TLSv1.3" # -- Define SNI to fallback if none is presented by client fallbackSNI: "" -# -- Customize the list of APISIX plugins to enable. By default, APISIX's default plugins are automatically used. See [config-default.yaml](https://github.com/apache/apisix/blob/master/conf/config-default.yaml) -plugins: [] From 3684cf7a9773e69842a81abd404078121e0deb6a Mon Sep 17 00:00:00 2001 From: neil85 Date: Tue, 31 Oct 2023 15:59:46 +0800 Subject: [PATCH 6/7] adding ingress class --- .../apisix-ingress-controller/templates/ingress-class.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 charts/apisix-ingress-controller/templates/ingress-class.yaml diff --git a/charts/apisix-ingress-controller/templates/ingress-class.yaml b/charts/apisix-ingress-controller/templates/ingress-class.yaml new file mode 100644 index 00000000..01206140 --- /dev/null +++ b/charts/apisix-ingress-controller/templates/ingress-class.yaml @@ -0,0 +1,6 @@ +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: {{ .Values.config.kubernetes.ingressClass | quote }} +spec: + controller: apisix.apache.org/apisix-ingress # fix: https://github.com/apache/apisix-ingress-controller/issues/1653 From 6a886b780a13451edcadd27188d0f7aaeb52f623 Mon Sep 17 00:00:00 2001 From: neil85 Date: Wed, 1 Nov 2023 17:35:23 +0800 Subject: [PATCH 7/7] update docs --- charts/apisix-ingress-controller/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/apisix-ingress-controller/README.md b/charts/apisix-ingress-controller/README.md index bebf7d8d..4f2bcf4a 100644 --- a/charts/apisix-ingress-controller/README.md +++ b/charts/apisix-ingress-controller/README.md @@ -170,7 +170,6 @@ The same for container level, you need to set: | labelsOverride | object | `{}` | Override default labels assigned to Apache APISIX ingress controller resource | | nameOverride | string | `""` | Default values for apisix-ingress-controller. This is a YAML-formatted file. Declare variables to be passed into your templates. | | nodeSelector | object | `{}` | | -| plugins | list | `[]` | Customize the list of APISIX plugins to enable. By default, APISIX's default plugins are automatically used. See [config-default.yaml](https://github.com/apache/apisix/blob/master/conf/config-default.yaml) | | podAnnotations | object | `{}` | | | podSecurityContext | object | `{}` | | | priorityClassName | string | `""` | |