From 6d8b1f813811f80097cf31fcbf9e92cf1defe8b7 Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Fri, 8 Nov 2019 10:44:42 -0500 Subject: [PATCH 01/16] Add Kourier as an Ingress option for e2e tests This copies the work done for Gloo, except for Kourier. The download-kourier.sh script just pulls in Kourier 0.2.2 (the latest) for now like Gloo does since we do not test with multiple versions of either. --- test/e2e-common.sh | 37 ++- third_party/kourier-latest/README.md | 5 + .../kourier-latest/download-kourier.sh | 20 ++ third_party/kourier-latest/kourier.yaml | 258 ++++++++++++++++++ 4 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 third_party/kourier-latest/README.md create mode 100755 third_party/kourier-latest/download-kourier.sh create mode 100644 third_party/kourier-latest/kourier.yaml diff --git a/test/e2e-common.sh b/test/e2e-common.sh index f267a652c943..b871b736176d 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -28,6 +28,7 @@ source $(dirname $0)/../vendor/knative.dev/test-infra/scripts/e2e-tests.sh CERT_MANAGER_VERSION="0.9.1" ISTIO_VERSION="" GLOO_VERSION="" +KOURIER_VERSION="" HTTPS=0 @@ -96,6 +97,13 @@ function parse_flags() { GATEWAY_SETUP=1 return 2 ;; + --kourier-version) + # currently, the value of --kourier-version is ignored + # latest version of Kourier pinned in third_party will be installed + readonly KOURIER_VERSION=$2 + GATEWAY_SETUP=1 + return 2 + ;; esac return 0 } @@ -174,6 +182,15 @@ function install_gloo() { kubectl apply -f ${INSTALL_GLOO_YAML} || return 1 } +function install_kourier() { + local kourier_base="./third_party/kourier-latest" + INSTALL_KOURIER_YAML="${kourier_base}/kourier.yaml" + echo "Kourier YAML: ${INSTALL_KOURIER_YAML}" + echo ">> Bringing up Kourier" + + kubectl apply -f ${INSTALL_KOURIER_YAML} || return 1 +} + # Installs Knative Serving in the current cluster, and waits for it to be ready. # If no parameters are passed, installs the current source-based build. # Parameters: $1 - Knative Serving YAML file @@ -186,8 +203,8 @@ function install_knative_serving_standard() { build_knative_from_source INSTALL_RELEASE_YAML="${SERVING_YAML}" - # install serving core if installing for Gloo - if [[ -n "${GLOO_VERSION}" ]]; then + # install serving core if installing for Gloo or Kourier + if [[ -n "${GLOO_VERSION}" || -n "${KOURIER_VERSION}" ]]; then INSTALL_RELEASE_YAML="${SERVING_CORE_YAML}" fi @@ -214,6 +231,9 @@ function install_knative_serving_standard() { if [[ -n "${GLOO_VERSION}" ]]; then install_gloo fi + if [[ -n "${KOURIER_VERSION}" ]]; then + install_kourier + fi echo ">> Installing Cert-Manager" kubectl apply -f "${INSTALL_CERT_MANAGER_YAML}" --validate=false || return 1 @@ -276,10 +296,13 @@ EOF # Some versions of Istio don't provide an HPA for pilot. kubectl autoscale -n istio-system deploy istio-pilot --min=3 --max=10 --cpu-percent=60 || return 1 fi - else + elif [[ -n "${GLOO_VERSION}" ]]; then # Scale replicas of the Gloo proxies to handle large qps kubectl scale -n gloo-system deployment knative-external-proxy --replicas=6 kubectl scale -n gloo-system deployment knative-internal-proxy --replicas=6 + elif [[ -n "${KOURIER_VERSION}" ]]; then + # Scale replicas of the Kourier gateways to handle large qps + kubectl scale -n kourier-system deployment 3scale-kourier-gateway --replicas=6 fi if [[ -n "${INSTALL_MONITORING_YAML}" ]]; then @@ -374,6 +397,14 @@ function test_setup() { wait_until_pods_running gloo-system || return 1 wait_until_service_has_external_ip gloo-system knative-external-proxy fi + if [[ -n "${KOURIER_VERSION}" ]]; then + # we must set these override values to allow the test spoofing client to work with Kourier + # see https://github.com/knative/pkg/blob/release-0.7/test/ingress/ingress.go#L37 + export GATEWAY_OVERRIDE=kourier-external + export GATEWAY_NAMESPACE_OVERRIDE=kourier-system + wait_until_pods_running kourier-system || return 1 + wait_until_service_has_external_ip kourier-system kourier-external + fi if [[ -n "${INSTALL_MONITORING_YAML}" ]]; then wait_until_pods_running knative-monitoring || return 1 fi diff --git a/third_party/kourier-latest/README.md b/third_party/kourier-latest/README.md new file mode 100644 index 000000000000..c4fcf438c5ba --- /dev/null +++ b/third_party/kourier-latest/README.md @@ -0,0 +1,5 @@ +The kourier.yaml file is generated by running + +``` +./download-kourier.sh +``` diff --git a/third_party/kourier-latest/download-kourier.sh b/third_party/kourier-latest/download-kourier.sh new file mode 100755 index 000000000000..c28d5077c3c7 --- /dev/null +++ b/third_party/kourier-latest/download-kourier.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -ex + +# Download Kourier +KOURIER_VERSION=0.2.2 +KOURIER_YAML=kourier-knative.yaml +DOWNLOAD_URL=https://raw.githubusercontent.com/3scale/kourier/v${KOURIER_VERSION}/deploy/${KOURIER_YAML} + +wget ${DOWNLOAD_URL} + +cat ${KOURIER_YAML} \ + `# Install Kourier into the kourier-system namespace` \ + | sed 's/namespace: knative-serving/namespace: kourier-system/' \ + `# Expose Kourier services with LoadBalancer IPs instead of ClusterIP` \ + | sed 's/ClusterIP/LoadBalancer/' \ + > kourier.yaml + +# Clean up. +rm ${KOURIER_YAML} diff --git a/third_party/kourier-latest/kourier.yaml b/third_party/kourier-latest/kourier.yaml new file mode 100644 index 000000000000..32dc05d22bf2 --- /dev/null +++ b/third_party/kourier-latest/kourier.yaml @@ -0,0 +1,258 @@ +apiVersion: v1 +kind: Service +metadata: + name: kourier + namespace: kourier-system +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: 3scale-kourier + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: 3scale-kourier-gateway + namespace: kourier-system +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: 3scale-kourier-gateway + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: 3scale-kourier-gateway + spec: + containers: + - args: + - -c + - /tmp/config/envoy-bootstrap.yaml + image: quay.io/3scale/kourier-gateway:v0.1.0 + imagePullPolicy: Always + name: kourier-gateway + ports: + - containerPort: 8080 + protocol: TCP + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - name: config-volume + mountPath: /tmp/config + volumes: + - name: config-volume + configMap: + name: kourier-bootstrap + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: 3scale-kourier-control + namespace: kourier-system +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: 3scale-kourier-control + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: 3scale-kourier-control + spec: + containers: + - image: quay.io/3scale/kourier:v0.2.2 + imagePullPolicy: Always + name: kourier-control + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + env: + - name: CERTS_SECRET_NAMESPACE + value: "" + - name: CERTS_SECRET_NAME + value: "" + - name: KOURIER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccountName: 3scale-kourier + terminationGracePeriodSeconds: 30 +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: 3scale-kourier + namespace: kourier-system +rules: + - apiGroups: [""] + resources: [ "endpoints", "namespaces", "services", "secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.internal.knative.dev"] + resources: ["clusteringresses","ingresses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.internal.knative.dev"] + resources: ["ingresses/status","clusteringresses/status"] + verbs: ["update"] + - apiGroups: [ "apiextensions.k8s.io" ] + resources: [ "customresourcedefinitions" ] + verbs: ["get", "list", "watch"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: 3scale-kourier + namespace: kourier-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: 3scale-kourier +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 3scale-kourier +subjects: + - kind: ServiceAccount + name: 3scale-kourier + namespace: kourier-system +--- +apiVersion: v1 +kind: Service +metadata: + name: kourier + namespace: kourier-system +spec: + ports: + - name: http2 + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: 3scale-kourier-gateway + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: kourier-internal + namespace: kourier-system +spec: + ports: + - name: http2 + port: 80 + protocol: TCP + targetPort: 8081 + selector: + app: 3scale-kourier-gateway + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: kourier-external + namespace: kourier-system +spec: + ports: + - name: http2 + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: 3scale-kourier-gateway + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: kourier-control + namespace: kourier-system +spec: + ports: + - port: 18000 + protocol: TCP + targetPort: 18000 + selector: + app: 3scale-kourier-control + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kourier-bootstrap + namespace: kourier-system +data: + envoy-bootstrap.yaml: | + admin: + access_log_path: /tmp/test + address: + socket_address: + address: 0.0.0.0 + port_value: 19000 + dynamic_resources: + ads_config: + api_type: GRPC + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + cds_config: + ads: {} + lds_config: + ads: {} + node: + cluster: kourier-knative + id: 3scale-kourier-gateway + static_resources: + clusters: + - connect_timeout: 0.2s + load_assignment: + cluster_name: xds_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: kourier-control + port_value: 18000 + http2_protocol_options: {} + upstream_connection_options: + tcp_keepalive: {} + lb_policy: ROUND_ROBIN + name: xds_cluster + type: STRICT_DNS From d6056f2f3bf36bcb279b13966c03847b4d97812c Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Fri, 8 Nov 2019 10:45:27 -0500 Subject: [PATCH 02/16] HACK: Run all tests w/ Kourier instead of Istio This commit needs to be removed from this PR before merging. --- test/e2e-common.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index b871b736176d..57a74225309b 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -49,7 +49,11 @@ function parse_flags() { case "$1" in --istio-version) [[ $2 =~ ^[0-9]+\.[0-9]+(\.[0-9]+|\-latest)$ ]] || abort "version format must be '[0-9].[0-9].[0-9]' or '[0-9].[0-9]-latest" - readonly ISTIO_VERSION=$2 + # TOTAL HACK BELOW - DO NOT COMMIT + # FORCE KOURIER TO RUN INSTEAD OF ISTIO + # NEVER MERGE THIS PR WITH THIS IN + #readonly ISTIO_VERSION=$2 + readonly KOURIER_VERSION="0.2.2" GATEWAY_SETUP=1 return 2 ;; From 760149c2a59f15c2069e19e71907166c6c5c7067 Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Fri, 8 Nov 2019 11:40:59 -0500 Subject: [PATCH 03/16] We need the kourier-system namespace as well I missed then when converting things to use the download-kourier.sh for generating the kourier.yaml --- third_party/kourier-latest/download-kourier.sh | 10 +++++++++- third_party/kourier-latest/kourier.yaml | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/third_party/kourier-latest/download-kourier.sh b/third_party/kourier-latest/download-kourier.sh index c28d5077c3c7..9d0d10e2d764 100755 --- a/third_party/kourier-latest/download-kourier.sh +++ b/third_party/kourier-latest/download-kourier.sh @@ -9,12 +9,20 @@ DOWNLOAD_URL=https://raw.githubusercontent.com/3scale/kourier/v${KOURIER_VERSION wget ${DOWNLOAD_URL} +cat < kourier.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: kourier-system +--- +EOF + cat ${KOURIER_YAML} \ `# Install Kourier into the kourier-system namespace` \ | sed 's/namespace: knative-serving/namespace: kourier-system/' \ `# Expose Kourier services with LoadBalancer IPs instead of ClusterIP` \ | sed 's/ClusterIP/LoadBalancer/' \ - > kourier.yaml + >> kourier.yaml # Clean up. rm ${KOURIER_YAML} diff --git a/third_party/kourier-latest/kourier.yaml b/third_party/kourier-latest/kourier.yaml index 32dc05d22bf2..e9204541c80b 100644 --- a/third_party/kourier-latest/kourier.yaml +++ b/third_party/kourier-latest/kourier.yaml @@ -1,4 +1,9 @@ apiVersion: v1 +kind: Namespace +metadata: + name: kourier-system +--- +apiVersion: v1 kind: Service metadata: name: kourier From bd6c58e733dbcde23a03f9f598ca108ed8b2d200 Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Fri, 8 Nov 2019 12:28:12 -0500 Subject: [PATCH 04/16] Actually use Kourier ingress when it's enabled in tests --- test/e2e-common.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 57a74225309b..8a0245316b54 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -237,6 +237,20 @@ function install_knative_serving_standard() { fi if [[ -n "${KOURIER_VERSION}" ]]; then install_kourier + + echo ">> Making Kourier the default ingress" + cat <> Installing Cert-Manager" From 68c67e7d21c3206241062eab8722e2027c27f42f Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Fri, 8 Nov 2019 12:39:33 -0500 Subject: [PATCH 05/16] Helps if it's not a dry-run --- test/e2e-common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 8a0245316b54..ebc6ab58035e 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -239,7 +239,7 @@ function install_knative_serving_standard() { install_kourier echo ">> Making Kourier the default ingress" - cat < Date: Fri, 8 Nov 2019 12:51:11 -0500 Subject: [PATCH 06/16] Make Kourier the default ingress AFTER serving is installed I guess I need to figure out how to run this entire e2e script locally... --- test/e2e-common.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index ebc6ab58035e..63db106459d1 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -237,7 +237,15 @@ function install_knative_serving_standard() { fi if [[ -n "${KOURIER_VERSION}" ]]; then install_kourier + fi + + echo ">> Installing Cert-Manager" + kubectl apply -f "${INSTALL_CERT_MANAGER_YAML}" --validate=false || return 1 + + echo ">> Bringing up Serving" + kubectl apply -f "${INSTALL_RELEASE_YAML}" || return 1 + if [[ -n "${KOURIER_VERSION}" ]]; then echo ">> Making Kourier the default ingress" cat <> Installing Cert-Manager" - kubectl apply -f "${INSTALL_CERT_MANAGER_YAML}" --validate=false || return 1 - - echo ">> Bringing up Serving" - kubectl apply -f "${INSTALL_RELEASE_YAML}" || return 1 - if (( RECONCILE_GATEWAY )); then echo ">> Turning on reconcileExternalGateway" cat < Date: Mon, 11 Nov 2019 09:45:59 -0500 Subject: [PATCH 07/16] Bump to Kouvier 0.2.3 that does not reconcile ClusterIngress --- third_party/kourier-latest/download-kourier.sh | 2 +- third_party/kourier-latest/kourier.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/kourier-latest/download-kourier.sh b/third_party/kourier-latest/download-kourier.sh index 9d0d10e2d764..73da7b51389a 100755 --- a/third_party/kourier-latest/download-kourier.sh +++ b/third_party/kourier-latest/download-kourier.sh @@ -3,7 +3,7 @@ set -ex # Download Kourier -KOURIER_VERSION=0.2.2 +KOURIER_VERSION=0.2.3 KOURIER_YAML=kourier-knative.yaml DOWNLOAD_URL=https://raw.githubusercontent.com/3scale/kourier/v${KOURIER_VERSION}/deploy/${KOURIER_YAML} diff --git a/third_party/kourier-latest/kourier.yaml b/third_party/kourier-latest/kourier.yaml index e9204541c80b..dde7d180a985 100644 --- a/third_party/kourier-latest/kourier.yaml +++ b/third_party/kourier-latest/kourier.yaml @@ -90,7 +90,7 @@ spec: app: 3scale-kourier-control spec: containers: - - image: quay.io/3scale/kourier:v0.2.2 + - image: quay.io/3scale/kourier:v0.2.3 imagePullPolicy: Always name: kourier-control resources: {} From 2f287645a1881982c0ecd7fc210cf9e460fbd32d Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Mon, 11 Nov 2019 12:12:13 -0500 Subject: [PATCH 08/16] Try Kourier w/ 1 dataplane pod --- test/e2e-common.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 63db106459d1..8faeef7ebd3e 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -322,7 +322,10 @@ EOF kubectl scale -n gloo-system deployment knative-internal-proxy --replicas=6 elif [[ -n "${KOURIER_VERSION}" ]]; then # Scale replicas of the Kourier gateways to handle large qps - kubectl scale -n kourier-system deployment 3scale-kourier-gateway --replicas=6 + # + # TODO: Bump this from 1 back to 6 once Kourier can ensure all + # gateways are ready + kubectl scale -n kourier-system deployment 3scale-kourier-gateway --replicas=1 fi if [[ -n "${INSTALL_MONITORING_YAML}" ]]; then From e2e02aacb07232b70237244460ef535f810e9f9a Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Wed, 13 Nov 2019 10:37:26 -0500 Subject: [PATCH 09/16] Temporarily enable route retrying logic to isolate test failures --- test/v1/route.go | 7 +++++++ test/v1alpha1/route.go | 6 ++++++ test/v1beta1/route.go | 7 +++++++ 3 files changed, 20 insertions(+) diff --git a/test/v1/route.go b/test/v1/route.go index a5dc11149b0c..b83d3e021161 100644 --- a/test/v1/route.go +++ b/test/v1/route.go @@ -19,6 +19,7 @@ package v1 import ( "context" "fmt" + "net/http" "testing" "github.com/davecgh/go-spew/spew" @@ -116,8 +117,14 @@ func IsRouteNotReady(r *v1.Route) (bool, error) { } // RetryingRouteInconsistency retries common requests seen when creating a new route +// - 404 until the route is propagated to the proxy +// - 503 to account for Openshift route inconsistency (https://jira.coreos.com/browse/SRVKS-157) +// TODO(5573): Remove this. func RetryingRouteInconsistency(innerCheck spoof.ResponseChecker) spoof.ResponseChecker { return func(resp *spoof.Response) (bool, error) { + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusServiceUnavailable { + return false, nil + } // If we didn't match any retryable codes, invoke the ResponseChecker that we wrapped. return innerCheck(resp) } diff --git a/test/v1alpha1/route.go b/test/v1alpha1/route.go index 6a152d968f54..c53f1db61ced 100644 --- a/test/v1alpha1/route.go +++ b/test/v1alpha1/route.go @@ -21,6 +21,7 @@ package v1alpha1 import ( "context" "fmt" + "net/http" "testing" "github.com/davecgh/go-spew/spew" @@ -51,9 +52,14 @@ func CreateRoute(t *testing.T, clients *test.Clients, names test.ResourceNames, } // RetryingRouteInconsistency retries common requests seen when creating a new route +// - 404 until the route is propagated to the proxy +// - 503 to account for Openshift route inconsistency (https://jira.coreos.com/browse/SRVKS-157) // TODO(5573): Remove this. func RetryingRouteInconsistency(innerCheck spoof.ResponseChecker) spoof.ResponseChecker { return func(resp *spoof.Response) (bool, error) { + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusServiceUnavailable { + return false, nil + } // If we didn't match any retryable codes, invoke the ResponseChecker that we wrapped. return innerCheck(resp) } diff --git a/test/v1beta1/route.go b/test/v1beta1/route.go index 8f25b7020052..c1d18ad0c474 100644 --- a/test/v1beta1/route.go +++ b/test/v1beta1/route.go @@ -19,6 +19,7 @@ package v1beta1 import ( "context" "fmt" + "net/http" "testing" "github.com/davecgh/go-spew/spew" @@ -118,8 +119,14 @@ func IsRouteNotReady(r *v1beta1.Route) (bool, error) { } // RetryingRouteInconsistency retries common requests seen when creating a new route +// - 404 until the route is propagated to the proxy +// - 503 to account for Openshift route inconsistency (https://jira.coreos.com/browse/SRVKS-157) +// TODO(5573): Remove this. func RetryingRouteInconsistency(innerCheck spoof.ResponseChecker) spoof.ResponseChecker { return func(resp *spoof.Response) (bool, error) { + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusServiceUnavailable { + return false, nil + } // If we didn't match any retryable codes, invoke the ResponseChecker that we wrapped. return innerCheck(resp) } From ce4237257afee88e8cdf1ad02e3f5b6059df43f5 Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Wed, 13 Nov 2019 12:25:14 -0500 Subject: [PATCH 10/16] Bump to Kourier v0.2.4 and multiple gateway pods Kourier 0.2.4 should now handle Ready status from multiple gateway pods so that Ready isn't indicated until all gateways are ready. So, bump to that and increase gateway replica count. --- test/e2e-common.sh | 5 +---- third_party/kourier-latest/download-kourier.sh | 2 +- third_party/kourier-latest/kourier.yaml | 6 +++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 8faeef7ebd3e..63db106459d1 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -322,10 +322,7 @@ EOF kubectl scale -n gloo-system deployment knative-internal-proxy --replicas=6 elif [[ -n "${KOURIER_VERSION}" ]]; then # Scale replicas of the Kourier gateways to handle large qps - # - # TODO: Bump this from 1 back to 6 once Kourier can ensure all - # gateways are ready - kubectl scale -n kourier-system deployment 3scale-kourier-gateway --replicas=1 + kubectl scale -n kourier-system deployment 3scale-kourier-gateway --replicas=6 fi if [[ -n "${INSTALL_MONITORING_YAML}" ]]; then diff --git a/third_party/kourier-latest/download-kourier.sh b/third_party/kourier-latest/download-kourier.sh index 73da7b51389a..c49741ad7386 100755 --- a/third_party/kourier-latest/download-kourier.sh +++ b/third_party/kourier-latest/download-kourier.sh @@ -3,7 +3,7 @@ set -ex # Download Kourier -KOURIER_VERSION=0.2.3 +KOURIER_VERSION=0.2.4 KOURIER_YAML=kourier-knative.yaml DOWNLOAD_URL=https://raw.githubusercontent.com/3scale/kourier/v${KOURIER_VERSION}/deploy/${KOURIER_YAML} diff --git a/third_party/kourier-latest/kourier.yaml b/third_party/kourier-latest/kourier.yaml index dde7d180a985..7d3c87df90d9 100644 --- a/third_party/kourier-latest/kourier.yaml +++ b/third_party/kourier-latest/kourier.yaml @@ -90,7 +90,7 @@ spec: app: 3scale-kourier-control spec: containers: - - image: quay.io/3scale/kourier:v0.2.3 + - image: quay.io/3scale/kourier:v0.2.4 imagePullPolicy: Always name: kourier-control resources: {} @@ -119,14 +119,14 @@ metadata: namespace: kourier-system rules: - apiGroups: [""] - resources: [ "endpoints", "namespaces", "services", "secrets"] + resources: ["pods", "endpoints", "namespaces", "services", "secrets"] verbs: ["get", "list", "watch"] - apiGroups: ["networking.internal.knative.dev"] resources: ["clusteringresses","ingresses"] verbs: ["get", "list", "watch"] - apiGroups: ["networking.internal.knative.dev"] resources: ["ingresses/status","clusteringresses/status"] - verbs: ["update"] + verbs: ["update"] - apiGroups: [ "apiextensions.k8s.io" ] resources: [ "customresourcedefinitions" ] verbs: ["get", "list", "watch"] From deb070e743fe9ee9ad6e270770029e4e4f2c6814 Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Wed, 13 Nov 2019 14:37:04 -0500 Subject: [PATCH 11/16] Kourier v0.2.5 --- third_party/kourier-latest/download-kourier.sh | 2 +- third_party/kourier-latest/kourier.yaml | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/third_party/kourier-latest/download-kourier.sh b/third_party/kourier-latest/download-kourier.sh index c49741ad7386..76389ff9080d 100755 --- a/third_party/kourier-latest/download-kourier.sh +++ b/third_party/kourier-latest/download-kourier.sh @@ -3,7 +3,7 @@ set -ex # Download Kourier -KOURIER_VERSION=0.2.4 +KOURIER_VERSION=0.2.5 KOURIER_YAML=kourier-knative.yaml DOWNLOAD_URL=https://raw.githubusercontent.com/3scale/kourier/v${KOURIER_VERSION}/deploy/${KOURIER_YAML} diff --git a/third_party/kourier-latest/kourier.yaml b/third_party/kourier-latest/kourier.yaml index 7d3c87df90d9..c0fa1836d194 100644 --- a/third_party/kourier-latest/kourier.yaml +++ b/third_party/kourier-latest/kourier.yaml @@ -45,7 +45,7 @@ spec: - args: - -c - /tmp/config/envoy-bootstrap.yaml - image: quay.io/3scale/kourier-gateway:v0.1.0 + image: quay.io/3scale/kourier-gateway:v0.1.1 imagePullPolicy: Always name: kourier-gateway ports: @@ -57,6 +57,11 @@ spec: volumeMounts: - name: config-volume mountPath: /tmp/config + readinessProbe: + exec: + command: ['ash','-c','curl -H "Host: internalkourier" http://localhost:8081/__internalkouriersnapshot'] + initialDelaySeconds: 5 + periodSeconds: 2 volumes: - name: config-volume configMap: @@ -90,7 +95,7 @@ spec: app: 3scale-kourier-control spec: containers: - - image: quay.io/3scale/kourier:v0.2.4 + - image: quay.io/3scale/kourier:v0.2.5 imagePullPolicy: Always name: kourier-control resources: {} From b36327c0b6f94c2f93794f0d4e1fd03ab68428bb Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Thu, 14 Nov 2019 09:46:24 -0500 Subject: [PATCH 12/16] Be smarter about serving-core.yaml vs serving.yaml in upgrade tests When testing on Gloo or Kourier use serving-core.yaml instead of serving.yaml for upgrade tests. --- test/e2e-upgrade-tests.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/e2e-upgrade-tests.sh b/test/e2e-upgrade-tests.sh index e12304a3df71..333ba3de377e 100755 --- a/test/e2e-upgrade-tests.sh +++ b/test/e2e-upgrade-tests.sh @@ -38,9 +38,15 @@ LATEST_SERVING_RELEASE_VERSION=$(git describe --match "v[0-9]*" --abbrev=0) function install_latest_release() { header "Installing Knative latest public release" local url="https://github.com/knative/serving/releases/download/${LATEST_SERVING_RELEASE_VERSION}" - # TODO: should this test install istio and build at all, or only serving? + local yaml="serving.yaml" + + # install serving core if installing for Gloo or Kourier + if [[ -n "${GLOO_VERSION}" || -n "${KOURIER_VERSION}" ]]; then + yaml="serving-core.yaml" + fi + install_knative_serving \ - "${url}/serving.yaml" \ + "${url}/${yaml}" \ || fail_test "Knative latest release installation failed" wait_until_pods_running knative-serving } From a8561425930762960bff10b073f4c5eb4a46f6f4 Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Thu, 14 Nov 2019 09:47:19 -0500 Subject: [PATCH 13/16] Revert "Temporarily enable route retrying logic to isolate test failures" This reverts commit e2e02aacb07232b70237244460ef535f810e9f9a. --- test/v1/route.go | 7 ------- test/v1alpha1/route.go | 6 ------ test/v1beta1/route.go | 7 ------- 3 files changed, 20 deletions(-) diff --git a/test/v1/route.go b/test/v1/route.go index b83d3e021161..a5dc11149b0c 100644 --- a/test/v1/route.go +++ b/test/v1/route.go @@ -19,7 +19,6 @@ package v1 import ( "context" "fmt" - "net/http" "testing" "github.com/davecgh/go-spew/spew" @@ -117,14 +116,8 @@ func IsRouteNotReady(r *v1.Route) (bool, error) { } // RetryingRouteInconsistency retries common requests seen when creating a new route -// - 404 until the route is propagated to the proxy -// - 503 to account for Openshift route inconsistency (https://jira.coreos.com/browse/SRVKS-157) -// TODO(5573): Remove this. func RetryingRouteInconsistency(innerCheck spoof.ResponseChecker) spoof.ResponseChecker { return func(resp *spoof.Response) (bool, error) { - if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusServiceUnavailable { - return false, nil - } // If we didn't match any retryable codes, invoke the ResponseChecker that we wrapped. return innerCheck(resp) } diff --git a/test/v1alpha1/route.go b/test/v1alpha1/route.go index c53f1db61ced..6a152d968f54 100644 --- a/test/v1alpha1/route.go +++ b/test/v1alpha1/route.go @@ -21,7 +21,6 @@ package v1alpha1 import ( "context" "fmt" - "net/http" "testing" "github.com/davecgh/go-spew/spew" @@ -52,14 +51,9 @@ func CreateRoute(t *testing.T, clients *test.Clients, names test.ResourceNames, } // RetryingRouteInconsistency retries common requests seen when creating a new route -// - 404 until the route is propagated to the proxy -// - 503 to account for Openshift route inconsistency (https://jira.coreos.com/browse/SRVKS-157) // TODO(5573): Remove this. func RetryingRouteInconsistency(innerCheck spoof.ResponseChecker) spoof.ResponseChecker { return func(resp *spoof.Response) (bool, error) { - if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusServiceUnavailable { - return false, nil - } // If we didn't match any retryable codes, invoke the ResponseChecker that we wrapped. return innerCheck(resp) } diff --git a/test/v1beta1/route.go b/test/v1beta1/route.go index c1d18ad0c474..8f25b7020052 100644 --- a/test/v1beta1/route.go +++ b/test/v1beta1/route.go @@ -19,7 +19,6 @@ package v1beta1 import ( "context" "fmt" - "net/http" "testing" "github.com/davecgh/go-spew/spew" @@ -119,14 +118,8 @@ func IsRouteNotReady(r *v1beta1.Route) (bool, error) { } // RetryingRouteInconsistency retries common requests seen when creating a new route -// - 404 until the route is propagated to the proxy -// - 503 to account for Openshift route inconsistency (https://jira.coreos.com/browse/SRVKS-157) -// TODO(5573): Remove this. func RetryingRouteInconsistency(innerCheck spoof.ResponseChecker) spoof.ResponseChecker { return func(resp *spoof.Response) (bool, error) { - if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusServiceUnavailable { - return false, nil - } // If we didn't match any retryable codes, invoke the ResponseChecker that we wrapped. return innerCheck(resp) } From 2986c9ce90ef68a84fe0b3fcb4a54f94982c0f8d Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Thu, 14 Nov 2019 09:59:30 -0500 Subject: [PATCH 14/16] Copyright header, trailing whitespace cleanup --- test/e2e-upgrade-tests.sh | 2 +- third_party/kourier-latest/download-kourier.sh | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/test/e2e-upgrade-tests.sh b/test/e2e-upgrade-tests.sh index 333ba3de377e..aa7ad5dfc050 100755 --- a/test/e2e-upgrade-tests.sh +++ b/test/e2e-upgrade-tests.sh @@ -39,7 +39,7 @@ function install_latest_release() { header "Installing Knative latest public release" local url="https://github.com/knative/serving/releases/download/${LATEST_SERVING_RELEASE_VERSION}" local yaml="serving.yaml" - + # install serving core if installing for Gloo or Kourier if [[ -n "${GLOO_VERSION}" || -n "${KOURIER_VERSION}" ]]; then yaml="serving-core.yaml" diff --git a/third_party/kourier-latest/download-kourier.sh b/third_party/kourier-latest/download-kourier.sh index 76389ff9080d..39f62b4f4e33 100755 --- a/third_party/kourier-latest/download-kourier.sh +++ b/third_party/kourier-latest/download-kourier.sh @@ -1,5 +1,19 @@ #!/usr/bin/env bash +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -ex # Download Kourier From 899e768cde8e694bc8a31d88de03f116cc76da72 Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Thu, 14 Nov 2019 10:35:39 -0500 Subject: [PATCH 15/16] Update to Kourier 0.2.6 --- test/e2e-common.sh | 2 +- third_party/kourier-latest/download-kourier.sh | 2 +- third_party/kourier-latest/kourier.yaml | 12 +----------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 63db106459d1..6b9357f67330 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -53,7 +53,7 @@ function parse_flags() { # FORCE KOURIER TO RUN INSTEAD OF ISTIO # NEVER MERGE THIS PR WITH THIS IN #readonly ISTIO_VERSION=$2 - readonly KOURIER_VERSION="0.2.2" + readonly KOURIER_VERSION="0.2.6" GATEWAY_SETUP=1 return 2 ;; diff --git a/third_party/kourier-latest/download-kourier.sh b/third_party/kourier-latest/download-kourier.sh index 39f62b4f4e33..3038ec834da7 100755 --- a/third_party/kourier-latest/download-kourier.sh +++ b/third_party/kourier-latest/download-kourier.sh @@ -17,7 +17,7 @@ set -ex # Download Kourier -KOURIER_VERSION=0.2.5 +KOURIER_VERSION=0.2.6 KOURIER_YAML=kourier-knative.yaml DOWNLOAD_URL=https://raw.githubusercontent.com/3scale/kourier/v${KOURIER_VERSION}/deploy/${KOURIER_YAML} diff --git a/third_party/kourier-latest/kourier.yaml b/third_party/kourier-latest/kourier.yaml index c0fa1836d194..1a5b1b876a0d 100644 --- a/third_party/kourier-latest/kourier.yaml +++ b/third_party/kourier-latest/kourier.yaml @@ -16,8 +16,6 @@ spec: selector: app: 3scale-kourier type: LoadBalancer -status: - loadBalancer: {} --- apiVersion: apps/v1 kind: Deployment @@ -95,7 +93,7 @@ spec: app: 3scale-kourier-control spec: containers: - - image: quay.io/3scale/kourier:v0.2.5 + - image: quay.io/3scale/kourier:v0.2.6 imagePullPolicy: Always name: kourier-control resources: {} @@ -169,8 +167,6 @@ spec: selector: app: 3scale-kourier-gateway type: LoadBalancer -status: - loadBalancer: {} --- apiVersion: v1 kind: Service @@ -186,8 +182,6 @@ spec: selector: app: 3scale-kourier-gateway type: LoadBalancer -status: - loadBalancer: {} --- apiVersion: v1 kind: Service @@ -203,8 +197,6 @@ spec: selector: app: 3scale-kourier-gateway type: LoadBalancer -status: - loadBalancer: {} --- apiVersion: v1 kind: Service @@ -219,8 +211,6 @@ spec: selector: app: 3scale-kourier-control type: LoadBalancer -status: - loadBalancer: {} --- apiVersion: v1 kind: ConfigMap From c7c173d16e2c3a80185c544378dae8a2f10806ca Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Fri, 15 Nov 2019 08:32:00 -0500 Subject: [PATCH 16/16] Revert "HACK: Run all tests w/ Kourier instead of Istio" This reverts commit d6056f2f3bf36bcb279b13966c03847b4d97812c. --- test/e2e-common.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 6b9357f67330..9c1ee41279c1 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -49,11 +49,7 @@ function parse_flags() { case "$1" in --istio-version) [[ $2 =~ ^[0-9]+\.[0-9]+(\.[0-9]+|\-latest)$ ]] || abort "version format must be '[0-9].[0-9].[0-9]' or '[0-9].[0-9]-latest" - # TOTAL HACK BELOW - DO NOT COMMIT - # FORCE KOURIER TO RUN INSTEAD OF ISTIO - # NEVER MERGE THIS PR WITH THIS IN - #readonly ISTIO_VERSION=$2 - readonly KOURIER_VERSION="0.2.6" + readonly ISTIO_VERSION=$2 GATEWAY_SETUP=1 return 2 ;;