From b3b2c4ef23495da02d727d647062e50a2f8c9dbf Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Mon, 25 Nov 2024 08:04:37 +0000 Subject: [PATCH 1/4] feature: VPC Egress Gateway Signed-off-by: zhangzujian --- .github/workflows/build-x86-image.yaml | 183 +++- Makefile.e2e | 9 + charts/kube-ovn/Chart.yaml | 2 +- .../kube-ovn/templates/controller-deploy.yaml | 1 + charts/kube-ovn/templates/kube-ovn-crd.yaml | 299 ++++++- charts/kube-ovn/templates/ovn-CR.yaml | 16 +- dist/images/bfdd-prestart.sh | 14 + dist/images/cleanup.sh | 1 + dist/images/init-vpc-egress-gateway.sh | 31 + dist/images/install.sh | 313 ++++++- dist/images/kubectl-ko | 1 + dist/images/start-bfdd.sh | 12 + mocks/pkg/ovs/interface.go | 238 ++++- pkg/apis/kubeovn/v1/condition.go | 2 + pkg/apis/kubeovn/v1/register.go | 2 + pkg/apis/kubeovn/v1/vpc-egress-gateway.go | 89 ++ pkg/apis/kubeovn/v1/vpc.go | 7 +- pkg/apis/kubeovn/v1/zz_generated.deepcopy.go | 258 ++++++ .../kubeovn/v1/fake/fake_kubeovn_client.go | 4 + .../kubeovn/v1/fake/fake_vpcegressgateway.go | 147 ++++ .../typed/kubeovn/v1/generated_expansion.go | 2 + .../typed/kubeovn/v1/kubeovn_client.go | 5 + .../typed/kubeovn/v1/vpcegressgateway.go | 69 ++ .../informers/externalversions/generic.go | 2 + .../externalversions/kubeovn/v1/interface.go | 7 + .../kubeovn/v1/vpcegressgateway.go | 90 ++ .../listers/kubeovn/v1/expansion_generated.go | 8 + .../listers/kubeovn/v1/vpcegressgateway.go | 70 ++ pkg/controller/config.go | 6 + pkg/controller/controller.go | 51 +- pkg/controller/deployment.go | 51 ++ pkg/controller/subnet.go | 2 +- pkg/controller/vpc.go | 19 +- pkg/controller/vpc_egress_gateway.go | 830 ++++++++++++++++++ pkg/ovn_ic_controller/ovn_ic_controller.go | 2 +- pkg/ovs/interface.go | 12 +- pkg/ovs/ovn-nb-bfd.go | 59 +- pkg/ovs/ovn-nb-bfd_test.go | 74 +- pkg/ovs/ovn-nb-logical_router_policy.go | 22 +- pkg/ovs/ovn-nb-logical_router_policy_test.go | 40 +- pkg/ovs/ovn-nb-logical_router_route.go | 61 ++ pkg/ovs/ovn.go | 3 + pkg/util/const.go | 4 + pkg/util/hash.go | 26 + pkg/util/k8s.go | 53 ++ pkg/util/pod_routes.go | 56 ++ pkg/util/strings.go | 9 - test/e2e/framework/framework.go | 11 + test/e2e/framework/pod.go | 36 + test/e2e/framework/provider-network.go | 7 + test/e2e/framework/vlan.go | 11 +- test/e2e/framework/vpc-egress-gateway.go | 189 ++++ test/e2e/kube-ovn/subnet/subnet.go | 39 +- test/e2e/kube-ovn/underlay/underlay.go | 5 +- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 4 +- test/e2e/vpc-egress-gateway/e2e_test.go | 520 +++++++++++ yamls/audit-policy.yaml | 2 + 57 files changed, 3909 insertions(+), 177 deletions(-) create mode 100644 dist/images/bfdd-prestart.sh create mode 100644 dist/images/init-vpc-egress-gateway.sh create mode 100644 dist/images/start-bfdd.sh create mode 100644 pkg/apis/kubeovn/v1/vpc-egress-gateway.go create mode 100644 pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcegressgateway.go create mode 100644 pkg/client/clientset/versioned/typed/kubeovn/v1/vpcegressgateway.go create mode 100644 pkg/client/informers/externalversions/kubeovn/v1/vpcegressgateway.go create mode 100644 pkg/client/listers/kubeovn/v1/vpcegressgateway.go create mode 100644 pkg/controller/deployment.go create mode 100644 pkg/controller/vpc_egress_gateway.go create mode 100644 pkg/util/hash.go create mode 100644 pkg/util/pod_routes.go create mode 100644 test/e2e/framework/vpc-egress-gateway.go create mode 100644 test/e2e/vpc-egress-gateway/e2e_test.go diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 038d5b41d7f..beca0fabe09 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -2002,7 +2002,7 @@ jobs: - name: Create kind cluster env: - k8s_version: v1.23.17 + k8s_version: v1.29.10 run: | pipx install jinjanator make kind-init @@ -2421,6 +2421,186 @@ jobs: - name: Cleanup run: timeout -k 10 180 sh -x dist/images/cleanup.sh + vpc-egress-gateway-e2e: + name: VPC Egress Gateway E2E + needs: + - build-kube-ovn + - build-e2e-binaries + runs-on: ubuntu-24.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + ip-family: + - ipv4 + - ipv6 + - dual + steps: + - uses: jlumbroso/free-disk-space@v1.3.1 + with: + android: true + dotnet: true + haskell: true + docker-images: false + large-packages: false + tool-cache: false + swap-storage: false + + - uses: actions/checkout@v4 + + - name: Create the default branch directory + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + run: mkdir -p test/e2e/source + + - name: Check out the default branch + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + path: test/e2e/source + + - name: Export E2E directory + run: | + if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then + echo "E2E_DIR=." >> "$GITHUB_ENV" + else + echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" + fi + + - uses: actions/setup-go@v5 + id: setup-go + with: + go-version-file: ${{ env.E2E_DIR }}/go.mod + check-latest: true + cache: false + + - name: Export Go full version + run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV" + + - name: Go cache + uses: actions/cache/restore@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} + restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86- + + - name: Install kind + uses: helm/kind-action@v1.10.0 + with: + version: ${{ env.KIND_VERSION }} + install_only: true + + - name: Install ginkgo + working-directory: ${{ env.E2E_DIR }} + run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo + + - name: Download image + uses: actions/download-artifact@v4 + with: + name: kube-ovn + + - name: Load image + run: docker load --input kube-ovn.tar + + - name: Create kind cluster + run: | + pipx install jinjanator + make kind-init-${{ matrix.ip-family }} + + - name: Install Kube-OVN + id: install + run: make kind-install-debug-valgrind-${{ matrix.ip-family }} + + - name: Install Multus + run: make kind-install-multus + + - name: Run E2E + id: e2e + working-directory: ${{ env.E2E_DIR }} + env: + E2E_BRANCH: ${{ github.base_ref || github.ref_name }} + E2E_IP_FAMILY: ${{ matrix.ip-family }} + run: make vpc-egress-gateway-e2e + + - name: Collect k8s events + if: failure() && steps.e2e.conclusion == 'failure' + run: | + kubectl get events -A -o yaml > kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.yaml + tar zcf kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.tar.gz kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.yaml + + - name: Upload k8s events + uses: actions/upload-artifact@v4 + if: failure() && steps.e2e.conclusion == 'failure' + with: + name: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events + path: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.tar.gz + + - name: Collect apiserver audit logs + if: failure() && steps.e2e.conclusion == 'failure' + run: | + docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log . + tar zcf kube-ovn-conformance-e2e-${{ matrix.ip-family }}-audit-log.tar.gz kube-apiserver-audit.log + + - name: Upload apiserver audit logs + uses: actions/upload-artifact@v4 + if: failure() && steps.e2e.conclusion == 'failure' + with: + name: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-audit-log + path: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-audit-log.tar.gz + + - name: kubectl ko log + if: failure() && steps.e2e.conclusion == 'failure' + run: | + make kubectl-ko-log + mv kubectl-ko-log.tar.gz kube-ovn-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz + + - name: upload kubectl ko log + uses: actions/upload-artifact@v4 + if: failure() && steps.e2e.conclusion == 'failure' + with: + name: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-ko-log + path: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz + + - name: Check kube ovn pod restarts + if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }} + run: make check-kube-ovn-pod-restarts + + - name: Check valgrind result + run: | + kubectl -n kube-system rollout restart ds ovs-ovn + kubectl -n kube-system rollout status ds ovs-ovn + sleep 10 + kubectl -n kube-system rollout restart deploy ovn-central + kubectl -n kube-system rollout status deploy ovn-central + while true; do + if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then + break + fi + sleep 1 + done + kubectl ko log ovn + kubectl ko log ovs + + for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do + echo "Checking if valgrind log file for $daemon exists..." + find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1 + done + + find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do + if grep -qw 'definitely lost' "$f"; then + echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')." + echo $f + cat "$f" + exit 1 + fi; + done + + - name: Cleanup + run: timeout -k 10 180 sh -x dist/images/cleanup.sh + iptables-vpc-nat-gw-conformance-e2e: name: Iptables VPC NAT Gateway E2E needs: @@ -2966,6 +3146,7 @@ jobs: - kube-ovn-ic-conformance-e2e - kube-ovn-ipsec-e2e - multus-conformance-e2e + - vpc-egress-gateway-e2e - ovn-vpc-nat-gw-conformance-e2e - iptables-vpc-nat-gw-conformance-e2e - webhook-e2e diff --git a/Makefile.e2e b/Makefile.e2e index 39b1220fe42..b94734cebaa 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -74,6 +74,7 @@ e2e-build: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/multus ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vpc-egress-gateway ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-vpc-nat-gw ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ha @@ -168,6 +169,14 @@ vip-conformance-e2e: ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/vip/vip.test -- $(TEST_BIN_ARGS) +.PHONY: vpc-egress-gateway-e2e +vpc-egress-gateway-e2e: + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vpc-egress-gateway + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v --timeout=30m \ + --focus=CNI:Kube-OVN ./test/e2e/vpc-egress-gateway/vpc-egress-gateway.test -- $(TEST_BIN_ARGS) + .PHONY: iptables-vpc-nat-gw-conformance-e2e iptables-vpc-nat-gw-conformance-e2e: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw diff --git a/charts/kube-ovn/Chart.yaml b/charts/kube-ovn/Chart.yaml index 99a7fec74f2..5f992651feb 100644 --- a/charts/kube-ovn/Chart.yaml +++ b/charts/kube-ovn/Chart.yaml @@ -23,4 +23,4 @@ version: v1.14.0 # It is recommended to use it with quotes. appVersion: "1.14.0" -kubeVersion: ">= 1.23.0-0" +kubeVersion: ">= 1.29.0-0" diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml index 06fdba72d95..b4364fa0adb 100644 --- a/charts/kube-ovn/templates/controller-deploy.yaml +++ b/charts/kube-ovn/templates/controller-deploy.yaml @@ -139,6 +139,7 @@ spec: - --enable-anp={{- .Values.func.ENABLE_ANP }} - --ovsdb-con-timeout={{- .Values.func.OVSDB_CON_TIMEOUT }} - --ovsdb-inactivity-timeout={{- .Values.func.OVSDB_INACTIVITY_TIMEOUT }} + - --image={{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} securityContext: runAsUser: {{ include "kubeovn.runAsUser" . }} privileged: false diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index d9dab30bbab..6f1bbefacb5 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -823,6 +823,302 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + name: vpc-egress-gateways.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vpc-egress-gateways + singular: vpc-egress-gateway + shortNames: + - vpc-egress-gw + - veg + kind: VpcEgressGateway + listKind: VpcEgressGatewayList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vpc + name: VPC + type: string + - jsonPath: .spec.replicas + name: REPLICAS + type: integer + - jsonPath: .spec.bfd.enabled + name: BFD ENABLED + type: boolean + - jsonPath: .spec.externalSubnet + name: EXTERNAL SUBNET + type: string + - jsonPath: .status.phase + name: PHASE + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .status.internalIPs + name: INTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.externalIPs + name: EXTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.workload.nodes + name: WORKING NODES + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + internalIPs: + items: + type: string + type: array + externalIPs: + items: + type: string + type: array + phase: + type: string + default: Pending + enum: + - Pending + - Processing + - Completed + ready: + type: boolean + default: false + workload: + type: object + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + nodes: + type: array + items: + type: string + required: + - conditions + - phase + type: object + spec: + type: object + required: + - externalSubnet + - policies + x-kubernetes-validations: + - rule: "!has(self.prefix) || self.prefix == '' || self.prefix == oldSelf.prefix" + message: 'Size of Internal IPs MUST be equal to or greater than Replicas' + fieldPath: ".prefix" + - rule: "!has(self.internalIPs) || size(self.internalIPs) == 0 || size(self.internalIPs) >= self.replicas" + message: 'Size of Internal IPs MUST be equal to or greater than Replicas' + fieldPath: ".internalIPs" + - rule: "!has(self.externalIPs) || size(self.externalIPs) == 0 || size(self.externalIPs) >= self.replicas" + message: 'Size of External IPs MUST be equal to or greater than Replicas' + fieldPath: ".externalIPs" + properties: + replicas: + type: integer + default: 1 + minimum: 1 + maximum: 10 + prefix: + type: string + anyOf: + - pattern: ^$ + - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*[-\.]?$ + x-kubernetes-validations: + - rule: "self == oldSelf" + message: "This field is immutable." + vpc: + type: string + internalSubnet: + type: string + externalSubnet: + type: string + internalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + externalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + image: + type: string + bfd: + type: object + properties: + enabled: + type: boolean + default: false + minRX: + type: integer + default: 1000 + minTX: + type: integer + default: 1000 + multiplier: + type: integer + default: 3 + policies: + type: array + minItems: 1 + items: + type: object + properties: + snat: + type: boolean + default: false + ipBlocks: + type: array + x-kubernetes-list-type: set + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + - format: cidr + subnets: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + x-kubernetes-validations: + - rule: "size(self.ipBlocks) != 0 || size(self.subnets) != 0" + message: 'Each policy MUST have at least one ipBlock or subnet' + nodeSelector: + type: array + items: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: name: iptables-eips.kubeovn.io spec: @@ -1769,9 +2065,6 @@ spec: bfdPort: type: object properties: - enabled: - type: boolean - default: false ip: type: string name: diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml index 856c9cd5b86..22fa5370854 100644 --- a/charts/kube-ovn/templates/ovn-CR.yaml +++ b/charts/kube-ovn/templates/ovn-CR.yaml @@ -13,6 +13,8 @@ rules: - vpcs/status - vpc-nat-gateways - vpc-nat-gateways/status + - vpc-egress-gateways + - vpc-egress-gateways/status - subnets - subnets/status - ippools @@ -98,6 +100,18 @@ rules: - daemonsets verbs: - get + - apiGroups: + - apps + resources: + - deployments + - deployments/scale + verbs: + - get + - list + - watch + - create + - update + - delete - apiGroups: - "" resources: @@ -124,8 +138,6 @@ rules: - apps resources: - statefulsets - - deployments - - deployments/scale verbs: - get - list diff --git a/dist/images/bfdd-prestart.sh b/dist/images/bfdd-prestart.sh new file mode 100644 index 00000000000..eae5e68da7c --- /dev/null +++ b/dist/images/bfdd-prestart.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +bfdd-control session new set mintx "${BFD_MIN_TX:-1000}" +bfdd-control session new set minrx "${BFD_MIN_RX:-1000}" +bfdd-control session new set multi "${BFD_MULTI:-3}" + +PEER_IPS=($(echo "${BFD_PEER_IPS:-::}" | tr ',' ' ')) +for ip in ${PEER_IPS[*]}; do + bfdd-control allow ${ip} +done + +bfdd-control log type command no diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh index 8c2d6c850e4..31e55a6e79f 100644 --- a/dist/images/cleanup.sh +++ b/dist/images/cleanup.sh @@ -125,6 +125,7 @@ kubectl delete --ignore-not-found crd \ security-groups.kubeovn.io \ ippools.kubeovn.io \ vpc-nat-gateways.kubeovn.io \ + vpc-egress-gateways.kubeovn.io \ vpcs.kubeovn.io \ vlans.kubeovn.io \ provider-networks.kubeovn.io \ diff --git a/dist/images/init-vpc-egress-gateway.sh b/dist/images/init-vpc-egress-gateway.sh new file mode 100644 index 00000000000..605d2421d0c --- /dev/null +++ b/dist/images/init-vpc-egress-gateway.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -ex + +INTERNAL_GATEWAY_IPV4=${INTERNAL_GATEWAY_IPV4:-} +INTERNAL_GATEWAY_IPV6=${INTERNAL_GATEWAY_IPV6:-} +INTERNAL_ROUTE_DST_IPV4=($(echo "${INTERNAL_ROUTE_DST_IPV4:-}" | tr ',' ' ')) +INTERNAL_ROUTE_DST_IPV6=($(echo "${INTERNAL_ROUTE_DST_IPV6:-}" | tr ',' ' ')) +SNAT_SOURCES_IPV4=($(echo "${SNAT_SOURCES_IPV4:-}" | tr ',' ' ')) +SNAT_SOURCES_IPV6=($(echo "${SNAT_SOURCES_IPV6:-}" | tr ',' ' ')) + +sysctl -w net.ipv4.ip_forward=1 +sysctl -w net.ipv6.conf.all.forwarding=1 + +iptables -V + +for dst in ${INTERNAL_ROUTE_DST_IPV4[*]}; do + ip route add "${dst}" via "${INTERNAL_GATEWAY_IPV4}" +done + +for dst in ${INTERNAL_ROUTE_DST_IPV6[*]}; do + ip route add "${dst}" via "${INTERNAL_GATEWAY_IPV6}" +done + +for src in ${SNAT_SOURCES_IPV4[*]}; do + iptables -t nat -A POSTROUTING -s "${src}" -j MASQUERADE --random-fully +done + +for src in ${SNAT_SOURCES_IPV6[*]}; do + ip6tables -t nat -A POSTROUTING -s "${src}" -j MASQUERADE --random-fully +done diff --git a/dist/images/install.sh b/dist/images/install.sh index 40a78ef0dbb..72cb8c3ad0d 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -1070,6 +1070,299 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + name: vpc-egress-gateways.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vpc-egress-gateways + singular: vpc-egress-gateway + shortNames: + - vpc-egress-gw + - veg + kind: VpcEgressGateway + listKind: VpcEgressGatewayList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vpc + name: VPC + type: string + - jsonPath: .spec.replicas + name: REPLICAS + type: integer + - jsonPath: .spec.bfd.enabled + name: BFD ENABLED + type: boolean + - jsonPath: .spec.externalSubnet + name: EXTERNAL SUBNET + type: string + - jsonPath: .status.phase + name: PHASE + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .status.internalIPs + name: INTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.externalIPs + name: EXTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.workload.nodes + name: WORKING NODES + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + internalIPs: + items: + type: string + type: array + externalIPs: + items: + type: string + type: array + phase: + type: string + default: Pending + enum: + - Pending + - Processing + - Completed + ready: + type: boolean + default: false + workload: + type: object + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + nodes: + type: array + items: + type: string + required: + - conditions + - phase + type: object + spec: + type: object + required: + - externalSubnet + - policies + x-kubernetes-validations: + - rule: "!has(self.internalIPs) || size(self.internalIPs) == 0 || size(self.internalIPs) >= self.replicas" + message: 'Size of Internal IPs MUST be equal to or greater than Replicas' + fieldPath: ".internalIPs" + - rule: "!has(self.externalIPs) || size(self.externalIPs) == 0 || size(self.externalIPs) >= self.replicas" + message: 'Size of External IPs MUST be equal to or greater than Replicas' + fieldPath: ".externalIPs" + properties: + replicas: + type: integer + default: 1 + minimum: 1 + maximum: 10 + prefix: + type: string + anyOf: + - pattern: ^$ + - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*[-\.]?$ + x-kubernetes-validations: + - rule: "self == oldSelf" + message: "This field is immutable." + vpc: + type: string + internalSubnet: + type: string + externalSubnet: + type: string + internalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + externalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + image: + type: string + bfd: + type: object + properties: + enabled: + type: boolean + default: false + minRX: + type: integer + default: 1000 + minTX: + type: integer + default: 1000 + multiplier: + type: integer + default: 3 + policies: + type: array + minItems: 1 + items: + type: object + properties: + snat: + type: boolean + default: false + ipBlocks: + type: array + x-kubernetes-list-type: set + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + - format: cidr + subnets: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + x-kubernetes-validations: + - rule: "size(self.ipBlocks) != 0 || size(self.subnets) != 0" + message: 'Each policy MUST have at least one ipBlock or subnet' + nodeSelector: + type: array + items: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: name: iptables-eips.kubeovn.io spec: @@ -2016,9 +2309,6 @@ spec: bfdPort: type: object properties: - enabled: - type: boolean - default: false ip: type: string name: @@ -3076,6 +3366,8 @@ rules: - vpcs/status - vpc-nat-gateways - vpc-nat-gateways/status + - vpc-egress-gateways + - vpc-egress-gateways/status - subnets - subnets/status - ippools @@ -3161,6 +3453,18 @@ rules: - daemonsets verbs: - get + - apiGroups: + - apps + resources: + - deployments + - deployments/scale + verbs: + - get + - list + - watch + - create + - update + - delete - apiGroups: - "" resources: @@ -3187,8 +3491,6 @@ rules: - apps resources: - statefulsets - - deployments - - deployments/scale verbs: - get - list @@ -4391,6 +4693,7 @@ spec: - --enable-anp=$ENABLE_ANP - --ovsdb-con-timeout=$OVSDB_CON_TIMEOUT - --ovsdb-inactivity-timeout=$OVSDB_INACTIVITY_TIMEOUT + - --image=$REGISTRY/kube-ovn:$VERSION securityContext: runAsUser: ${RUN_AS_USER} privileged: false diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko index 9b34a32e24d..bef1de8a5c8 100755 --- a/dist/images/kubectl-ko +++ b/dist/images/kubectl-ko @@ -625,6 +625,7 @@ diagnose(){ gcCommands=() kubectl get crd vpcs.kubeovn.io kubectl get crd vpc-nat-gateways.kubeovn.io + kubectl get crd vpc-egress-gateways.kubeovn.io kubectl get crd subnets.kubeovn.io kubectl get crd ips.kubeovn.io kubectl get crd vlans.kubeovn.io diff --git a/dist/images/start-bfdd.sh b/dist/images/start-bfdd.sh new file mode 100644 index 00000000000..f186ebbe8bd --- /dev/null +++ b/dist/images/start-bfdd.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -ex + +POD_IPS=${POD_IPS:-::} +LISTEN_IPS=($(echo "${POD_IPS}" | tr ',' ' ')) +LISTEN_ARGS="" +for ip in ${LISTEN_IPS[*]}; do + LISTEN_ARGS="${LISTEN_ARGS} --listen=${ip}" +done + +bfdd-beacon --nofork --tee ${LISTEN_ARGS} diff --git a/mocks/pkg/ovs/interface.go b/mocks/pkg/ovs/interface.go index f5bcdfeef3a..98cbc0553c9 100644 --- a/mocks/pkg/ovs/interface.go +++ b/mocks/pkg/ovs/interface.go @@ -660,32 +660,61 @@ func (m *MockBFD) EXPECT() *MockBFDMockRecorder { } // CreateBFD mocks base method. -func (m *MockBFD) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int) (*ovnnb.BFD, error) { +func (m *MockBFD) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int, externalIDs map[string]string) (*ovnnb.BFD, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateBFD", lrpName, dstIP, minRx, minTx, detectMult) + ret := m.ctrl.Call(m, "CreateBFD", lrpName, dstIP, minRx, minTx, detectMult, externalIDs) ret0, _ := ret[0].(*ovnnb.BFD) ret1, _ := ret[1].(error) return ret0, ret1 } // CreateBFD indicates an expected call of CreateBFD. -func (mr *MockBFDMockRecorder) CreateBFD(lrpName, dstIP, minRx, minTx, detectMult any) *gomock.Call { +func (mr *MockBFDMockRecorder) CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, externalIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBFD", reflect.TypeOf((*MockBFD)(nil).CreateBFD), lrpName, dstIP, minRx, minTx, detectMult) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBFD", reflect.TypeOf((*MockBFD)(nil).CreateBFD), lrpName, dstIP, minRx, minTx, detectMult, externalIDs) } // DeleteBFD mocks base method. -func (m *MockBFD) DeleteBFD(lrpName, dstIP string) error { +func (m *MockBFD) DeleteBFD(uuid string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteBFD", lrpName, dstIP) + ret := m.ctrl.Call(m, "DeleteBFD", uuid) ret0, _ := ret[0].(error) return ret0 } // DeleteBFD indicates an expected call of DeleteBFD. -func (mr *MockBFDMockRecorder) DeleteBFD(lrpName, dstIP any) *gomock.Call { +func (mr *MockBFDMockRecorder) DeleteBFD(uuid any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFD", reflect.TypeOf((*MockBFD)(nil).DeleteBFD), lrpName, dstIP) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFD", reflect.TypeOf((*MockBFD)(nil).DeleteBFD), uuid) +} + +// DeleteBFDByDstIP mocks base method. +func (m *MockBFD) DeleteBFDByDstIP(lrpName, dstIP string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteBFDByDstIP", lrpName, dstIP) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteBFDByDstIP indicates an expected call of DeleteBFDByDstIP. +func (mr *MockBFDMockRecorder) DeleteBFDByDstIP(lrpName, dstIP any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFDByDstIP", reflect.TypeOf((*MockBFD)(nil).DeleteBFDByDstIP), lrpName, dstIP) +} + +// FindBFD mocks base method. +func (m *MockBFD) FindBFD(externalIDs map[string]string) ([]ovnnb.BFD, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindBFD", externalIDs) + ret0, _ := ret[0].([]ovnnb.BFD) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindBFD indicates an expected call of FindBFD. +func (mr *MockBFDMockRecorder) FindBFD(externalIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindBFD", reflect.TypeOf((*MockBFD)(nil).FindBFD), externalIDs) } // ListBFDs mocks base method. @@ -2161,6 +2190,34 @@ func (mr *MockLogicalRouterStaticRouteMockRecorder) DeleteLogicalRouterStaticRou return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRoute", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).DeleteLogicalRouterStaticRoute), lrName, routeTable, policy, ipPrefix, nextHop) } +// DeleteLogicalRouterStaticRouteByExternalIDs mocks base method. +func (m *MockLogicalRouterStaticRoute) DeleteLogicalRouterStaticRouteByExternalIDs(lrName string, externalIDs map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLogicalRouterStaticRouteByExternalIDs", lrName, externalIDs) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLogicalRouterStaticRouteByExternalIDs indicates an expected call of DeleteLogicalRouterStaticRouteByExternalIDs. +func (mr *MockLogicalRouterStaticRouteMockRecorder) DeleteLogicalRouterStaticRouteByExternalIDs(lrName, externalIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRouteByExternalIDs", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).DeleteLogicalRouterStaticRouteByExternalIDs), lrName, externalIDs) +} + +// DeleteLogicalRouterStaticRouteByUUID mocks base method. +func (m *MockLogicalRouterStaticRoute) DeleteLogicalRouterStaticRouteByUUID(lrName, uuid string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLogicalRouterStaticRouteByUUID", lrName, uuid) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLogicalRouterStaticRouteByUUID indicates an expected call of DeleteLogicalRouterStaticRouteByUUID. +func (mr *MockLogicalRouterStaticRouteMockRecorder) DeleteLogicalRouterStaticRouteByUUID(lrName, uuid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRouteByUUID", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).DeleteLogicalRouterStaticRouteByUUID), lrName, uuid) +} + // ListLogicalRouterStaticRoutes mocks base method. func (m *MockLogicalRouterStaticRoute) ListLogicalRouterStaticRoutes(lrName string, routeTable, policy *string, ipPrefix string, externalIDs map[string]string) ([]*ovnnb.LogicalRouterStaticRoute, error) { m.ctrl.T.Helper() @@ -2206,6 +2263,25 @@ func (mr *MockLogicalRouterStaticRouteMockRecorder) LogicalRouterStaticRouteExis return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogicalRouterStaticRouteExists", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).LogicalRouterStaticRouteExists), lrName, routeTable, policy, ipPrefix, nexthop) } +// UpdateLogicalRouterStaticRoute mocks base method. +func (m *MockLogicalRouterStaticRoute) UpdateLogicalRouterStaticRoute(route *ovnnb.LogicalRouterStaticRoute, fields ...any) error { + m.ctrl.T.Helper() + varargs := []any{route} + for _, a := range fields { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UpdateLogicalRouterStaticRoute", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLogicalRouterStaticRoute indicates an expected call of UpdateLogicalRouterStaticRoute. +func (mr *MockLogicalRouterStaticRouteMockRecorder) UpdateLogicalRouterStaticRoute(route any, fields ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{route}, fields...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterStaticRoute", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).UpdateLogicalRouterStaticRoute), varargs...) +} + // MockLogicalRouterPolicy is a mock of LogicalRouterPolicy interface. type MockLogicalRouterPolicy struct { ctrl *gomock.Controller @@ -2231,17 +2307,17 @@ func (m *MockLogicalRouterPolicy) EXPECT() *MockLogicalRouterPolicyMockRecorder } // AddLogicalRouterPolicy mocks base method. -func (m *MockLogicalRouterPolicy) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops []string, externalIDs map[string]string) error { +func (m *MockLogicalRouterPolicy) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops, bfdSessions []string, externalIDs map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddLogicalRouterPolicy", lrName, priority, match, action, nextHops, externalIDs) + ret := m.ctrl.Call(m, "AddLogicalRouterPolicy", lrName, priority, match, action, nextHops, bfdSessions, externalIDs) ret0, _ := ret[0].(error) return ret0 } // AddLogicalRouterPolicy indicates an expected call of AddLogicalRouterPolicy. -func (mr *MockLogicalRouterPolicyMockRecorder) AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, externalIDs any) *gomock.Call { +func (mr *MockLogicalRouterPolicyMockRecorder) AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, bfdSessions, externalIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLogicalRouterPolicy", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).AddLogicalRouterPolicy), lrName, priority, match, action, nextHops, externalIDs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLogicalRouterPolicy", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).AddLogicalRouterPolicy), lrName, priority, match, action, nextHops, bfdSessions, externalIDs) } // ClearLogicalRouterPolicy mocks base method. @@ -2359,6 +2435,25 @@ func (mr *MockLogicalRouterPolicyMockRecorder) ListLogicalRouterPolicies(lrName, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLogicalRouterPolicies", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).ListLogicalRouterPolicies), lrName, priority, externalIDs, ignoreExtIDEmptyValue) } +// UpdateLogicalRouterPolicy mocks base method. +func (m *MockLogicalRouterPolicy) UpdateLogicalRouterPolicy(policy *ovnnb.LogicalRouterPolicy, fields ...any) error { + m.ctrl.T.Helper() + varargs := []any{policy} + for _, a := range fields { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UpdateLogicalRouterPolicy", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLogicalRouterPolicy indicates an expected call of UpdateLogicalRouterPolicy. +func (mr *MockLogicalRouterPolicyMockRecorder) UpdateLogicalRouterPolicy(policy any, fields ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{policy}, fields...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterPolicy", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).UpdateLogicalRouterPolicy), varargs...) +} + // MockNAT is a mock of NAT interface. type MockNAT struct { ctrl *gomock.Controller @@ -2623,17 +2718,17 @@ func (mr *MockNbClientMockRecorder) AddLoadBalancerHealthCheck(lbName, vip, exte } // AddLogicalRouterPolicy mocks base method. -func (m *MockNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops []string, externalIDs map[string]string) error { +func (m *MockNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops, bfdSessions []string, externalIDs map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddLogicalRouterPolicy", lrName, priority, match, action, nextHops, externalIDs) + ret := m.ctrl.Call(m, "AddLogicalRouterPolicy", lrName, priority, match, action, nextHops, bfdSessions, externalIDs) ret0, _ := ret[0].(error) return ret0 } // AddLogicalRouterPolicy indicates an expected call of AddLogicalRouterPolicy. -func (mr *MockNbClientMockRecorder) AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, externalIDs any) *gomock.Call { +func (mr *MockNbClientMockRecorder) AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, bfdSessions, externalIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLogicalRouterPolicy", reflect.TypeOf((*MockNbClient)(nil).AddLogicalRouterPolicy), lrName, priority, match, action, nextHops, externalIDs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLogicalRouterPolicy", reflect.TypeOf((*MockNbClient)(nil).AddLogicalRouterPolicy), lrName, priority, match, action, nextHops, bfdSessions, externalIDs) } // AddLogicalRouterStaticRoute mocks base method. @@ -2745,18 +2840,18 @@ func (mr *MockNbClientMockRecorder) CreateAddressSet(asName, externalIDs any) *g } // CreateBFD mocks base method. -func (m *MockNbClient) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int) (*ovnnb.BFD, error) { +func (m *MockNbClient) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int, externalIDs map[string]string) (*ovnnb.BFD, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateBFD", lrpName, dstIP, minRx, minTx, detectMult) + ret := m.ctrl.Call(m, "CreateBFD", lrpName, dstIP, minRx, minTx, detectMult, externalIDs) ret0, _ := ret[0].(*ovnnb.BFD) ret1, _ := ret[1].(error) return ret0, ret1 } // CreateBFD indicates an expected call of CreateBFD. -func (mr *MockNbClientMockRecorder) CreateBFD(lrpName, dstIP, minRx, minTx, detectMult any) *gomock.Call { +func (mr *MockNbClientMockRecorder) CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, externalIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBFD", reflect.TypeOf((*MockNbClient)(nil).CreateBFD), lrpName, dstIP, minRx, minTx, detectMult) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBFD", reflect.TypeOf((*MockNbClient)(nil).CreateBFD), lrpName, dstIP, minRx, minTx, detectMult, externalIDs) } // CreateBareLogicalSwitch mocks base method. @@ -3116,17 +3211,31 @@ func (mr *MockNbClientMockRecorder) DeleteAddressSets(externalIDs any) *gomock.C } // DeleteBFD mocks base method. -func (m *MockNbClient) DeleteBFD(lrpName, dstIP string) error { +func (m *MockNbClient) DeleteBFD(uuid string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteBFD", lrpName, dstIP) + ret := m.ctrl.Call(m, "DeleteBFD", uuid) ret0, _ := ret[0].(error) return ret0 } // DeleteBFD indicates an expected call of DeleteBFD. -func (mr *MockNbClientMockRecorder) DeleteBFD(lrpName, dstIP any) *gomock.Call { +func (mr *MockNbClientMockRecorder) DeleteBFD(uuid any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFD", reflect.TypeOf((*MockNbClient)(nil).DeleteBFD), lrpName, dstIP) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFD", reflect.TypeOf((*MockNbClient)(nil).DeleteBFD), uuid) +} + +// DeleteBFDByDstIP mocks base method. +func (m *MockNbClient) DeleteBFDByDstIP(lrpName, dstIP string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteBFDByDstIP", lrpName, dstIP) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteBFDByDstIP indicates an expected call of DeleteBFDByDstIP. +func (mr *MockNbClientMockRecorder) DeleteBFDByDstIP(lrpName, dstIP any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFDByDstIP", reflect.TypeOf((*MockNbClient)(nil).DeleteBFDByDstIP), lrpName, dstIP) } // DeleteDHCPOptions mocks base method. @@ -3343,6 +3452,34 @@ func (mr *MockNbClientMockRecorder) DeleteLogicalRouterStaticRoute(lrName, route return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRoute", reflect.TypeOf((*MockNbClient)(nil).DeleteLogicalRouterStaticRoute), lrName, routeTable, policy, ipPrefix, nextHop) } +// DeleteLogicalRouterStaticRouteByExternalIDs mocks base method. +func (m *MockNbClient) DeleteLogicalRouterStaticRouteByExternalIDs(lrName string, externalIDs map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLogicalRouterStaticRouteByExternalIDs", lrName, externalIDs) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLogicalRouterStaticRouteByExternalIDs indicates an expected call of DeleteLogicalRouterStaticRouteByExternalIDs. +func (mr *MockNbClientMockRecorder) DeleteLogicalRouterStaticRouteByExternalIDs(lrName, externalIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRouteByExternalIDs", reflect.TypeOf((*MockNbClient)(nil).DeleteLogicalRouterStaticRouteByExternalIDs), lrName, externalIDs) +} + +// DeleteLogicalRouterStaticRouteByUUID mocks base method. +func (m *MockNbClient) DeleteLogicalRouterStaticRouteByUUID(lrName, uuid string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLogicalRouterStaticRouteByUUID", lrName, uuid) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLogicalRouterStaticRouteByUUID indicates an expected call of DeleteLogicalRouterStaticRouteByUUID. +func (mr *MockNbClientMockRecorder) DeleteLogicalRouterStaticRouteByUUID(lrName, uuid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRouteByUUID", reflect.TypeOf((*MockNbClient)(nil).DeleteLogicalRouterStaticRouteByUUID), lrName, uuid) +} + // DeleteLogicalSwitch mocks base method. func (m *MockNbClient) DeleteLogicalSwitch(lsName string) error { m.ctrl.T.Helper() @@ -3459,6 +3596,21 @@ func (mr *MockNbClientMockRecorder) EnablePortLayer2forward(lspName any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnablePortLayer2forward", reflect.TypeOf((*MockNbClient)(nil).EnablePortLayer2forward), lspName) } +// FindBFD mocks base method. +func (m *MockNbClient) FindBFD(externalIDs map[string]string) ([]ovnnb.BFD, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindBFD", externalIDs) + ret0, _ := ret[0].([]ovnnb.BFD) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindBFD indicates an expected call of FindBFD. +func (mr *MockNbClientMockRecorder) FindBFD(externalIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindBFD", reflect.TypeOf((*MockNbClient)(nil).FindBFD), externalIDs) +} + // GetEntityInfo mocks base method. func (m *MockNbClient) GetEntityInfo(entity any) error { m.ctrl.T.Helper() @@ -4766,6 +4918,25 @@ func (mr *MockNbClientMockRecorder) UpdateLogicalRouter(lr any, fields ...any) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouter", reflect.TypeOf((*MockNbClient)(nil).UpdateLogicalRouter), varargs...) } +// UpdateLogicalRouterPolicy mocks base method. +func (m *MockNbClient) UpdateLogicalRouterPolicy(policy *ovnnb.LogicalRouterPolicy, fields ...any) error { + m.ctrl.T.Helper() + varargs := []any{policy} + for _, a := range fields { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UpdateLogicalRouterPolicy", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLogicalRouterPolicy indicates an expected call of UpdateLogicalRouterPolicy. +func (mr *MockNbClientMockRecorder) UpdateLogicalRouterPolicy(policy any, fields ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{policy}, fields...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterPolicy", reflect.TypeOf((*MockNbClient)(nil).UpdateLogicalRouterPolicy), varargs...) +} + // UpdateLogicalRouterPortNetworks mocks base method. func (m *MockNbClient) UpdateLogicalRouterPortNetworks(lrpName string, networks []string) error { m.ctrl.T.Helper() @@ -4808,6 +4979,25 @@ func (mr *MockNbClientMockRecorder) UpdateLogicalRouterPortRA(lrpName, ipv6RACon return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterPortRA", reflect.TypeOf((*MockNbClient)(nil).UpdateLogicalRouterPortRA), lrpName, ipv6RAConfigsStr, enableIPv6RA) } +// UpdateLogicalRouterStaticRoute mocks base method. +func (m *MockNbClient) UpdateLogicalRouterStaticRoute(route *ovnnb.LogicalRouterStaticRoute, fields ...any) error { + m.ctrl.T.Helper() + varargs := []any{route} + for _, a := range fields { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UpdateLogicalRouterStaticRoute", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLogicalRouterStaticRoute indicates an expected call of UpdateLogicalRouterStaticRoute. +func (mr *MockNbClientMockRecorder) UpdateLogicalRouterStaticRoute(route any, fields ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{route}, fields...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterStaticRoute", reflect.TypeOf((*MockNbClient)(nil).UpdateLogicalRouterStaticRoute), varargs...) +} + // UpdateLogicalSwitchACL mocks base method. func (m *MockNbClient) UpdateLogicalSwitchACL(lsName, cidrBlock string, subnetAcls []v1.ACL, allowEWTraffic bool) error { m.ctrl.T.Helper() diff --git a/pkg/apis/kubeovn/v1/condition.go b/pkg/apis/kubeovn/v1/condition.go index 748e7827408..0f36692e4de 100644 --- a/pkg/apis/kubeovn/v1/condition.go +++ b/pkg/apis/kubeovn/v1/condition.go @@ -11,6 +11,8 @@ const ( Ready = "Ready" // Validated => Spec passed validating Validated = "Validated" + // Init => controller is initializing this resource + Init = "Init" // Error => last recorded error Error = "Error" diff --git a/pkg/apis/kubeovn/v1/register.go b/pkg/apis/kubeovn/v1/register.go index c08e6f35694..020f13f8dc2 100644 --- a/pkg/apis/kubeovn/v1/register.go +++ b/pkg/apis/kubeovn/v1/register.go @@ -69,6 +69,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &VpcList{}, &VpcDns{}, &VpcDnsList{}, + &VpcEgressGateway{}, + &VpcEgressGatewayList{}, &VpcNatGateway{}, &VpcNatGatewayList{}, ) diff --git a/pkg/apis/kubeovn/v1/vpc-egress-gateway.go b/pkg/apis/kubeovn/v1/vpc-egress-gateway.go new file mode 100644 index 00000000000..b879ae5be87 --- /dev/null +++ b/pkg/apis/kubeovn/v1/vpc-egress-gateway.go @@ -0,0 +1,89 @@ +package v1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Phase represents resource phase +type Phase string + +const ( + PhasePending Phase = "Pending" + PhaseProcessing Phase = "Processing" + PhaseCompleted Phase = "Completed" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type VpcEgressGatewayList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []VpcEgressGateway `json:"items"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +resourceName=vpc-egress-gateways +type VpcEgressGateway struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec VpcEgressGatewaySpec `json:"spec"` + Status VpcEgressGatewayStatus `json:"status,omitempty"` +} + +func (g *VpcEgressGateway) Ready() bool { + return g.Status.Ready && g.Status.Conditions.IsReady(g.Generation) +} + +type VpcEgressGatewaySpec struct { + VPC string `json:"vpc,omitempty"` + Replicas int32 `json:"replicas,omitempty"` + Prefix string `json:"prefix,omitempty"` + Image string `json:"image,omitempty"` + InternalSubnet string `json:"internalSubnet,omitempty"` + ExternalSubnet string `json:"externalSubnet"` + InternalIPs []string `json:"internalIPs,omitempty"` + ExternalIPs []string `json:"externalIPs,omitempty"` + + BFD VpcEgressGatewayBFDConfig `json:"bfd,omitempty"` + Policies []VpcEgressGatewayPolicy `json:"policies,omitempty"` + NodeSelector []VpcEgressGatewayNodeSelector `json:"nodeSelector,omitempty"` +} + +type VpcEgressGatewayBFDConfig struct { + Enabled bool `json:"enabled"` + MinRX int32 `json:"minRX,omitempty"` + MinTX int32 `json:"minTX,omitempty"` + Multiplier int32 `json:"multiplier,omitempty"` +} + +type VpcEgressGatewayPolicy struct { + SNAT bool `json:"snat"` + IPBlocks []string `json:"ipBlocks,omitempty"` + Subnets []string `json:"subnets,omitempty"` +} + +type VpcEgressGatewayNodeSelector struct { + MatchLabels map[string]string `json:"matchLabels,omitempty"` + MatchExpressions []corev1.NodeSelectorRequirement `json:"matchExpressions,omitempty"` + MatchFields []corev1.NodeSelectorRequirement `json:"matchFields,omitempty"` +} + +type VpcEgressGatewayStatus struct { + Ready bool `json:"ready"` + Phase Phase `json:"phase"` + InternalIPs []string `json:"internalIPs,omitempty"` + ExternalIPs []string `json:"externalIPs,omitempty"` + Conditions Conditions `json:"conditions,omitempty"` + + Workload VpcEgressWorkload `json:"workload,omitempty"` +} + +type VpcEgressWorkload struct { + APIVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + Nodes []string `json:"nodes,omitempty"` +} diff --git a/pkg/apis/kubeovn/v1/vpc.go b/pkg/apis/kubeovn/v1/vpc.go index 423becc736c..ad502056824 100644 --- a/pkg/apis/kubeovn/v1/vpc.go +++ b/pkg/apis/kubeovn/v1/vpc.go @@ -87,10 +87,9 @@ type PolicyRoute struct { } type BFDPortStatus struct { - Enabled bool `json:"enabled"` - Name string `json:"name,omitempty"` - IP string `json:"ip,omitempty"` - Nodes []string `json:"nodes,omitempty"` + Name string `json:"name,omitempty"` + IP string `json:"ip,omitempty"` + Nodes []string `json:"nodes,omitempty"` } type VpcStatus struct { diff --git a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go index 52bed52889b..5fd23154f3f 100644 --- a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go +++ b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go @@ -103,6 +103,28 @@ func (in *Condition) DeepCopy() *Condition { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in Conditions) DeepCopyInto(out *Conditions) { + { + in := &in + *out = make(Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + return + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Conditions. +func (in Conditions) DeepCopy() Conditions { + if in == nil { + return nil + } + out := new(Conditions) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CustomInterface) DeepCopyInto(out *CustomInterface) { *out = *in @@ -2268,6 +2290,242 @@ func (in *VpcDnsList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGateway) DeepCopyInto(out *VpcEgressGateway) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGateway. +func (in *VpcEgressGateway) DeepCopy() *VpcEgressGateway { + if in == nil { + return nil + } + out := new(VpcEgressGateway) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VpcEgressGateway) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewayBFDConfig) DeepCopyInto(out *VpcEgressGatewayBFDConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayBFDConfig. +func (in *VpcEgressGatewayBFDConfig) DeepCopy() *VpcEgressGatewayBFDConfig { + if in == nil { + return nil + } + out := new(VpcEgressGatewayBFDConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewayList) DeepCopyInto(out *VpcEgressGatewayList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VpcEgressGateway, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayList. +func (in *VpcEgressGatewayList) DeepCopy() *VpcEgressGatewayList { + if in == nil { + return nil + } + out := new(VpcEgressGatewayList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VpcEgressGatewayList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewayNodeSelector) DeepCopyInto(out *VpcEgressGatewayNodeSelector) { + *out = *in + if in.MatchLabels != nil { + in, out := &in.MatchLabels, &out.MatchLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.MatchExpressions != nil { + in, out := &in.MatchExpressions, &out.MatchExpressions + *out = make([]corev1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.MatchFields != nil { + in, out := &in.MatchFields, &out.MatchFields + *out = make([]corev1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayNodeSelector. +func (in *VpcEgressGatewayNodeSelector) DeepCopy() *VpcEgressGatewayNodeSelector { + if in == nil { + return nil + } + out := new(VpcEgressGatewayNodeSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewayPolicy) DeepCopyInto(out *VpcEgressGatewayPolicy) { + *out = *in + if in.IPBlocks != nil { + in, out := &in.IPBlocks, &out.IPBlocks + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayPolicy. +func (in *VpcEgressGatewayPolicy) DeepCopy() *VpcEgressGatewayPolicy { + if in == nil { + return nil + } + out := new(VpcEgressGatewayPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewaySpec) DeepCopyInto(out *VpcEgressGatewaySpec) { + *out = *in + if in.InternalIPs != nil { + in, out := &in.InternalIPs, &out.InternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalIPs != nil { + in, out := &in.ExternalIPs, &out.ExternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.BFD = in.BFD + if in.Policies != nil { + in, out := &in.Policies, &out.Policies + *out = make([]VpcEgressGatewayPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make([]VpcEgressGatewayNodeSelector, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewaySpec. +func (in *VpcEgressGatewaySpec) DeepCopy() *VpcEgressGatewaySpec { + if in == nil { + return nil + } + out := new(VpcEgressGatewaySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewayStatus) DeepCopyInto(out *VpcEgressGatewayStatus) { + *out = *in + if in.InternalIPs != nil { + in, out := &in.InternalIPs, &out.InternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalIPs != nil { + in, out := &in.ExternalIPs, &out.ExternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Workload.DeepCopyInto(&out.Workload) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayStatus. +func (in *VpcEgressGatewayStatus) DeepCopy() *VpcEgressGatewayStatus { + if in == nil { + return nil + } + out := new(VpcEgressGatewayStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressWorkload) DeepCopyInto(out *VpcEgressWorkload) { + *out = *in + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressWorkload. +func (in *VpcEgressWorkload) DeepCopy() *VpcEgressWorkload { + if in == nil { + return nil + } + out := new(VpcEgressWorkload) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VpcList) DeepCopyInto(out *VpcList) { *out = *in diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go index be653ef62ab..a4857aada11 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go @@ -104,6 +104,10 @@ func (c *FakeKubeovnV1) VpcDnses() v1.VpcDnsInterface { return &FakeVpcDnses{c} } +func (c *FakeKubeovnV1) VpcEgressGateways(namespace string) v1.VpcEgressGatewayInterface { + return &FakeVpcEgressGateways{c, namespace} +} + func (c *FakeKubeovnV1) VpcNatGateways() v1.VpcNatGatewayInterface { return &FakeVpcNatGateways{c} } diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcegressgateway.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcegressgateway.go new file mode 100644 index 00000000000..013cca6d5f7 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcegressgateway.go @@ -0,0 +1,147 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeVpcEgressGateways implements VpcEgressGatewayInterface +type FakeVpcEgressGateways struct { + Fake *FakeKubeovnV1 + ns string +} + +var vpcegressgatewaysResource = v1.SchemeGroupVersion.WithResource("vpc-egress-gateways") + +var vpcegressgatewaysKind = v1.SchemeGroupVersion.WithKind("VpcEgressGateway") + +// Get takes name of the vpcEgressGateway, and returns the corresponding vpcEgressGateway object, and an error if there is any. +func (c *FakeVpcEgressGateways) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.VpcEgressGateway, err error) { + emptyResult := &v1.VpcEgressGateway{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(vpcegressgatewaysResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1.VpcEgressGateway), err +} + +// List takes label and field selectors, and returns the list of VpcEgressGateways that match those selectors. +func (c *FakeVpcEgressGateways) List(ctx context.Context, opts metav1.ListOptions) (result *v1.VpcEgressGatewayList, err error) { + emptyResult := &v1.VpcEgressGatewayList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(vpcegressgatewaysResource, vpcegressgatewaysKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1.VpcEgressGatewayList{ListMeta: obj.(*v1.VpcEgressGatewayList).ListMeta} + for _, item := range obj.(*v1.VpcEgressGatewayList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested vpcEgressGateways. +func (c *FakeVpcEgressGateways) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(vpcegressgatewaysResource, c.ns, opts)) + +} + +// Create takes the representation of a vpcEgressGateway and creates it. Returns the server's representation of the vpcEgressGateway, and an error, if there is any. +func (c *FakeVpcEgressGateways) Create(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.CreateOptions) (result *v1.VpcEgressGateway, err error) { + emptyResult := &v1.VpcEgressGateway{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(vpcegressgatewaysResource, c.ns, vpcEgressGateway, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1.VpcEgressGateway), err +} + +// Update takes the representation of a vpcEgressGateway and updates it. Returns the server's representation of the vpcEgressGateway, and an error, if there is any. +func (c *FakeVpcEgressGateways) Update(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.UpdateOptions) (result *v1.VpcEgressGateway, err error) { + emptyResult := &v1.VpcEgressGateway{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(vpcegressgatewaysResource, c.ns, vpcEgressGateway, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1.VpcEgressGateway), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeVpcEgressGateways) UpdateStatus(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.UpdateOptions) (result *v1.VpcEgressGateway, err error) { + emptyResult := &v1.VpcEgressGateway{} + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceActionWithOptions(vpcegressgatewaysResource, "status", c.ns, vpcEgressGateway, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1.VpcEgressGateway), err +} + +// Delete takes name of the vpcEgressGateway and deletes it. Returns an error if one occurs. +func (c *FakeVpcEgressGateways) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(vpcegressgatewaysResource, c.ns, name, opts), &v1.VpcEgressGateway{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeVpcEgressGateways) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(vpcegressgatewaysResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1.VpcEgressGatewayList{}) + return err +} + +// Patch applies the patch and returns the patched vpcEgressGateway. +func (c *FakeVpcEgressGateways) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.VpcEgressGateway, err error) { + emptyResult := &v1.VpcEgressGateway{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(vpcegressgatewaysResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1.VpcEgressGateway), err +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go index 15876f7e55b..a4a3dcf75f0 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go @@ -56,4 +56,6 @@ type VpcExpansion interface{} type VpcDnsExpansion interface{} +type VpcEgressGatewayExpansion interface{} + type VpcNatGatewayExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go index bd9b017ea8c..aa73b0f4b66 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go @@ -47,6 +47,7 @@ type KubeovnV1Interface interface { VlansGetter VpcsGetter VpcDnsesGetter + VpcEgressGatewaysGetter VpcNatGatewaysGetter } @@ -131,6 +132,10 @@ func (c *KubeovnV1Client) VpcDnses() VpcDnsInterface { return newVpcDnses(c) } +func (c *KubeovnV1Client) VpcEgressGateways(namespace string) VpcEgressGatewayInterface { + return newVpcEgressGateways(c, namespace) +} + func (c *KubeovnV1Client) VpcNatGateways() VpcNatGatewayInterface { return newVpcNatGateways(c) } diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/vpcegressgateway.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/vpcegressgateway.go new file mode 100644 index 00000000000..09f4bafa669 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/vpcegressgateway.go @@ -0,0 +1,69 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + scheme "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// VpcEgressGatewaysGetter has a method to return a VpcEgressGatewayInterface. +// A group's client should implement this interface. +type VpcEgressGatewaysGetter interface { + VpcEgressGateways(namespace string) VpcEgressGatewayInterface +} + +// VpcEgressGatewayInterface has methods to work with VpcEgressGateway resources. +type VpcEgressGatewayInterface interface { + Create(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.CreateOptions) (*v1.VpcEgressGateway, error) + Update(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.UpdateOptions) (*v1.VpcEgressGateway, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.UpdateOptions) (*v1.VpcEgressGateway, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.VpcEgressGateway, error) + List(ctx context.Context, opts metav1.ListOptions) (*v1.VpcEgressGatewayList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.VpcEgressGateway, err error) + VpcEgressGatewayExpansion +} + +// vpcEgressGateways implements VpcEgressGatewayInterface +type vpcEgressGateways struct { + *gentype.ClientWithList[*v1.VpcEgressGateway, *v1.VpcEgressGatewayList] +} + +// newVpcEgressGateways returns a VpcEgressGateways +func newVpcEgressGateways(c *KubeovnV1Client, namespace string) *vpcEgressGateways { + return &vpcEgressGateways{ + gentype.NewClientWithList[*v1.VpcEgressGateway, *v1.VpcEgressGatewayList]( + "vpc-egress-gateways", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1.VpcEgressGateway { return &v1.VpcEgressGateway{} }, + func() *v1.VpcEgressGatewayList { return &v1.VpcEgressGatewayList{} }), + } +} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index 61c4b5fc9fa..03bcdd8f8ef 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -91,6 +91,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().Vpcs().Informer()}, nil case v1.SchemeGroupVersion.WithResource("vpc-dnses"): return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcDnses().Informer()}, nil + case v1.SchemeGroupVersion.WithResource("vpc-egress-gateways"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcEgressGateways().Informer()}, nil case v1.SchemeGroupVersion.WithResource("vpc-nat-gateways"): return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcNatGateways().Informer()}, nil diff --git a/pkg/client/informers/externalversions/kubeovn/v1/interface.go b/pkg/client/informers/externalversions/kubeovn/v1/interface.go index 2b13d05651d..69cad95925a 100644 --- a/pkg/client/informers/externalversions/kubeovn/v1/interface.go +++ b/pkg/client/informers/externalversions/kubeovn/v1/interface.go @@ -62,6 +62,8 @@ type Interface interface { Vpcs() VpcInformer // VpcDnses returns a VpcDnsInformer. VpcDnses() VpcDnsInformer + // VpcEgressGateways returns a VpcEgressGatewayInformer. + VpcEgressGateways() VpcEgressGatewayInformer // VpcNatGateways returns a VpcNatGatewayInformer. VpcNatGateways() VpcNatGatewayInformer } @@ -172,6 +174,11 @@ func (v *version) VpcDnses() VpcDnsInformer { return &vpcDnsInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} } +// VpcEgressGateways returns a VpcEgressGatewayInformer. +func (v *version) VpcEgressGateways() VpcEgressGatewayInformer { + return &vpcEgressGatewayInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // VpcNatGateways returns a VpcNatGatewayInformer. func (v *version) VpcNatGateways() VpcNatGatewayInformer { return &vpcNatGatewayInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/informers/externalversions/kubeovn/v1/vpcegressgateway.go b/pkg/client/informers/externalversions/kubeovn/v1/vpcegressgateway.go new file mode 100644 index 00000000000..aa8f3e94067 --- /dev/null +++ b/pkg/client/informers/externalversions/kubeovn/v1/vpcegressgateway.go @@ -0,0 +1,90 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + time "time" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces" + v1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// VpcEgressGatewayInformer provides access to a shared informer and lister for +// VpcEgressGateways. +type VpcEgressGatewayInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1.VpcEgressGatewayLister +} + +type vpcEgressGatewayInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewVpcEgressGatewayInformer constructs a new informer for VpcEgressGateway type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewVpcEgressGatewayInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredVpcEgressGatewayInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredVpcEgressGatewayInformer constructs a new informer for VpcEgressGateway type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredVpcEgressGatewayInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().VpcEgressGateways(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().VpcEgressGateways(namespace).Watch(context.TODO(), options) + }, + }, + &kubeovnv1.VpcEgressGateway{}, + resyncPeriod, + indexers, + ) +} + +func (f *vpcEgressGatewayInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredVpcEgressGatewayInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *vpcEgressGatewayInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&kubeovnv1.VpcEgressGateway{}, f.defaultInformer) +} + +func (f *vpcEgressGatewayInformer) Lister() v1.VpcEgressGatewayLister { + return v1.NewVpcEgressGatewayLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/listers/kubeovn/v1/expansion_generated.go b/pkg/client/listers/kubeovn/v1/expansion_generated.go index 998c1ddaa14..b474d11b77d 100644 --- a/pkg/client/listers/kubeovn/v1/expansion_generated.go +++ b/pkg/client/listers/kubeovn/v1/expansion_generated.go @@ -94,6 +94,14 @@ type VpcListerExpansion interface{} // VpcDnsLister. type VpcDnsListerExpansion interface{} +// VpcEgressGatewayListerExpansion allows custom methods to be added to +// VpcEgressGatewayLister. +type VpcEgressGatewayListerExpansion interface{} + +// VpcEgressGatewayNamespaceListerExpansion allows custom methods to be added to +// VpcEgressGatewayNamespaceLister. +type VpcEgressGatewayNamespaceListerExpansion interface{} + // VpcNatGatewayListerExpansion allows custom methods to be added to // VpcNatGatewayLister. type VpcNatGatewayListerExpansion interface{} diff --git a/pkg/client/listers/kubeovn/v1/vpcegressgateway.go b/pkg/client/listers/kubeovn/v1/vpcegressgateway.go new file mode 100644 index 00000000000..db9295e3149 --- /dev/null +++ b/pkg/client/listers/kubeovn/v1/vpcegressgateway.go @@ -0,0 +1,70 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/listers" + "k8s.io/client-go/tools/cache" +) + +// VpcEgressGatewayLister helps list VpcEgressGateways. +// All objects returned here must be treated as read-only. +type VpcEgressGatewayLister interface { + // List lists all VpcEgressGateways in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.VpcEgressGateway, err error) + // VpcEgressGateways returns an object that can list and get VpcEgressGateways. + VpcEgressGateways(namespace string) VpcEgressGatewayNamespaceLister + VpcEgressGatewayListerExpansion +} + +// vpcEgressGatewayLister implements the VpcEgressGatewayLister interface. +type vpcEgressGatewayLister struct { + listers.ResourceIndexer[*v1.VpcEgressGateway] +} + +// NewVpcEgressGatewayLister returns a new VpcEgressGatewayLister. +func NewVpcEgressGatewayLister(indexer cache.Indexer) VpcEgressGatewayLister { + return &vpcEgressGatewayLister{listers.New[*v1.VpcEgressGateway](indexer, v1.Resource("vpcegressgateway"))} +} + +// VpcEgressGateways returns an object that can list and get VpcEgressGateways. +func (s *vpcEgressGatewayLister) VpcEgressGateways(namespace string) VpcEgressGatewayNamespaceLister { + return vpcEgressGatewayNamespaceLister{listers.NewNamespaced[*v1.VpcEgressGateway](s.ResourceIndexer, namespace)} +} + +// VpcEgressGatewayNamespaceLister helps list and get VpcEgressGateways. +// All objects returned here must be treated as read-only. +type VpcEgressGatewayNamespaceLister interface { + // List lists all VpcEgressGateways in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.VpcEgressGateway, err error) + // Get retrieves the VpcEgressGateway from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1.VpcEgressGateway, error) + VpcEgressGatewayNamespaceListerExpansion +} + +// vpcEgressGatewayNamespaceLister implements the VpcEgressGatewayNamespaceLister +// interface. +type vpcEgressGatewayNamespaceLister struct { + listers.ResourceIndexer[*v1.VpcEgressGateway] +} diff --git a/pkg/controller/config.go b/pkg/controller/config.go index 9963e31445d..e98c14b3a29 100644 --- a/pkg/controller/config.go +++ b/pkg/controller/config.go @@ -109,6 +109,9 @@ type Configuration struct { BfdDetectMult int NodeLocalDNSIPs []string + + // used to set vpc-egress-gateway image + Image string } // ParseFlags parses cmd args then init kubeclient and conf @@ -186,6 +189,8 @@ func ParseFlags() (*Configuration, error) { argBfdMinTx = pflag.Int("bfd-min-tx", 100, "This is the minimum interval, in milliseconds, ovn would like to use when transmitting BFD Control packets") argBfdMinRx = pflag.Int("bfd-min-rx", 100, "This is the minimum interval, in milliseconds, between received BFD Control packets") argBfdDetectMult = pflag.Int("detect-mult", 3, "The negotiated transmit interval, multiplied by this value, provides the Detection Time for the receiving system in Asynchronous mode.") + + argImage = pflag.String("image", "", "The image for vpc-egress-gateway") ) klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) @@ -269,6 +274,7 @@ func ParseFlags() (*Configuration, error) { BfdMinRx: *argBfdMinRx, BfdDetectMult: *argBfdDetectMult, EnableANP: *argEnableANP, + Image: *argImage, } if config.NetworkType == util.NetworkTypeVlan && config.DefaultHostInterface == "" { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 7146382f554..14e7a1886a1 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -17,6 +17,7 @@ import ( kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/scheme" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + appsv1 "k8s.io/client-go/listers/apps/v1" certListerv1 "k8s.io/client-go/listers/certificates/v1" v1 "k8s.io/client-go/listers/core/v1" netv1 "k8s.io/client-go/listers/networking/v1" @@ -95,6 +96,12 @@ type Controller struct { updateVpcSubnetQueue workqueue.TypedRateLimitingInterface[string] vpcNatGwKeyMutex keymutex.KeyMutex + vpcEgressGatewayLister kubeovnlister.VpcEgressGatewayLister + vpcEgressGatewaySynced cache.InformerSynced + addOrUpdateVpcEgressGatewayQueue workqueue.TypedRateLimitingInterface[string] + delVpcEgressGatewayQueue workqueue.TypedRateLimitingInterface[string] + vpcEgressGatewayKeyMutex keymutex.KeyMutex + switchLBRuleLister kubeovnlister.SwitchLBRuleLister switchLBRuleSynced cache.InformerSynced addSwitchLBRuleQueue workqueue.TypedRateLimitingInterface[string] @@ -218,6 +225,9 @@ type Controller struct { addOrUpdateEndpointQueue workqueue.TypedRateLimitingInterface[string] epKeyMutex keymutex.KeyMutex + deploymentsLister appsv1.DeploymentLister + deploymentsSynced cache.InformerSynced + npsLister netv1.NetworkPolicyLister npsSynced cache.InformerSynced updateNpQueue workqueue.TypedRateLimitingInterface[string] @@ -303,6 +313,7 @@ func Run(ctx context.Context, config *Configuration) { vpcInformer := kubeovnInformerFactory.Kubeovn().V1().Vpcs() vpcNatGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcNatGateways() + vpcEgressGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcEgressGateways() subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() ippoolInformer := kubeovnInformerFactory.Kubeovn().V1().IPPools() ipInformer := kubeovnInformerFactory.Kubeovn().V1().IPs() @@ -319,6 +330,7 @@ func Run(ctx context.Context, config *Configuration) { nodeInformer := informerFactory.Core().V1().Nodes() serviceInformer := informerFactory.Core().V1().Services() endpointInformer := informerFactory.Core().V1().Endpoints() + deploymentInformer := informerFactory.Apps().V1().Deployments() qosPolicyInformer := kubeovnInformerFactory.Kubeovn().V1().QoSPolicies() configMapInformer := cmInformerFactory.Core().V1().ConfigMaps() npInformer := informerFactory.Networking().V1().NetworkPolicies() @@ -362,6 +374,12 @@ func Run(ctx context.Context, config *Configuration) { updateVpcSubnetQueue: newTypedRateLimitingQueue("UpdateVpcSubnet", custCrdRateLimiter), vpcNatGwKeyMutex: keymutex.NewHashed(numKeyLocks), + vpcEgressGatewayLister: vpcEgressGatewayInformer.Lister(), + vpcEgressGatewaySynced: vpcEgressGatewayInformer.Informer().HasSynced, + addOrUpdateVpcEgressGatewayQueue: newTypedRateLimitingQueue("AddOrUpdateVpcEgressGateway", custCrdRateLimiter), + delVpcEgressGatewayQueue: newTypedRateLimitingQueue("DeleteVpcEgressGateway", custCrdRateLimiter), + vpcEgressGatewayKeyMutex: keymutex.NewHashed(numKeyLocks), + subnetsLister: subnetInformer.Lister(), subnetSynced: subnetInformer.Informer().HasSynced, addOrUpdateSubnetQueue: newTypedRateLimitingQueue[string]("AddSubnet", nil), @@ -462,6 +480,9 @@ func Run(ctx context.Context, config *Configuration) { addOrUpdateEndpointQueue: newTypedRateLimitingQueue[string]("UpdateEndpoint", nil), epKeyMutex: keymutex.NewHashed(numKeyLocks), + deploymentsLister: deploymentInformer.Lister(), + deploymentsSynced: deploymentInformer.Informer().HasSynced, + qosPoliciesLister: qosPolicyInformer.Lister(), qosPolicySynced: qosPolicyInformer.Informer().HasSynced, addQoSPolicyQueue: newTypedRateLimitingQueue("AddQoSPolicy", custCrdRateLimiter), @@ -593,11 +614,12 @@ func Run(ctx context.Context, config *Configuration) { klog.Info("Waiting for informer caches to sync") cacheSyncs := []cache.InformerSynced{ - controller.vpcNatGatewaySynced, controller.vpcSynced, controller.subnetSynced, + controller.vpcNatGatewaySynced, controller.vpcEgressGatewaySynced, + controller.vpcSynced, controller.subnetSynced, controller.ipSynced, controller.virtualIpsSynced, controller.iptablesEipSynced, controller.iptablesFipSynced, controller.iptablesDnatRuleSynced, controller.iptablesSnatRuleSynced, controller.vlanSynced, controller.podsSynced, controller.namespacesSynced, controller.nodesSynced, - controller.serviceSynced, controller.endpointsSynced, controller.configMapsSynced, + controller.serviceSynced, controller.endpointsSynced, controller.deploymentsSynced, controller.configMapsSynced, controller.ovnEipSynced, controller.ovnFipSynced, controller.ovnSnatRuleSynced, controller.ovnDnatRuleSynced, } @@ -653,6 +675,13 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to add endpoint event handler") } + if _, err = deploymentInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddDeployment, + UpdateFunc: controller.enqueueUpdateDeployment, + }); err != nil { + util.LogFatalAndExit(err, "failed to add deployment event handler") + } + if _, err = vpcInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueAddVpc, UpdateFunc: controller.enqueueUpdateVpc, @@ -669,6 +698,14 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to add vpc nat gateway event handler") } + if _, err = vpcEgressGatewayInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddVpcEgressGateway, + UpdateFunc: controller.enqueueUpdateVpcEgressGateway, + DeleteFunc: controller.enqueueDeleteVpcEgressGateway, + }); err != nil { + util.LogFatalAndExit(err, "failed to add vpc egress gateway event handler") + } + if _, err = subnetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueAddSubnet, UpdateFunc: controller.enqueueUpdateSubnet, @@ -964,6 +1001,9 @@ func (c *Controller) shutdown() { c.updateVpcSnatQueue.ShutDown() c.updateVpcSubnetQueue.ShutDown() + c.addOrUpdateVpcEgressGatewayQueue.ShutDown() + c.delVpcEgressGatewayQueue.ShutDown() + if c.config.EnableLb { c.addSwitchLBRuleQueue.ShutDown() c.delSwitchLBRuleQueue.ShutDown() @@ -1045,10 +1085,14 @@ func (c *Controller) startWorkers(ctx context.Context) { klog.Info("Starting workers") go wait.Until(runWorker("add/update vpc", c.addOrUpdateVpcQueue, c.handleAddOrUpdateVpc), time.Second, ctx.Done()) + go wait.Until(runWorker("delete vpc", c.delVpcQueue, c.handleDelVpc), time.Second, ctx.Done()) + go wait.Until(runWorker("update status of vpc", c.updateVpcStatusQueue, c.handleUpdateVpcStatus), time.Second, ctx.Done()) go wait.Until(runWorker("add/update vpc nat gateway", c.addOrUpdateVpcNatGatewayQueue, c.handleAddOrUpdateVpcNatGw), time.Second, ctx.Done()) go wait.Until(runWorker("init vpc nat gateway", c.initVpcNatGatewayQueue, c.handleInitVpcNatGw), time.Second, ctx.Done()) go wait.Until(runWorker("delete vpc nat gateway", c.delVpcNatGatewayQueue, c.handleDelVpcNatGw), time.Second, ctx.Done()) + go wait.Until(runWorker("add/update vpc egress gateway", c.addOrUpdateVpcEgressGatewayQueue, c.handleAddOrUpdateVpcEgressGateway), time.Second, ctx.Done()) + go wait.Until(runWorker("delete vpc egress gateway", c.delVpcEgressGatewayQueue, c.handleDelVpcEgressGateway), time.Second, ctx.Done()) go wait.Until(runWorker("update fip for vpc nat gateway", c.updateVpcFloatingIPQueue, c.handleUpdateVpcFloatingIP), time.Second, ctx.Done()) go wait.Until(runWorker("update eip for vpc nat gateway", c.updateVpcEipQueue, c.handleUpdateVpcEip), time.Second, ctx.Done()) go wait.Until(runWorker("update dnat for vpc nat gateway", c.updateVpcDnatQueue, c.handleUpdateVpcDnat), time.Second, ctx.Done()) @@ -1100,9 +1144,6 @@ func (c *Controller) startWorkers(ctx context.Context) { } } - go wait.Until(runWorker("delete vpc", c.delVpcQueue, c.handleDelVpc), time.Second, ctx.Done()) - go wait.Until(runWorker("update status of vpc", c.updateVpcStatusQueue, c.handleUpdateVpcStatus), time.Second, ctx.Done()) - if c.config.EnableLb { go wait.Until(runWorker("add service", c.addServiceQueue, c.handleAddService), time.Second, ctx.Done()) // run in a single worker to avoid delete the last vip, which will lead ovn to delete the loadbalancer diff --git a/pkg/controller/deployment.go b/pkg/controller/deployment.go new file mode 100644 index 00000000000..6286782beff --- /dev/null +++ b/pkg/controller/deployment.go @@ -0,0 +1,51 @@ +package controller + +import ( + "fmt" + "reflect" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/klog/v2" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" +) + +var ( + deploymentGroupVersion string + deploymentKind string + vpcEgressGatewayGroupVersion string + vpcEgressGatewayKind string +) + +func init() { + name := reflect.TypeOf(&appsv1.Deployment{}).Elem().Name() + gvk := appsv1.SchemeGroupVersion.WithKind(name) + deploymentGroupVersion = gvk.GroupVersion().String() + deploymentKind = gvk.Kind + + name = reflect.TypeOf(&kubeovnv1.VpcEgressGateway{}).Elem().Name() + gvk = kubeovnv1.SchemeGroupVersion.WithKind(name) + vpcEgressGatewayGroupVersion = gvk.GroupVersion().String() + vpcEgressGatewayKind = gvk.Kind +} + +func (c *Controller) enqueueAddDeployment(obj interface{}) { + deploy := obj.(*appsv1.Deployment) + var vegName string + for _, ref := range deploy.OwnerReferences { + if ref.APIVersion == vpcEgressGatewayGroupVersion && ref.Kind == vpcEgressGatewayKind { + vegName = ref.Name + break + } + } + + if vegName != "" { + key := fmt.Sprintf("%s/%s", deploy.Namespace, vegName) + klog.V(3).Infof("enqueue update vpc-egress-gateway %s", key) + c.addOrUpdateVpcEgressGatewayQueue.Add(key) + } +} + +func (c *Controller) enqueueUpdateDeployment(_, newObj interface{}) { + c.enqueueAddDeployment(newObj) +} diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index 2a1a4193641..7c506f252ff 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -1300,7 +1300,7 @@ func (c *Controller) reconcileCustomVpcBfdStaticRoute(vpcName, subnetName string klog.Error(err) return err } - bfd, err := c.OVNNbClient.CreateBFD(lrpEipName, eip.Status.V4Ip, c.config.BfdMinRx, c.config.BfdMinTx, c.config.BfdDetectMult) + bfd, err := c.OVNNbClient.CreateBFD(lrpEipName, eip.Status.V4Ip, c.config.BfdMinRx, c.config.BfdMinTx, c.config.BfdDetectMult, nil) if err != nil { klog.Error(err) return err diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index e73951fac70..fbe72de33bc 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -489,7 +489,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { // add new policies for _, item := range policyRouteNeedAdd { klog.Infof("add policy route for router: %s, match %s, action %s, nexthop %s, externalID %v", c.config.ClusterRouter, item.Match, string(item.Action), item.NextHopIP, externalIDs) - if err = c.OVNNbClient.AddLogicalRouterPolicy(vpc.Name, item.Priority, item.Match, string(item.Action), []string{item.NextHopIP}, externalIDs); err != nil { + if err = c.OVNNbClient.AddLogicalRouterPolicy(vpc.Name, item.Priority, item.Match, string(item.Action), []string{item.NextHopIP}, nil, externalIDs); err != nil { klog.Errorf("add policy route to vpc %s failed, %v", vpc.Name, err) return err } @@ -573,7 +573,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { lrpEipName := fmt.Sprintf("%s-%s", key, c.config.ExternalGatewaySwitch) v4ExtGw, _ := util.SplitStringIP(externalSubnet.Spec.Gateway) // TODO: dualstack - if _, err := c.OVNNbClient.CreateBFD(lrpEipName, v4ExtGw, c.config.BfdMinRx, c.config.BfdMinTx, c.config.BfdDetectMult); err != nil { + if _, err := c.OVNNbClient.CreateBFD(lrpEipName, v4ExtGw, c.config.BfdMinRx, c.config.BfdMinTx, c.config.BfdDetectMult, nil); err != nil { klog.Error(err) return err } @@ -619,7 +619,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { if !cachedVpc.Spec.EnableBfd && cachedVpc.Status.EnableBfd { lrpEipName := fmt.Sprintf("%s-%s", key, c.config.ExternalGatewaySwitch) - if err := c.OVNNbClient.DeleteBFD(lrpEipName, ""); err != nil { + if err := c.OVNNbClient.DeleteBFDByDstIP(lrpEipName, ""); err != nil { klog.Error(err) return err } @@ -659,13 +659,12 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { return err } if vpc.Spec.BFDPort == nil || !vpc.Spec.BFDPort.Enabled { - vpc.Status.BFDPort = kubeovnv1.BFDPortStatus{Enabled: false} + vpc.Status.BFDPort = kubeovnv1.BFDPortStatus{} } else { vpc.Status.BFDPort = kubeovnv1.BFDPortStatus{ - Enabled: true, - Name: bfdPortName, - IP: vpc.Spec.BFDPort.IP, - Nodes: bfdPortNodes, + Name: bfdPortName, + IP: vpc.Spec.BFDPort.IP, + Nodes: bfdPortNodes, } } if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs(). @@ -766,7 +765,7 @@ func (c *Controller) addPolicyRouteToVpc(vpcName string, policy *kubeovnv1.Polic nextHops = strings.Split(policy.NextHopIP, ",") } - if err = c.OVNNbClient.AddLogicalRouterPolicy(vpcName, policy.Priority, policy.Match, string(policy.Action), nextHops, externalIDs); err != nil { + if err = c.OVNNbClient.AddLogicalRouterPolicy(vpcName, policy.Priority, policy.Match, string(policy.Action), nextHops, nil, externalIDs); err != nil { klog.Errorf("add policy route to vpc %s failed, %v", vpcName, err) return err } @@ -1261,7 +1260,7 @@ func (c *Controller) handleDelVpcExternalSubnet(key, subnet string) error { return err } } - if err := c.OVNNbClient.DeleteBFD(lrpName, ""); err != nil { + if err := c.OVNNbClient.DeleteBFDByDstIP(lrpName, ""); err != nil { klog.Error(err) return err } diff --git a/pkg/controller/vpc_egress_gateway.go b/pkg/controller/vpc_egress_gateway.go new file mode 100644 index 00000000000..0dc8c89d27a --- /dev/null +++ b/pkg/controller/vpc_egress_gateway.go @@ -0,0 +1,830 @@ +package controller + +import ( + "context" + "fmt" + "maps" + "net/netip" + "reflect" + "slices" + "strconv" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "k8s.io/utils/set" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (c *Controller) enqueueAddVpcEgressGateway(obj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + utilruntime.HandleError(err) + return + } + klog.V(3).Infof("enqueue add vpc-egress-gateway %s", key) + c.addOrUpdateVpcEgressGatewayQueue.Add(key) +} + +func (c *Controller) enqueueUpdateVpcEgressGateway(_, newObj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(newObj) + if err != nil { + utilruntime.HandleError(err) + return + } + klog.V(3).Infof("enqueue update vpc-egress-gateway %s", key) + c.addOrUpdateVpcEgressGatewayQueue.Add(key) +} + +func (c *Controller) enqueueDeleteVpcEgressGateway(obj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + utilruntime.HandleError(err) + return + } + klog.V(3).Infof("enqueue delete vpc-egress-gateway %s", key) + c.delVpcEgressGatewayQueue.Add(key) +} + +func (c *Controller) handleAddOrUpdateVpcEgressGateway(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.vpcEgressGatewayKeyMutex.LockKey(key) + defer func() { _ = c.vpcEgressGatewayKeyMutex.UnlockKey(key) }() + + cachedGateway, err := c.vpcEgressGatewayLister.VpcEgressGateways(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + if !cachedGateway.DeletionTimestamp.IsZero() { + c.delVpcEgressGatewayQueue.Add(key) + return nil + } + + klog.Infof("reconciling vpc-egress-gateway %s", key) + gw := cachedGateway.DeepCopy() + if gw, err = c.initVpcEgressGatewayStatus(gw); err != nil { + return err + } + + vpcName := gw.Spec.VPC + if vpcName == "" { + vpcName = c.config.ClusterRouter + } + vpc, err := c.vpcsLister.Get(vpcName) + if err != nil { + klog.Error(err) + return err + } + if gw.Spec.BFD.Enabled && vpc.Status.BFDPort.IP == "" { + err = fmt.Errorf("vpc %s bfd port is not enabled or not ready", vpc.Name) + klog.Error(err) + gw.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "VpcBfdPortNotEnabled", err.Error(), gw.Generation) + _, _ = c.updateVpcEgressGatewayStatus(gw) + return err + } + + if controllerutil.AddFinalizer(gw, util.KubeOVNControllerFinalizer) { + updatedGateway, err := c.config.KubeOvnClient.KubeovnV1().VpcEgressGateways(gw.Namespace). + Update(context.Background(), gw, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to add finalizer for vpc-egress-gateway %s/%s: %w", gw.Namespace, gw.Name, err) + klog.Error(err) + return err + } + gw = updatedGateway + } + + var bfdIP, bfdIPv4, bfdIPv6 string + if gw.Spec.BFD.Enabled { + bfdIP = vpc.Status.BFDPort.IP + bfdIPv4, bfdIPv6 = util.SplitStringIP(bfdIP) + } + + attachmentNetworkName, ipv4Src, ipv6Src, deploy, err := c.reconcileVpcEgressGatewayWorkload(gw, vpc, bfdIP, bfdIPv4, bfdIPv6) + if err != nil { + klog.Error(err) + gw.Status.Conditions.SetCondition(kubeovnv1.Ready, corev1.ConditionFalse, "ReconcileWorkloadFailed", err.Error(), gw.Generation) + _, _ = c.updateVpcEgressGatewayStatus(gw) + return err + } + + var podIPs []string + gw.Status.InternalIPs = nil + gw.Status.ExternalIPs = nil + gw.Status.Workload.APIVersion = deploy.APIVersion + gw.Status.Workload.Kind = deploy.Kind + gw.Status.Workload.Name = deploy.Name + gw.Status.Workload.Nodes = nil + if !util.DeploymentIsReady(deploy) { + gw.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + gw.Status.Conditions.SetCondition(kubeovnv1.Ready, corev1.ConditionFalse, "Processing", msg, gw.Generation) + } else { + podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) + if err != nil { + err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return err + } + + pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) + if err != nil { + err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return err + } + + gw.Status.Workload.Nodes = make([]string, 0, len(pods)) + for _, pod := range pods { + gw.Status.Workload.Nodes = append(gw.Status.Workload.Nodes, pod.Spec.NodeName) + ips := util.PodIPs(*pod) + podIPs = append(podIPs, ips...) + gw.Status.InternalIPs = append(gw.Status.InternalIPs, strings.Join(ips, ",")) + extIPs, err := util.PodAttachmentIPs(pod, attachmentNetworkName) + if err != nil { + klog.Error(err) + gw.Status.ExternalIPs = append(gw.Status.ExternalIPs, "") + continue + } + gw.Status.ExternalIPs = append(gw.Status.ExternalIPs, strings.Join(extIPs, ",")) + } + } + if gw, err = c.updateVpcEgressGatewayStatus(gw); err != nil { + klog.Error(err) + return err + } + if len(gw.Status.Workload.Nodes) == 0 { + return nil + } + + nextHopsIPv4, hextHopsIPv6 := util.SplitIpsByProtocol(podIPs) + if err = c.reconcileVpcEgressGatewayOVNRoutes(gw, 4, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv4, set.New(nextHopsIPv4...), ipv4Src); err != nil { + klog.Error(err) + return err + } + if err = c.reconcileVpcEgressGatewayOVNRoutes(gw, 6, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv6, set.New(hextHopsIPv6...), ipv6Src); err != nil { + klog.Error(err) + return err + } + + gw.Status.Ready = true + gw.Status.Phase = kubeovnv1.PhaseCompleted + gw.Status.Conditions.SetReady("ReconcileSuccess", gw.Generation) + if _, err = c.updateVpcEgressGatewayStatus(gw); err != nil { + return err + } + + return nil +} + +func (c *Controller) initVpcEgressGatewayStatus(gw *kubeovnv1.VpcEgressGateway) (*kubeovnv1.VpcEgressGateway, error) { + var err error + if gw.Status.Phase == "" || gw.Status.Phase == kubeovnv1.PhasePending { + gw.Status.Phase = kubeovnv1.PhaseProcessing + gw, err = c.updateVpcEgressGatewayStatus(gw) + } + return gw, err +} + +func (c *Controller) updateVpcEgressGatewayStatus(gw *kubeovnv1.VpcEgressGateway) (*kubeovnv1.VpcEgressGateway, error) { + if len(gw.Status.Conditions) == 0 { + gw.Status.Conditions.SetCondition(kubeovnv1.Init, corev1.ConditionUnknown, "Processing", "", gw.Generation) + } + if !gw.Status.Ready { + gw.Status.Phase = kubeovnv1.PhaseProcessing + } + + updateGateway, err := c.config.KubeOvnClient.KubeovnV1().VpcEgressGateways(gw.Namespace). + UpdateStatus(context.Background(), gw, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to update status of vpc-egress-gateway %s/%s: %w", gw.Namespace, gw.Name, err) + klog.Error(err) + return nil, err + } + + return updateGateway, nil +} + +func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGateway, vpc *kubeovnv1.Vpc, bfdIP, bfdIPv4, bfdIPv6 string) (string, set.Set[string], set.Set[string], *appsv1.Deployment, error) { + image := c.config.Image + if gw.Spec.Image != "" { + image = gw.Spec.Image + } + if image == "" { + err := fmt.Errorf("no image specified for vpc egress gateway %s/%s", gw.Namespace, gw.Name) + klog.Error(err) + return "", nil, nil, nil, err + } + + if len(gw.Spec.InternalIPs) != 0 && len(gw.Spec.InternalIPs) < int(gw.Spec.Replicas) { + err := fmt.Errorf("internal IPs count %d is less than replicas %d", len(gw.Spec.InternalIPs), gw.Spec.Replicas) + klog.Error(err) + return "", nil, nil, nil, err + } + if len(gw.Spec.ExternalIPs) != 0 && len(gw.Spec.ExternalIPs) < int(gw.Spec.Replicas) { + err := fmt.Errorf("external IPs count %d is less than replicas %d", len(gw.Spec.ExternalIPs), gw.Spec.Replicas) + klog.Error(err) + return "", nil, nil, nil, err + } + + internalSubnet := gw.Spec.InternalSubnet + if internalSubnet == "" { + internalSubnet = vpc.Status.DefaultLogicalSwitch + } + if internalSubnet == "" { + err := fmt.Errorf("default subnet of vpc %s not found, please set internal subnet of the egress gateway", vpc.Name) + klog.Error(err) + return "", nil, nil, nil, err + } + intSubnet, err := c.subnetsLister.Get(internalSubnet) + if err != nil { + klog.Error(err) + return "", nil, nil, nil, err + } + extSubnet, err := c.subnetsLister.Get(gw.Spec.ExternalSubnet) + if err != nil { + klog.Error(err) + return "", nil, nil, nil, err + } + if !strings.ContainsRune(extSubnet.Spec.Provider, '.') { + err = fmt.Errorf("please set correct provider of subnet %s to get the network-attachment-definition", extSubnet.Name) + klog.Error(err) + return "", nil, nil, nil, err + } + subStrings := strings.Split(extSubnet.Spec.Provider, ".") + nadName, nadNamespace := subStrings[0], subStrings[1] + if _, err = c.config.AttachNetClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(nadNamespace). + Get(context.Background(), nadName, metav1.GetOptions{}); err != nil { + klog.Errorf("failed to get net-attach-def %s/%s: %v", nadNamespace, nadName, err) + return "", nil, nil, nil, err + } + attachmentNetworkName := fmt.Sprintf("%s/%s", nadNamespace, nadName) + + routes := util.NewPodRoutes() + intCIDRIPv4, intCIDRIPv6 := util.SplitStringIP(intSubnet.Spec.CIDRBlock) + intGatewayIPv4, intGatewayIPv6 := util.SplitStringIP(intSubnet.Spec.Gateway) + extGatewayIPv4, extGatewayIPv6 := util.SplitStringIP(extSubnet.Spec.Gateway) + routes.Add(util.OvnProvider, bfdIPv4, intGatewayIPv4) + routes.Add(util.OvnProvider, bfdIPv6, intGatewayIPv6) + routes.Add(extSubnet.Spec.Provider, "0.0.0.0/0", extGatewayIPv4) + routes.Add(extSubnet.Spec.Provider, "::/0", extGatewayIPv6) + + annotations, err := routes.ToAnnotations() + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + annotations[util.AttachmentNetworkAnnotation] = attachmentNetworkName + annotations[util.LogicalSwitchAnnotation] = intSubnet.Name + if len(gw.Spec.InternalIPs) != 0 { + annotations[util.IPPoolAnnotation] = strings.Join(gw.Spec.InternalIPs, ";") + } + if len(gw.Spec.ExternalIPs) != 0 { + annotations[fmt.Sprintf(util.IPPoolAnnotationTemplate, extSubnet.Spec.Provider)] = strings.Join(gw.Spec.ExternalIPs, ";") + } + + ipv4ForwardSrc, ipv6ForwardSrc := set.New[string](), set.New[string]() + ipv4SNATSrc, ipv6SNATSrc := set.New[string](), set.New[string]() + for _, policy := range gw.Spec.Policies { + ipv4, ipv6 := util.SplitIpsByProtocol(policy.IPBlocks) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4...) + ipv6SNATSrc.Insert(ipv6...) + } else { + ipv4ForwardSrc.Insert(ipv4...) + ipv6ForwardSrc.Insert(ipv6...) + } + for _, subnetName := range policy.Subnets { + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + if subnet.Status.IsNotValidated() { + err = fmt.Errorf("subnet %s is not validated", subnet.Name) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + // TODO: check subnet's vpc and vlan + ipv4, ipv6 := util.SplitStringIP(subnet.Spec.CIDRBlock) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4) + ipv6SNATSrc.Insert(ipv6) + } else { + ipv4ForwardSrc.Insert(ipv4) + ipv6ForwardSrc.Insert(ipv6) + } + } + } + + intRouteDstIPv4, intRouteDstIPv6 := ipv4ForwardSrc.Union(ipv4SNATSrc), ipv6ForwardSrc.Union(ipv6SNATSrc) + intRouteDstIPv4.Insert(bfdIPv4) + intRouteDstIPv6.Insert(bfdIPv6) + intRouteDstIPv4.Delete("") + intRouteDstIPv6.Delete("") + ipv4SNATSrc.Delete("") + ipv6SNATSrc.Delete("") + + initEnv, err := vpcEgressGatewayInitContainerEnv(4, intGatewayIPv4, intCIDRIPv4, intRouteDstIPv4, ipv4SNATSrc) + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + ipv6Env, err := vpcEgressGatewayInitContainerEnv(6, intGatewayIPv6, intCIDRIPv6, intRouteDstIPv6, ipv6SNATSrc) + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + initEnv = append(initEnv, ipv6Env...) + + // generate workload + labels := map[string]string{util.VpcEgressGatewayAnnotation: gw.Name} + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: gw.Spec.Prefix + gw.Name, + Namespace: gw.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: ptr.To(intstr.FromInt(1)), + MaxSurge: ptr.To(intstr.FromInt(1)), + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: mergeNodeSelector(gw.Spec.NodeSelector), + }, + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: "kubernetes.io/hostname", + }}, + }, + }, + InitContainers: []corev1.Container{{ + Name: "init", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"bash", "/kube-ovn/init-vpc-egress-gateway.sh"}, + Env: initEnv, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + }}, + Containers: []corev1.Container{{ + Name: "gateway", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sleep", "infinity"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](65534), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "usr-local-sbin", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }}, + TerminationGracePeriodSeconds: ptr.To[int64](0), + }, + }, + }, + } + if err = util.SetOwnerReference(gw, deploy); err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + + if bfdIP != "" { + container := vpcEgressGatewayContainerBFDD(image, bfdIP, gw.Spec.BFD.MinTX, gw.Spec.BFD.MinRX, gw.Spec.BFD.Multiplier) + deploy.Spec.Template.Spec.Containers[0] = container + } + + hash, err := util.Sha256HashObject(deploy) + if err != nil { + err = fmt.Errorf("failed to hash generated deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + + hash = hash[:12] + deploy.Spec.Replicas = ptr.To(gw.Spec.Replicas) + deploy.Annotations = map[string]string{util.GenerateHashAnnotation: hash} + if currentDeploy, err := c.deploymentsLister.Deployments(gw.Namespace).Get(deploy.Name); err != nil { + if !k8serrors.IsNotFound(err) { + err = fmt.Errorf("failed to get deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + if deploy, err = c.config.KubeClient.AppsV1().Deployments(gw.Namespace). + Create(context.Background(), deploy, metav1.CreateOptions{}); err != nil { + err = fmt.Errorf("failed to create deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + } else if !reflect.DeepEqual(currentDeploy.Spec.Replicas, deploy.Spec.Replicas) || + currentDeploy.Annotations[util.GenerateHashAnnotation] != hash { + if deploy, err = c.config.KubeClient.AppsV1().Deployments(gw.Namespace). + Update(context.Background(), deploy, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to update deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + } else { + deploy = currentDeploy + } + + intRouteDstIPv4.Delete(bfdIPv4) + intRouteDstIPv6.Delete(bfdIPv6) + deploy.APIVersion, deploy.Kind = deploymentGroupVersion, deploymentKind + + return attachmentNetworkName, intRouteDstIPv4, intRouteDstIPv6, deploy, nil +} + +func (c *Controller) reconcileVpcEgressGatewayOVNRoutes(gw *kubeovnv1.VpcEgressGateway, af int, lrName, lrpName, bfdIP string, nextHops, sources set.Set[string]) error { + externalIDs := map[string]string{ + ovs.ExternalIDVendor: util.CniTypeName, + ovs.ExternalIDVpcEgressGateway: fmt.Sprintf("%s/%s", gw.Namespace, gw.Name), + "af": strconv.Itoa(af), + } + bfdList, err := c.OVNNbClient.FindBFD(externalIDs) + if err != nil { + klog.Error(err) + return err + } + + bfdIDs := set.New[string]() + bfdDstIPs := nextHops.Clone() + bfdMap := make(map[string]*string, nextHops.Len()) + for _, bfd := range bfdList { + if bfdIP == "" || bfd.LogicalPort != lrpName || !bfdDstIPs.Has(bfd.DstIP) { + if err = c.OVNNbClient.DeleteBFD(bfd.UUID); err != nil { + err = fmt.Errorf("failed to delete bfd %s: %w", bfd.UUID, err) + klog.Error(err) + return err + } + } + if bfdIP == "" || bfd.LogicalPort == lrpName && bfdDstIPs.Has(bfd.DstIP) { + // TODO: update min_rx, min_tx and multiplier + if bfdIP != "" { + bfdIDs.Insert(bfd.UUID) + bfdMap[bfd.DstIP] = ptr.To(bfd.UUID) + } + bfdDstIPs.Delete(bfd.DstIP) + } + } + if bfdIP != "" { + for _, dstIP := range bfdDstIPs.UnsortedList() { + bfd, err := c.OVNNbClient.CreateBFD(lrpName, dstIP, int(gw.Spec.BFD.MinRX), int(gw.Spec.BFD.MinTX), int(gw.Spec.BFD.Multiplier), externalIDs) + if err != nil { + klog.Error(err) + return err + } + bfdIDs.Insert(bfd.UUID) + bfdMap[bfd.DstIP] = ptr.To(bfd.UUID) + } + } + + if lrName == c.config.ClusterRouter { + // reconcile LR policy + policies, err := c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayPolicyPriority, externalIDs, false) + if err != nil { + klog.Error(err) + return err + } + matches := set.New[string]() + for _, src := range sources.UnsortedList() { + matches.Insert(fmt.Sprintf("ip%d.src == %s", af, src)) + } + for _, policy := range policies { + if matches.Has(policy.Match) { + if !nextHops.Equal(set.New(policy.Nexthops...)) || !bfdIDs.Equal(set.New(policy.BFDSessions...)) { + policy.Nexthops, policy.BFDSessions = nextHops.UnsortedList(), bfdIDs.UnsortedList() + if err = c.OVNNbClient.UpdateLogicalRouterPolicy(policy, &policy.Nexthops, &policy.BFDSessions); err != nil { + err = fmt.Errorf("failed to update bfd sessions of logical router policy %s: %w", policy.UUID, err) + klog.Error(err) + return err + } + } + matches.Delete(policy.Match) + continue + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { + err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) + klog.Error(err) + return err + } + } + bfdSessions := bfdIDs.UnsortedList() + for _, match := range matches.UnsortedList() { + if err := c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayPolicyPriority, match, + ovnnb.LogicalRouterPolicyActionReroute, nextHops.UnsortedList(), bfdSessions, externalIDs); err != nil { + klog.Error(err) + return err + } + } + } else { + // reconcile LR static route + routes, err := c.OVNNbClient.ListLogicalRouterStaticRoutes(lrName, nil, nil, "", externalIDs) + if err != nil { + klog.Error(err) + return err + } + expected := make(map[string]set.Set[string], nextHops.Len()) + for _, nextHop := range nextHops.UnsortedList() { + expected[nextHop] = sources.Clone() + } + for _, route := range routes { + if expected[route.Nexthop].Has(route.IPPrefix) { + if !reflect.DeepEqual(route.BFD, bfdMap[route.Nexthop]) { + route.BFD = bfdMap[route.Nexthop] + if err = c.OVNNbClient.UpdateLogicalRouterStaticRoute(route, &route.BFD); err != nil { + klog.Error(err) + return err + } + } + expected[route.Nexthop].Delete(route.IPPrefix) + continue + } + if err = c.OVNNbClient.DeleteLogicalRouterStaticRouteByUUID(lrName, route.UUID); err != nil { + klog.Error(err) + return err + } + } + for nextHop, sources := range expected { + for _, src := range sources.UnsortedList() { + if err = c.OVNNbClient.AddLogicalRouterStaticRoute(lrName, util.MainRouteTable, ovnnb.LogicalRouterStaticRoutePolicySrcIP, src, bfdMap[nextHop], nextHop); err != nil { + klog.Error(err) + return err + } + } + } + } + + return nil +} + +func mergeNodeSelector(nodeSelector []kubeovnv1.VpcEgressGatewayNodeSelector) *corev1.NodeSelector { + if len(nodeSelector) == 0 { + return nil + } + + result := &corev1.NodeSelector{ + NodeSelectorTerms: make([]corev1.NodeSelectorTerm, len(nodeSelector)), + } + for i, selector := range nodeSelector { + result.NodeSelectorTerms[i] = corev1.NodeSelectorTerm{ + MatchExpressions: make([]corev1.NodeSelectorRequirement, len(selector.MatchExpressions), len(selector.MatchLabels)+len(selector.MatchExpressions)), + MatchFields: make([]corev1.NodeSelectorRequirement, len(selector.MatchFields)), + } + for j := range selector.MatchExpressions { + selector.MatchExpressions[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchExpressions[j]) + } + for _, key := range slices.Sorted(maps.Keys(selector.MatchLabels)) { + result.NodeSelectorTerms[i].MatchExpressions = append(result.NodeSelectorTerms[i].MatchExpressions, corev1.NodeSelectorRequirement{ + Key: key, + Operator: corev1.NodeSelectorOpIn, + Values: []string{selector.MatchLabels[key]}, + }) + } + for j := range selector.MatchFields { + selector.MatchFields[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchFields[j]) + } + } + + return result +} + +func vpcEgressGatewayInitContainerEnv(af int, internalGateway, internalCIDR string, routeDst, snatSrc set.Set[string]) ([]corev1.EnvVar, error) { + if internalGateway == "" { + return nil, nil + } + + cidr, err := netip.ParsePrefix(internalCIDR) + if err != nil { + err = fmt.Errorf("failed to parse internal CIDR %s: %w", internalCIDR, err) + klog.Error(err) + return nil, err + } + + dstList := make([]string, 0, routeDst.Len()) + for _, dst := range routeDst.SortedList() { + if !strings.ContainsRune(dst, '/') { + if af == 4 { + dst += "/32" + } else { + dst += "/128" + } + } + prefix, err := netip.ParsePrefix(dst) + if err != nil { + err = fmt.Errorf("failed to parse internal route destination %s: %w", dst, err) + klog.Error(err) + return nil, err + } + if !cidr.Overlaps(prefix) { + dstList = append(dstList, prefix.Masked().String()) + } + } + + return []corev1.EnvVar{{ + Name: fmt.Sprintf("INTERNAL_GATEWAY_IPV%d", af), + Value: internalGateway, + }, { + Name: fmt.Sprintf("INTERNAL_ROUTE_DST_IPV%d", af), + Value: strings.Join(dstList, ","), + }, { + Name: fmt.Sprintf("SNAT_SOURCES_IPV%d", af), + Value: strings.Join(snatSrc.SortedList(), ","), + }}, nil +} + +func vpcEgressGatewayContainerBFDD(image, bfdIP string, minTX, minRX, multiplier int32) corev1.Container { + return corev1.Container{ + Name: "bfdd", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"bash", "/kube-ovn/start-bfdd.sh"}, + Env: []corev1.EnvVar{{ + Name: "POD_IPS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIPs", + }, + }, + }, { + Name: "BFD_PEER_IPS", + Value: bfdIP, + }, { + Name: "BFD_MIN_TX", + Value: strconv.Itoa(int(minTX)), + }, { + Name: "BFD_MIN_RX", + Value: strconv.Itoa(int(minRX)), + }, { + Name: "BFD_MULTI", + Value: strconv.Itoa(int(multiplier)), + }}, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bash", "/kube-ovn/bfdd-prestart.sh"}, + }, + }, + InitialDelaySeconds: 1, + FailureThreshold: 1, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bfdd-control", "status"}, + }, + }, + InitialDelaySeconds: 1, + PeriodSeconds: 5, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bfdd-control", "status"}, + }, + }, + InitialDelaySeconds: 3, + PeriodSeconds: 3, + FailureThreshold: 1, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](65534), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_BIND_SERVICE", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + } +} + +func (c *Controller) handleDelVpcEgressGateway(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.vpcEgressGatewayKeyMutex.LockKey(key) + defer func() { _ = c.vpcEgressGatewayKeyMutex.UnlockKey(key) }() + + cachedGateway, err := c.vpcEgressGatewayLister.VpcEgressGateways(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + err = fmt.Errorf("failed to get vpc-egress-gateway %s: %w", key, err) + klog.Error(err) + return err + } + return nil + } + + klog.Infof("handle deleting vpc-egress-gateway %s", key) + if err = c.cleanOVNforVpcEgressGateway(key, cachedGateway.Spec.VPC); err != nil { + klog.Error(err) + return err + } + + gw := cachedGateway.DeepCopy() + if controllerutil.RemoveFinalizer(gw, util.KubeOVNControllerFinalizer) { + if _, err = c.config.KubeOvnClient.KubeovnV1().VpcEgressGateways(gw.Namespace). + Update(context.Background(), gw, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to remove finalizer from vpc-egress-gateway %s: %w", key, err) + klog.Error(err) + } + } + + return nil +} + +func (c *Controller) cleanOVNforVpcEgressGateway(key, lrName string) error { + externalIDs := map[string]string{ + ovs.ExternalIDVendor: util.CniTypeName, + ovs.ExternalIDVpcEgressGateway: key, + } + + bfdList, err := c.OVNNbClient.FindBFD(externalIDs) + if err != nil { + klog.Error(err) + return err + } + for _, bfd := range bfdList { + if err = c.OVNNbClient.DeleteBFD(bfd.UUID); err != nil { + klog.Error(err) + return err + } + } + + if lrName == "" { + lrName = c.config.ClusterRouter + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, -1, externalIDs); err != nil { + klog.Error(err) + return err + } + if err = c.OVNNbClient.DeleteLogicalRouterStaticRouteByExternalIDs(lrName, externalIDs); err != nil { + klog.Error(err) + return err + } + + return nil +} diff --git a/pkg/ovn_ic_controller/ovn_ic_controller.go b/pkg/ovn_ic_controller/ovn_ic_controller.go index 3757862ef77..d83784e7224 100644 --- a/pkg/ovn_ic_controller/ovn_ic_controller.go +++ b/pkg/ovn_ic_controller/ovn_ic_controller.go @@ -612,7 +612,7 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) { match = util.MatchV6Dst + " == " + prefix } - if err = c.OVNNbClient.AddLogicalRouterPolicy(lr.Name, util.OvnICPolicyPriority, match, ovnnb.LogicalRouterPolicyActionAllow, nil, map[string]string{key: value, "vendor": util.CniTypeName}); err != nil { + if err = c.OVNNbClient.AddLogicalRouterPolicy(lr.Name, util.OvnICPolicyPriority, match, ovnnb.LogicalRouterPolicyActionAllow, nil, nil, map[string]string{key: value, "vendor": util.CniTypeName}); err != nil { klog.Errorf("failed to add router policy: %v", err) } diff --git a/pkg/ovs/interface.go b/pkg/ovs/interface.go index dc3d786235d..67a230b9a16 100644 --- a/pkg/ovs/interface.go +++ b/pkg/ovs/interface.go @@ -62,11 +62,13 @@ type GatewayChassis interface { } type BFD interface { - CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int) (*ovnnb.BFD, error) - DeleteBFD(lrpName, dstIP string) error + CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int, externalIDs map[string]string) (*ovnnb.BFD, error) + DeleteBFD(uuid string) error + DeleteBFDByDstIP(lrpName, dstIP string) error ListBFDs(lrpName, dstIP string) ([]ovnnb.BFD, error) ListDownBFDs(dstIP string) ([]ovnnb.BFD, error) ListUpBFDs(dstIP string) ([]ovnnb.BFD, error) + FindBFD(externalIDs map[string]string) ([]ovnnb.BFD, error) UpdateBFD(bfd *ovnnb.BFD, fields ...interface{}) error MonitorBFD() } @@ -175,15 +177,18 @@ type AddressSet interface { type LogicalRouterStaticRoute interface { AddLogicalRouterStaticRoute(lrName, routeTable, policy, ipPrefix string, bfdID *string, nexthops ...string) error + UpdateLogicalRouterStaticRoute(route *ovnnb.LogicalRouterStaticRoute, fields ...interface{}) error ClearLogicalRouterStaticRoute(lrName string) error DeleteLogicalRouterStaticRoute(lrName string, routeTable, policy *string, ipPrefix, nextHop string) error + DeleteLogicalRouterStaticRouteByUUID(lrName, uuid string) error + DeleteLogicalRouterStaticRouteByExternalIDs(lrName string, externalIDs map[string]string) error ListLogicalRouterStaticRoutesByOption(lrName, routeTable, key, value string) ([]*ovnnb.LogicalRouterStaticRoute, error) ListLogicalRouterStaticRoutes(lrName string, routeTable, policy *string, ipPrefix string, externalIDs map[string]string) ([]*ovnnb.LogicalRouterStaticRoute, error) LogicalRouterStaticRouteExists(lrName, routeTable, policy, ipPrefix, nexthop string) (bool, error) } type LogicalRouterPolicy interface { - AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops []string, externalIDs map[string]string) error + AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops, bfdSessions []string, externalIDs map[string]string) error DeleteLogicalRouterPolicy(lrName string, priority int, match string) error DeleteLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string) error DeleteLogicalRouterPolicyByUUID(lrName, uuid string) error @@ -192,6 +197,7 @@ type LogicalRouterPolicy interface { ListLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string, ignoreExtIDEmptyValue bool) ([]*ovnnb.LogicalRouterPolicy, error) GetLogicalRouterPolicy(lrName string, priority int, match string, ignoreNotFound bool) ([]*ovnnb.LogicalRouterPolicy, error) GetLogicalRouterPoliciesByExtID(lrName, key, value string) ([]*ovnnb.LogicalRouterPolicy, error) + UpdateLogicalRouterPolicy(policy *ovnnb.LogicalRouterPolicy, fields ...interface{}) error } type NAT interface { diff --git a/pkg/ovs/ovn-nb-bfd.go b/pkg/ovs/ovn-nb-bfd.go index 2ea3fa5d910..31e3396b1e5 100644 --- a/pkg/ovs/ovn-nb-bfd.go +++ b/pkg/ovs/ovn-nb-bfd.go @@ -5,13 +5,12 @@ import ( "fmt" "time" + "github.com/ovn-org/libovsdb/cache" + "github.com/ovn-org/libovsdb/model" "k8s.io/klog/v2" "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" - - "github.com/ovn-org/libovsdb/cache" - "github.com/ovn-org/libovsdb/model" ) func (c *OVNNbClient) ListBFDs(lrpName, dstIP string) ([]ovnnb.BFD, error) { @@ -68,7 +67,7 @@ func (c *OVNNbClient) ListUpBFDs(dstIP string) ([]ovnnb.BFD, error) { return bfdList, nil } -func (c *OVNNbClient) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int) (*ovnnb.BFD, error) { +func (c *OVNNbClient) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int, externalIDs map[string]string) (*ovnnb.BFD, error) { bfdList, err := c.ListBFDs(lrpName, dstIP) if err != nil { klog.Error(err) @@ -84,6 +83,7 @@ func (c *OVNNbClient) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult MinRx: &minRx, MinTx: &minTx, DetectMult: &detectMult, + ExternalIDs: externalIDs, } ops, err := c.Create(bfd) if err != nil { @@ -124,7 +124,22 @@ func (c *OVNNbClient) UpdateBFD(bfd *ovnnb.BFD, fields ...interface{}) error { return nil } -func (c *OVNNbClient) DeleteBFD(lrpName, dstIP string) error { +func (c *OVNNbClient) DeleteBFD(uuid string) error { + ops, err := c.Where(&ovnnb.BFD{UUID: uuid}).Delete() + if err != nil { + err := fmt.Errorf("failed to generate operations for BFD deletion with UUID %s: %w", uuid, err) + klog.Error(err) + return err + } + if err = c.Transact("bfd-del", ops); err != nil { + err = fmt.Errorf("failed to delete BFD with with UUID %s: %w", uuid, err) + klog.Error(err) + return err + } + return nil +} + +func (c *OVNNbClient) DeleteBFDByDstIP(lrpName, dstIP string) error { bfdList, err := c.ListBFDs(lrpName, dstIP) if err != nil { klog.Error(err) @@ -199,6 +214,10 @@ func (c *OVNNbClient) bfdAddL3HAHandler(table string, model model.Model) { } bfd := model.(*ovnnb.BFD) + if bfd.ExternalIDs[ExternalIDVpcEgressGateway] != "" { + return + } + klog.Infof("lrp %s add BFD to dst ip %s", bfd.LogicalPort, bfd.DstIP) needRecheck := false if bfd.Status == nil { @@ -230,6 +249,9 @@ func (c *OVNNbClient) bfdUpdateL3HAHandler(table string, oldModel, newModel mode oldBfd := oldModel.(*ovnnb.BFD) newBfd := newModel.(*ovnnb.BFD) + if newBfd.ExternalIDs[ExternalIDVpcEgressGateway] != "" { + return + } if oldBfd.Status == nil || newBfd.Status == nil { return } @@ -343,5 +365,32 @@ func (c *OVNNbClient) bfdDelL3HAHandler(table string, model model.Model) { return } bfd := model.(*ovnnb.BFD) + if bfd.ExternalIDs[ExternalIDVpcEgressGateway] != "" { + return + } klog.Infof("lrp %s del BFD to dst ip %s", bfd.LogicalPort, bfd.DstIP) } + +func (c *OVNNbClient) FindBFD(externalIDs map[string]string) ([]ovnnb.BFD, error) { + ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) + defer cancel() + + bfdList := make([]ovnnb.BFD, 0) + if err := c.ovsDbClient.WhereCache(func(bfd *ovnnb.BFD) bool { + if len(bfd.ExternalIDs) == 0 && len(externalIDs) != 0 { + return false + } + for k, v := range externalIDs { + if bfd.ExternalIDs[k] != v { + return false + } + } + return true + }).List(ctx, &bfdList); err != nil { + err := fmt.Errorf("failed to find ovn BFD: %w", err) + klog.Error(err) + return nil, err + } + + return bfdList, nil +} diff --git a/pkg/ovs/ovn-nb-bfd_test.go b/pkg/ovs/ovn-nb-bfd_test.go index d63e34912a0..1642a1e0237 100644 --- a/pkg/ovs/ovn-nb-bfd_test.go +++ b/pkg/ovs/ovn-nb-bfd_test.go @@ -22,7 +22,7 @@ func (suite *OvnClientTestSuite) testCreateBFD() { lrpName := "test-create-bfd" - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) require.Equal(t, lrpName, bfd.LogicalPort) @@ -40,11 +40,11 @@ func (suite *OvnClientTestSuite) testCreateBFD() { lrpName := "test-create-existing-bfd" - bfd1, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd1, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd1) - bfd2, err := nbClient.CreateBFD(lrpName, dstIP, minRx+1, minTx+1, detectMult+1) + bfd2, err := nbClient.CreateBFD(lrpName, dstIP, minRx+1, minTx+1, detectMult+1, nil) require.NoError(t, err) require.NotNil(t, bfd2) require.Equal(t, bfd1, bfd2) @@ -71,11 +71,11 @@ func (suite *OvnClientTestSuite) testListBFD() { t.Run("list BFDs", func(t *testing.T) { t.Parallel() - bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1) + bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1, nil) require.NoError(t, err) require.NotNil(t, bfd1) - bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx2, minTx2, detectMult2) + bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx2, minTx2, detectMult2, nil) require.NoError(t, err) require.NotNil(t, bfd2) @@ -101,7 +101,7 @@ func (suite *OvnClientTestSuite) testListBFD() { t.Run("closed server list failed BFDs", func(t *testing.T) { t.Parallel() - failedBFD1, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1) + failedBFD1, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1, nil) require.Error(t, err) require.Nil(t, failedBFD1) // cache db should be empty @@ -123,14 +123,14 @@ func (suite *OvnClientTestSuite) testDeleteBFD() { minRx1, minTx1, detectMult1 := 101, 102, 19 minRx2, minTx2, detectMult2 := 201, 202, 29 - _, err := nbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1) + _, err := nbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1, nil) require.NoError(t, err) - bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx2, minTx2, detectMult2) + bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx2, minTx2, detectMult2, nil) require.NoError(t, err) t.Run("delete BFD", func(t *testing.T) { - err = nbClient.DeleteBFD(lrpName, dstIP1) + err = nbClient.DeleteBFDByDstIP(lrpName, dstIP1) require.NoError(t, err) bfdList, err := nbClient.ListBFDs(lrpName, dstIP1) @@ -144,7 +144,7 @@ func (suite *OvnClientTestSuite) testDeleteBFD() { }) t.Run("delete multiple BFDs", func(t *testing.T) { - err = nbClient.DeleteBFD(lrpName, "") + err = nbClient.DeleteBFDByDstIP(lrpName, "") require.NoError(t, err) bfdList, err := nbClient.ListBFDs(lrpName, "") @@ -155,16 +155,16 @@ func (suite *OvnClientTestSuite) testDeleteBFD() { t.Run("delete non-existent BFD", func(t *testing.T) { t.Parallel() - err := nbClient.DeleteBFD(lrpName, "192.168.124.17") + err := nbClient.DeleteBFDByDstIP(lrpName, "192.168.124.17") require.NoError(t, err) }) t.Run("closed server delete non-existent BFD", func(t *testing.T) { t.Parallel() - _, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1) + _, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1, nil) require.Error(t, err) - err = failedNbClient.DeleteBFD(lrpName, "192.168.124.17") + err = failedNbClient.DeleteBFDByDstIP(lrpName, "192.168.124.17") // cache db should be empty require.NoError(t, err) }) @@ -185,19 +185,19 @@ func (suite *OvnClientTestSuite) testListDownBFDs() { t.Run("list down BFDs", func(t *testing.T) { t.Parallel() - bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult) + bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd1) // closed server create failed BFD - failedBFD1, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult) + failedBFD1, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult, nil) require.Error(t, err) require.Nil(t, failedBFD1) - bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx, minTx, detectMult) + bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd2) - bfd3, err := nbClient.CreateBFD(lrpName, dstIP3, minRx, minTx, detectMult) + bfd3, err := nbClient.CreateBFD(lrpName, dstIP3, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd3) @@ -247,7 +247,7 @@ func (suite *OvnClientTestSuite) testListDownBFDs() { t.Parallel() // Create a BFD with UP status - bfd, err := nbClient.CreateBFD(lrpName, "192.168.124.10", minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, "192.168.124.10", minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -277,15 +277,15 @@ func (suite *OvnClientTestSuite) testListUpBFDs() { t.Run("list up BFDs", func(t *testing.T) { t.Parallel() - bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult) + bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd1) - bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx, minTx, detectMult) + bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd2) - bfd3, err := nbClient.CreateBFD(lrpName, dstIP3, minRx, minTx, detectMult) + bfd3, err := nbClient.CreateBFD(lrpName, dstIP3, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd3) @@ -340,7 +340,7 @@ func (suite *OvnClientTestSuite) testIsLrpBfdUp() { t.Parallel() lrpName := "test-is-lrp-bfd-up" - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -358,7 +358,7 @@ func (suite *OvnClientTestSuite) testIsLrpBfdUp() { t.Parallel() lrpName := "test-is-lrp-bfd-down" - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -377,7 +377,7 @@ func (suite *OvnClientTestSuite) testIsLrpBfdUp() { t.Parallel() lrpName := "test-is-lrp-bfd-status-nil" - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -414,7 +414,7 @@ func (suite *OvnClientTestSuite) testBfdAddL3HAHandler() { dstIP := "192.168.124.19" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -436,7 +436,7 @@ func (suite *OvnClientTestSuite) testBfdAddL3HAHandler() { dstIP := "192.168.124.20" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -460,7 +460,7 @@ func (suite *OvnClientTestSuite) testBfdAddL3HAHandler() { dstIP := "192.168.124.21" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -484,7 +484,7 @@ func (suite *OvnClientTestSuite) testBfdAddL3HAHandler() { dstIP := "192.168.124.22" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -510,7 +510,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.26" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -533,7 +533,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.27" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) bfd.Status = nil @@ -558,7 +558,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.28" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) upStatus := ovnnb.BFDStatusUp @@ -579,7 +579,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.23" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -602,7 +602,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.24" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -625,7 +625,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.25" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -648,7 +648,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.28" minRx, minTx, detectMult := 101, 102, 19 - failedBFD, err := failedNbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + failedBFD, err := failedNbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.Error(t, err) require.Nil(t, failedBFD) newBfd := &ovnnb.BFD{ @@ -677,7 +677,7 @@ func (suite *OvnClientTestSuite) testBfdDelL3HAHandler() { dstIP := "192.168.124.30" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -696,7 +696,7 @@ func (suite *OvnClientTestSuite) testBfdDelL3HAHandler() { dstIP := "192.168.124.31" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) diff --git a/pkg/ovs/ovn-nb-logical_router_policy.go b/pkg/ovs/ovn-nb-logical_router_policy.go index c3eaee2cef5..97f9a10ba84 100644 --- a/pkg/ovs/ovn-nb-logical_router_policy.go +++ b/pkg/ovs/ovn-nb-logical_router_policy.go @@ -19,7 +19,7 @@ import ( ) // AddLogicalRouterPolicy add a policy route to logical router -func (c *OVNNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops []string, externalIDs map[string]string) error { +func (c *OVNNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops, bfdSessions []string, externalIDs map[string]string) error { fnFilter := func(policy *ovnnb.LogicalRouterPolicy) bool { return policy.Priority == priority && policy.Match == match } @@ -52,7 +52,7 @@ func (c *OVNNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, if policyFound == nil { klog.Infof("creating lr policy with priority = %d, match = %q, action = %q, nextHops = %q", priority, match, action, nextHops) - policy := c.newLogicalRouterPolicy(priority, match, action, nextHops, externalIDs) + policy := c.newLogicalRouterPolicy(priority, match, action, nextHops, bfdSessions, externalIDs) if err := c.CreateLogicalRouterPolicies(lrName, policy); err != nil { klog.Error(err) return fmt.Errorf("add policy to logical router %s: %w", lrName, err) @@ -281,13 +281,14 @@ func (c *OVNNbClient) ListLogicalRouterPolicies(lrName string, priority int, ext } // newLogicalRouterPolicy return logical router policy with basic information -func (c *OVNNbClient) newLogicalRouterPolicy(priority int, match, action string, nextHops []string, externalIDs map[string]string) *ovnnb.LogicalRouterPolicy { +func (c *OVNNbClient) newLogicalRouterPolicy(priority int, match, action string, nextHops, bfdSessions []string, externalIDs map[string]string) *ovnnb.LogicalRouterPolicy { return &ovnnb.LogicalRouterPolicy{ UUID: ovsclient.NamedUUID(), Priority: priority, Match: match, Action: action, Nexthops: nextHops, + BFDSessions: bfdSessions, ExternalIDs: externalIDs, } } @@ -324,6 +325,19 @@ func policyFilter(priority int, externalIDs map[string]string, ignoreExtIDEmptyV } } +func (c *OVNNbClient) UpdateLogicalRouterPolicy(policy *ovnnb.LogicalRouterPolicy, fields ...interface{}) error { + ops, err := c.ovsDbClient.Where(policy).Update(policy, fields...) + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to generate update operations for logical router policy %s: %w", policy.UUID, err) + } + if err = c.Transact("lr-policy-update", ops); err != nil { + klog.Error(err) + return fmt.Errorf("failed to update logical router policy %s: %w", policy.UUID, err) + } + return nil +} + func (c *OVNNbClient) DeleteRouterPolicy(lr *ovnnb.LogicalRouter, uuid string) error { ops, err := c.ovsDbClient.Where(lr).Mutate(lr, model.Mutation{ Field: &lr.Policies, @@ -336,7 +350,7 @@ func (c *OVNNbClient) DeleteRouterPolicy(lr *ovnnb.LogicalRouter, uuid string) e } if err = c.Transact("lr-policy-delete", ops); err != nil { klog.Error(err) - return fmt.Errorf("failed to delete route policy %s: %w", uuid, err) + return fmt.Errorf("failed to delete router policy %s: %w", uuid, err) } return nil } diff --git a/pkg/ovs/ovn-nb-logical_router_policy_test.go b/pkg/ovs/ovn-nb-logical_router_policy_test.go index 61f87e2c702..e404a1a4858 100644 --- a/pkg/ovs/ovn-nb-logical_router_policy_test.go +++ b/pkg/ovs/ovn-nb-logical_router_policy_test.go @@ -36,7 +36,7 @@ func (suite *OvnClientTestSuite) testAddLogicalRouterPolicy() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) lr, err := nbClient.GetLogicalRouter(lrName, false) @@ -48,28 +48,28 @@ func (suite *OvnClientTestSuite) testAddLogicalRouterPolicy() { require.Contains(t, lr.Policies, policyList[0].UUID) t.Run("normal add policy", func(t *testing.T) { - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) action = ovnnb.LogicalRouterPolicyActionDrop - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) }) t.Run("should log err when logical router does not exist", func(t *testing.T) { - err = nbClient.AddLogicalRouterPolicy("test-nonexist-lr", priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy("test-nonexist-lr", priority, match, action, nextHops, nil, nil) require.Error(t, err) }) t.Run("handle duplicate policies with matching action and nextHops", func(t *testing.T) { - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) - duplicatePolicy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, nil) + duplicatePolicy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, nil, nil) err = nbClient.CreateLogicalRouterPolicies(lrName, duplicatePolicy) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) finalPolicyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) @@ -80,7 +80,7 @@ func (suite *OvnClientTestSuite) testAddLogicalRouterPolicy() { t.Run("update policy with different externalIDs", func(t *testing.T) { initialExternalIDs := map[string]string{"key1": "value1"} - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, initialExternalIDs) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, initialExternalIDs) require.NoError(t, err) policyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) @@ -90,7 +90,7 @@ func (suite *OvnClientTestSuite) testAddLogicalRouterPolicy() { newExternalIDs := map[string]string{"key2": "value2"} - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, newExternalIDs) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, newExternalIDs) require.NoError(t, err) updatedPolicyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) @@ -119,7 +119,7 @@ func (suite *OvnClientTestSuite) testCreateLogicalRouterPolicies() { t.Run("add policies to logical router", func(t *testing.T) { for i := 0; i < 3; i++ { match := fmt.Sprintf("%s && tcp.dst == %d", matchPrefix, basePort+i) - policy := nbClient.newLogicalRouterPolicy(priority, match, action, nil, nil) + policy := nbClient.newLogicalRouterPolicy(priority, match, action, nil, nil, nil) policies = append(policies, policy) } @@ -170,7 +170,7 @@ func (suite *OvnClientTestSuite) testDeleteLogicalRouterPolicy() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) t.Run("no err when delete existent logical switch port", func(t *testing.T) { @@ -222,7 +222,7 @@ func (suite *OvnClientTestSuite) testDeleteLogicalRouterPolicies() { prepare := func() { for i := 0; i < 3; i++ { priority := basePriority + i - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, externalIDs) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, externalIDs) require.NoError(t, err) } @@ -296,7 +296,7 @@ func (suite *OvnClientTestSuite) testClearLogicalRouterPolicy() { for i := 0; i < 3; i++ { priority := basePriority + i - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) } @@ -342,7 +342,7 @@ func (suite *OvnClientTestSuite) testGetLogicalRouterPolicy() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, ovnnb.LogicalRouterPolicyActionAllow, nil, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, ovnnb.LogicalRouterPolicyActionAllow, nil, nil, nil) require.NoError(t, err) t.Run("priority and match are same", func(t *testing.T) { @@ -409,7 +409,7 @@ func (suite *OvnClientTestSuite) testGetLogicalRouterPolicyByUUID() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) lr, err := nbClient.GetLogicalRouter(lrName, false) @@ -453,7 +453,7 @@ func (suite *OvnClientTestSuite) testGetLogicalRouterPolicyByExtID() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, extID) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, extID) require.NoError(t, err) lr, err := nbClient.GetLogicalRouter(lrName, false) @@ -491,7 +491,7 @@ func (suite *OvnClientTestSuite) testGetLogicalRouterPolicyByExtID() { t.Run("get lrp with empty ExternalIDs", func(t *testing.T) { t.Parallel() - emptyExtIDPolicy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, nil) + emptyExtIDPolicy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, nil, nil) err = nbClient.CreateLogicalRouterPolicies(lrName, emptyExtIDPolicy) require.NoError(t, err) @@ -523,7 +523,7 @@ func (suite *OvnClientTestSuite) testNewLogicalRouterPolicy() { ExternalIDs: map[string]string{"key": "value"}, } - policy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, map[string]string{"key": "value"}) + policy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, nil, map[string]string{"key": "value"}) expect.UUID = policy.UUID require.Equal(t, expect, policy) } @@ -622,7 +622,7 @@ func (suite *OvnClientTestSuite) testDeleteRouterPolicy() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) lr, err := nbClient.GetLogicalRouter(lrName, false) @@ -651,7 +651,7 @@ func (suite *OvnClientTestSuite) testDeleteLogicalRouterPolicyByNexthop() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) lr, err := nbClient.GetLogicalRouter(lrName, false) diff --git a/pkg/ovs/ovn-nb-logical_router_route.go b/pkg/ovs/ovn-nb-logical_router_route.go index fd5a393ab10..6138c1ce89a 100644 --- a/pkg/ovs/ovn-nb-logical_router_route.go +++ b/pkg/ovs/ovn-nb-logical_router_route.go @@ -184,6 +184,67 @@ func (c *OVNNbClient) DeleteLogicalRouterStaticRoute(lrName string, routeTable, return nil } +// DeleteLogicalRouterStaticRoute delete a logical router static route +func (c *OVNNbClient) DeleteLogicalRouterStaticRouteByUUID(lrName, uuid string) error { + lr, err := c.GetLogicalRouter(lrName, true) + if err != nil { + return err + } + if lr == nil { + return nil + } + + // remove static route from logical router + ops, err := c.LogicalRouterUpdateStaticRouteOp(lrName, []string{uuid}, ovsdb.MutateOperationDelete) + if err != nil { + klog.Error(err) + return fmt.Errorf("generate operations for removing static route %s from logical router %s: %w", uuid, lrName, err) + } + if err = c.Transact("lr-route-del", ops); err != nil { + klog.Error(err) + return fmt.Errorf("delete static route %s from logical router %s: %w", uuid, lrName, err) + } + + return nil +} + +func (c *OVNNbClient) DeleteLogicalRouterStaticRouteByExternalIDs(lrName string, externalIDs map[string]string) error { + lr, err := c.GetLogicalRouter(lrName, true) + if err != nil { + return err + } + if lr == nil { + return nil + } + + routes, err := c.ListLogicalRouterStaticRoutes(lrName, nil, nil, "", externalIDs) + if err != nil { + klog.Error(err) + return err + } + if len(routes) == 0 { + return nil + } + + uuids := make([]string, 0, len(routes)) + for _, route := range routes { + uuids = append(uuids, route.UUID) + } + + // remove static route from logical router + ops, err := c.LogicalRouterUpdateStaticRouteOp(lrName, uuids, ovsdb.MutateOperationDelete) + if err != nil { + klog.Error(err) + return fmt.Errorf("generate operations for removing static routes %v from logical router %s: %w", uuids, lrName, err) + } + if err = c.Transact("lr-route-del", ops); err != nil { + klog.Error(err) + return fmt.Errorf("delete static routes %v from logical router %s: %w", uuids, lrName, err) + } + + return nil +} + // ClearLogicalRouterStaticRoute clear static route from logical router once func (c *OVNNbClient) ClearLogicalRouterStaticRoute(lrName string) error { lr, err := c.GetLogicalRouter(lrName, false) diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index 534621ebb6a..2a0f6f22e8c 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -44,6 +44,9 @@ const ( IfExists = "--if-exists" OVSDBWaitTimeout = 0 + + ExternalIDVendor = "vendor" + ExternalIDVpcEgressGateway = "vpc-egress-gateway" ) // NewLegacyClient init a legacy ovn client diff --git a/pkg/util/const.go b/pkg/util/const.go index 8acda2daaf0..1b9974e8488 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -48,6 +48,9 @@ const ( LogicalRouterAnnotation = "ovn.kubernetes.io/logical_router" VpcAnnotation = "ovn.kubernetes.io/vpc" + VpcEgressGatewayAnnotation = "ovn.kubernetes.io/vpc_egress_gateway" + GenerateHashAnnotation = "ovn.kubernetes.io/generate-hash" + Layer2ForwardAnnotationTemplate = "%s.kubernetes.io/layer2_forward" PortSecurityAnnotationTemplate = "%s.kubernetes.io/port_security" PortVipAnnotationTemplate = "%s.kubernetes.io/port_vips" @@ -210,6 +213,7 @@ const ( U2OSubnetPolicyPriority = 29400 GatewayRouterPolicyPriority = 29000 + EgressGatewayPolicyPriority = 29100 NorthGatewayRoutePolicyPriority = 29250 OvnICPolicyPriority = 29500 NodeRouterPolicyPriority = 30000 diff --git a/pkg/util/hash.go b/pkg/util/hash.go new file mode 100644 index 00000000000..da753e9740d --- /dev/null +++ b/pkg/util/hash.go @@ -0,0 +1,26 @@ +package util + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" +) + +func Sha256Hash(input []byte) string { + hasher := sha256.New() + hasher.Write(input) + hashedBytes := hasher.Sum(nil) + return hex.EncodeToString(hashedBytes) +} + +func Sha256HashObject(obj any) (string, error) { + buf, err := json.Marshal(obj) + if err != nil { + return "", err + } + + hasher := sha256.New() + hasher.Write(buf) + hashedBytes := hasher.Sum(nil) + return hex.EncodeToString(hashedBytes), nil +} diff --git a/pkg/util/k8s.go b/pkg/util/k8s.go index 61eddeac990..c82d8c77d63 100644 --- a/pkg/util/k8s.go +++ b/pkg/util/k8s.go @@ -4,12 +4,15 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "fmt" "net" "net/url" "strings" "time" + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -17,6 +20,9 @@ import ( "k8s.io/apimachinery/pkg/types" clientv1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" ) func DialTCP(host string, timeout time.Duration, verbose bool) error { @@ -87,6 +93,27 @@ func PodIPs(pod v1.Pod) []string { return ips } +func PodAttachmentIPs(pod *v1.Pod, networkName string) ([]string, error) { + if pod == nil { + return nil, errors.New("programmatic error: pod is nil") + } + + if pod.Annotations[nadv1.NetworkStatusAnnot] == "" { + return nil, fmt.Errorf("pod %s/%s has no network status annotation", pod.Namespace, pod.Name) + } + var status []nadv1.NetworkStatus + if err := json.Unmarshal([]byte(pod.Annotations[nadv1.NetworkStatusAnnot]), &status); err != nil { + return nil, fmt.Errorf("failed to unmarshal network status annotation of pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + for _, s := range status { + if s.Name == networkName { + return s.IPs, nil + } + } + + return nil, fmt.Errorf("pod %s/%s has no network status for network %s", pod.Namespace, pod.Name, networkName) +} + func ServiceClusterIPs(svc v1.Service) []string { if len(svc.Spec.ClusterIPs) == 0 && svc.Spec.ClusterIP != v1.ClusterIPNone && svc.Spec.ClusterIP != "" { return []string{svc.Spec.ClusterIP} @@ -152,3 +179,29 @@ func nodeMergePatch(cs clientv1.NodeInterface, node, patch string) error { } return nil } + +func SetOwnerReference(owner, object metav1.Object) error { + return controllerutil.SetOwnerReference(owner, object, scheme.Scheme) +} + +func DeploymentIsReady(deployment *appsv1.Deployment) bool { + if deployment.Generation > deployment.Status.ObservedGeneration { + return false + } + + for _, condition := range deployment.Status.Conditions { + if condition.Type == appsv1.DeploymentProgressing { + // deployment exceeded its progress deadline + if condition.Reason == "ProgressDeadlineExceeded" { + return false + } + break + } + } + if (deployment.Spec.Replicas != nil && deployment.Status.UpdatedReplicas < *deployment.Spec.Replicas) || + deployment.Status.Replicas > deployment.Status.UpdatedReplicas || + deployment.Status.AvailableReplicas < deployment.Status.UpdatedReplicas { + return false + } + return true +} diff --git a/pkg/util/pod_routes.go b/pkg/util/pod_routes.go new file mode 100644 index 00000000000..d1544f536d0 --- /dev/null +++ b/pkg/util/pod_routes.go @@ -0,0 +1,56 @@ +package util + +import ( + "encoding/json" + "fmt" + + "k8s.io/klog/v2" + + "github.com/kubeovn/kube-ovn/pkg/request" +) + +type PodRoutes map[string]map[string][]string + +func NewPodRoutes() PodRoutes { + return make(PodRoutes) +} + +func (r PodRoutes) Add(provider, destination, gateway string) { + if gateway == "" || destination == "" { + return + } + + if _, ok := r[provider]; !ok { + r[provider] = make(map[string][]string) + } + r[provider][gateway] = append(r[provider][gateway], destination) +} + +func (r PodRoutes) ToAnnotations() (map[string]string, error) { + annotations := make(map[string]string, len(r)) + for provider, routesMap := range r { + var routes []request.Route + for gw := range routesMap { + if gw == "" { + continue + } + for _, dst := range routesMap[gw] { + routes = append(routes, request.Route{ + Destination: dst, + Gateway: gw, + }) + } + } + if len(routes) == 0 { + continue + } + + buf, err := json.Marshal(routes) + if err != nil { + klog.Error(err) + return nil, err + } + annotations[fmt.Sprintf(RoutesAnnotationTemplate, provider)] = string(buf) + } + return annotations, nil +} diff --git a/pkg/util/strings.go b/pkg/util/strings.go index 81b10c50ee6..ac3d087f17b 100644 --- a/pkg/util/strings.go +++ b/pkg/util/strings.go @@ -1,8 +1,6 @@ package util import ( - "crypto/sha256" - "encoding/hex" "strings" ) @@ -27,10 +25,3 @@ func DoubleQuotedFields(s string) []string { return fields } - -func Sha256Hash(input []byte) string { - hasher := sha256.New() - hasher.Write(input) - hashedBytes := hasher.Sum(nil) - return hex.EncodeToString(hashedBytes) -} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 11777e1d787..b2923a0853a 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -34,6 +34,17 @@ const ( timeout = 2 * time.Minute ) +func LoadKubeOVNClientSet() (*kubeovncs.Clientset, error) { + config, err := framework.LoadConfig() + if err != nil { + return nil, err + } + + config.QPS = 20 + config.Burst = 50 + return kubeovncs.NewForConfig(config) +} + type Framework struct { KubeContext string *framework.Framework diff --git a/test/e2e/framework/pod.go b/test/e2e/framework/pod.go index bc37c4a9863..3751074f939 100644 --- a/test/e2e/framework/pod.go +++ b/test/e2e/framework/pod.go @@ -3,6 +3,9 @@ package framework import ( "context" "errors" + "fmt" + "slices" + "strings" "time" corev1 "k8s.io/api/core/v1" @@ -10,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" psaapi "k8s.io/pod-security-admission/api" "k8s.io/utils/ptr" @@ -142,3 +146,35 @@ func MakeRestrictedPod(ns, name string, labels, annotations map[string]string, i func MakePrivilegedPod(ns, name string, labels, annotations map[string]string, image string, command, args []string) *corev1.Pod { return makePod(ns, name, labels, annotations, image, command, args, psaapi.LevelPrivileged) } + +func CheckPodEgressRoutes(ns, pod string, ipv4, ipv6 bool, ttl int, expectedHops []string) { + ginkgo.GinkgoHelper() + + afs := make([]int, 0, 2) + dst := make([]string, 0, 2) + if ipv4 { + afs = append(afs, 4) + dst = append(dst, "1.1.1.1") + } + if ipv6 { + afs = append(afs, 6) + dst = append(dst, "2606:4700:4700::1111") + } + + for i, af := range afs { + ginkgo.By(fmt.Sprintf("Checking IPv%d egress routes for pod %s/%s", af, ns, pod)) + cmd := fmt.Sprintf("traceroute -%d -n -f%d -m%d %s", af, ttl, ttl, dst[i]) + WaitUntil(3*time.Second, 30*time.Second, func(_ context.Context) (bool, error) { + // traceroute to 1.1.1.1 (1.1.1.1), 2 hops max, 60 byte packets + // 2 172.19.0.2 0.663 ms 0.613 ms 0.605 ms + output, err := e2epodoutput.RunHostCmd(ns, pod, cmd) + if err != nil { + return false, nil + } + + lines := strings.Split(strings.TrimSpace(output), "\n") + fields := strings.Fields(lines[len(lines)-1]) + return len(fields) > 2 && slices.Contains(expectedHops, fields[1]), nil + }, "") + } +} diff --git a/test/e2e/framework/provider-network.go b/test/e2e/framework/provider-network.go index 76fdc007e7b..1c500ea5f83 100644 --- a/test/e2e/framework/provider-network.go +++ b/test/e2e/framework/provider-network.go @@ -18,6 +18,7 @@ import ( "github.com/onsi/gomega" apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" v1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -28,6 +29,12 @@ type ProviderNetworkClient struct { v1.ProviderNetworkInterface } +func NewProviderNetworkClient(cs clientset.Interface) *ProviderNetworkClient { + return &ProviderNetworkClient{ + ProviderNetworkInterface: cs.KubeovnV1().ProviderNetworks(), + } +} + func (f *Framework) ProviderNetworkClient() *ProviderNetworkClient { return &ProviderNetworkClient{ f: f, diff --git a/test/e2e/framework/vlan.go b/test/e2e/framework/vlan.go index 0da08795669..75e3bf8b19d 100644 --- a/test/e2e/framework/vlan.go +++ b/test/e2e/framework/vlan.go @@ -13,6 +13,7 @@ import ( "github.com/onsi/ginkgo/v2" apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" v1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -23,6 +24,12 @@ type VlanClient struct { v1.VlanInterface } +func NewVlanClient(cs clientset.Interface) *VlanClient { + return &VlanClient{ + VlanInterface: cs.KubeovnV1().Vlans(), + } +} + func (f *Framework) VlanClient() *VlanClient { return &VlanClient{ f: f, @@ -74,9 +81,9 @@ func (c *VlanClient) Patch(original, modified *apiv1.Vlan, timeout time.Duration } // Delete deletes a vlan if the vlan exists -func (c *VlanClient) Delete(name string, options metav1.DeleteOptions) { +func (c *VlanClient) Delete(name string) { ginkgo.GinkgoHelper() - err := c.VlanInterface.Delete(context.TODO(), name, options) + err := c.VlanInterface.Delete(context.TODO(), name, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { Failf("Failed to delete vlan %q: %v", name, err) } diff --git a/test/e2e/framework/vpc-egress-gateway.go b/test/e2e/framework/vpc-egress-gateway.go new file mode 100644 index 00000000000..3d82730c988 --- /dev/null +++ b/test/e2e/framework/vpc-egress-gateway.go @@ -0,0 +1,189 @@ +package framework + +import ( + "context" + "errors" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + v1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// VpcEgressGatewayClient is a struct for vpc egress gateway client. +type VpcEgressGatewayClient struct { + f *Framework + namespace string + v1.VpcEgressGatewayInterface +} + +func NewVpcEgressGatewayClient(cs clientset.Interface, namespapce string) *VpcEgressGatewayClient { + return &VpcEgressGatewayClient{ + namespace: namespapce, + VpcEgressGatewayInterface: cs.KubeovnV1().VpcEgressGateways(namespapce), + } +} + +func (f *Framework) VpcEgressGatewayClient() *VpcEgressGatewayClient { + return &VpcEgressGatewayClient{ + f: f, + namespace: f.Namespace.Name, + VpcEgressGatewayInterface: f.KubeOVNClientSet.KubeovnV1().VpcEgressGateways(f.Namespace.Name), + } +} + +func (f *Framework) VpcEgressGatewayClientNS(namespapce string) *VpcEgressGatewayClient { + return &VpcEgressGatewayClient{ + f: f, + namespace: namespapce, + VpcEgressGatewayInterface: f.KubeOVNClientSet.KubeovnV1().VpcEgressGateways(namespapce), + } +} + +func (c *VpcEgressGatewayClient) Get(name string) *apiv1.VpcEgressGateway { + ginkgo.GinkgoHelper() + gateway, err := c.VpcEgressGatewayInterface.Get(context.TODO(), name, metav1.GetOptions{}) + ExpectNoError(err) + return gateway +} + +// Create creates a new vpc-egress-gateway according to the framework specifications +func (c *VpcEgressGatewayClient) Create(gateway *apiv1.VpcEgressGateway) *apiv1.VpcEgressGateway { + ginkgo.GinkgoHelper() + g, err := c.VpcEgressGatewayInterface.Create(context.TODO(), gateway, metav1.CreateOptions{}) + ExpectNoError(err, "Error creating vpc-egress-gateway") + return g.DeepCopy() +} + +// CreateSync creates a new vpc-egress-gateway according to the framework specifications, and waits for it to be ready. +func (c *VpcEgressGatewayClient) CreateSync(gateway *apiv1.VpcEgressGateway) *apiv1.VpcEgressGateway { + ginkgo.GinkgoHelper() + _ = c.Create(gateway) + return c.WaitUntil(gateway.Name, func(g *apiv1.VpcEgressGateway) (bool, error) { + return g.Ready(), nil + }, "", 2*time.Second, timeout) +} + +// Patch patches the gateway +func (c *VpcEgressGatewayClient) Patch(original, modified *apiv1.VpcEgressGateway) *apiv1.VpcEgressGateway { + ginkgo.GinkgoHelper() + + patch, err := util.GenerateMergePatchPayload(original, modified) + ExpectNoError(err) + + var patchedGateway *apiv1.VpcEgressGateway + err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + g, err := c.VpcEgressGatewayInterface.Patch(ctx, original.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "") + if err != nil { + return handleWaitingAPIError(err, false, "patch vpc-egress-gateway %s/%s", original.Namespace, original.Name) + } + patchedGateway = g + return true, nil + }) + if err == nil { + return patchedGateway.DeepCopy() + } + + if errors.Is(err, context.DeadlineExceeded) { + Failf("timed out while retrying to patch vpc-egress-gateway %s/%s", original.Namespace, original.Name) + } + Failf("error occurred while retrying to patch vpc-egress-gateway %s/%s: %v", original.Namespace, original.Name, err) + + return nil +} + +// PatchSync patches the gateway and waits the gateway to meet the condition +func (c *VpcEgressGatewayClient) PatchSync(original, modified *apiv1.VpcEgressGateway) *apiv1.VpcEgressGateway { + ginkgo.GinkgoHelper() + _ = c.Patch(original, modified) + return c.WaitUntil(original.Name, func(g *apiv1.VpcEgressGateway) (bool, error) { + return g.Ready(), nil + }, "", 2*time.Second, timeout) +} + +// Delete deletes a vpc-egress-gateway if the vpc-egress-gateway exists +func (c *VpcEgressGatewayClient) Delete(name string) { + ginkgo.GinkgoHelper() + err := c.VpcEgressGatewayInterface.Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + Failf("Failed to delete vpc-egress-gateway %s/%s: %v", c.namespace, name, err) + } +} + +// DeleteSync deletes the vpc-egress-gateway and waits for the vpc-egress-gateway to disappear for `timeout`. +// If the vpc-egress-gateway doesn't disappear before the timeout, it will fail the test. +func (c *VpcEgressGatewayClient) DeleteSync(name string) { + ginkgo.GinkgoHelper() + c.Delete(name) + gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for vpc-egress-gateway %s/%s to disappear", c.namespace, name) +} + +// WaitUntil waits the given timeout duration for the specified condition to be met. +func (c *VpcEgressGatewayClient) WaitUntil(name string, cond func(g *apiv1.VpcEgressGateway) (bool, error), condDesc string, interval, timeout time.Duration) *apiv1.VpcEgressGateway { + var gateway *apiv1.VpcEgressGateway + err := wait.PollUntilContextTimeout(context.Background(), interval, timeout, false, func(_ context.Context) (bool, error) { + Logf("Waiting for vpc-egress-gateway %s/%s to meet condition %q", c.namespace, name, condDesc) + gateway = c.Get(name).DeepCopy() + met, err := cond(gateway) + if err != nil { + return false, fmt.Errorf("failed to check condition for vpc-egress-gateway %s/%s: %w", c.namespace, name, err) + } + if met { + Logf("vpc-egress-gateway %s/%s met condition %q", c.namespace, name, condDesc) + } else { + Logf("vpc-egress-gateway %s/%s not met condition %q", c.namespace, name, condDesc) + } + return met, nil + }) + if err == nil { + return gateway + } + + if errors.Is(err, context.DeadlineExceeded) { + Failf("timed out while waiting for vpc-egress-gateway %s/%s to meet condition %q", c.namespace, name, condDesc) + } + Failf("error occurred while waiting for vpc-egress-gateway %s/%s to meet condition %q: %v", c.namespace, name, condDesc, err) + + return nil +} + +// WaitToDisappear waits the given timeout duration for the specified vpc-egress-gateway to disappear. +func (c *VpcEgressGatewayClient) WaitToDisappear(name string, _, timeout time.Duration) error { + err := framework.Gomega().Eventually(context.Background(), framework.HandleRetry(func(ctx context.Context) (*apiv1.VpcEgressGateway, error) { + svc, err := c.VpcEgressGatewayInterface.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, nil + } + return svc, err + })).WithTimeout(timeout).Should(gomega.BeNil()) + if err != nil { + return fmt.Errorf("expected vpc-egress-gateway %s/%s to not be found: %w", c.namespace, name, err) + } + return nil +} + +func MakeVpcEgressGateway(namespace, name, vpc string, replicas int32, internalSubnet, externalSubnet string) *apiv1.VpcEgressGateway { + return &apiv1.VpcEgressGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: apiv1.VpcEgressGatewaySpec{ + Replicas: replicas, + VPC: vpc, + InternalSubnet: internalSubnet, + ExternalSubnet: externalSubnet, + }, + } +} diff --git a/test/e2e/kube-ovn/subnet/subnet.go b/test/e2e/kube-ovn/subnet/subnet.go index 49ad284cdaf..d9932dd350b 100644 --- a/test/e2e/kube-ovn/subnet/subnet.go +++ b/test/e2e/kube-ovn/subnet/subnet.go @@ -39,38 +39,6 @@ func getOvsPodOnNode(f *framework.Framework, node string) *corev1.Pod { return pod } -func checkNatOutgoingRoutes(f *framework.Framework, ns, pod string, gateways []string) { - ginkgo.GinkgoHelper() - - afs := make([]int, 0, 2) - dst := make([]string, 0, 2) - if f.HasIPv4() { - afs = append(afs, 4) - dst = append(dst, "1.1.1.1") - } - if f.HasIPv6() { - afs = append(afs, 6) - dst = append(dst, "2606:4700:4700::1111") - } - - for i, af := range afs { - ginkgo.By(fmt.Sprintf("Checking IPv%d NAT outgoing routes of %s/%s", af, ns, pod)) - cmd := fmt.Sprintf("traceroute -%d -n -f2 -m2 %s", af, dst[i]) - framework.WaitUntil(3*time.Second, 30*time.Second, func(_ context.Context) (bool, error) { - // traceroute to 1.1.1.1 (1.1.1.1), 2 hops max, 60 byte packets - // 2 172.19.0.2 0.663 ms 0.613 ms 0.605 ms - output, err := e2epodoutput.RunHostCmd(ns, pod, cmd) - if err != nil { - return false, nil - } - - lines := strings.Split(strings.TrimSpace(output), "\n") - fields := strings.Fields(lines[len(lines)-1]) - return len(fields) > 2 && slices.Contains(gateways, fields[1]), nil - }, "") - } -} - func checkSubnetNatOutgoingPolicyRuleStatus(subnetClient *framework.SubnetClient, subnetName string, rules []apiv1.NatOutgoingPolicyRule) *apiv1.Subnet { ginkgo.GinkgoHelper() @@ -382,7 +350,7 @@ var _ = framework.Describe("[group:subnet]", func() { pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) _ = podClient.CreateSync(pod) - checkNatOutgoingRoutes(f, namespaceName, podName, nodeIPs) + framework.CheckPodEgressRoutes(namespaceName, podName, f.HasIPv4(), f.HasIPv6(), 2, nodeIPs) }) framework.ConformanceIt("should be able to switch gateway mode to centralized", func() { @@ -489,7 +457,7 @@ var _ = framework.Describe("[group:subnet]", func() { pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) _ = podClient.CreateSync(pod) - checkNatOutgoingRoutes(f, namespaceName, podName, nodeIPs) + framework.CheckPodEgressRoutes(namespaceName, podName, f.HasIPv4(), f.HasIPv6(), 2, nodeIPs) }) framework.ConformanceIt("create centralized subnet without enableEcmp", func() { @@ -533,7 +501,8 @@ var _ = framework.Describe("[group:subnet]", func() { } else { gwIPv4, gwIPv6 = util.GetNodeInternalIP(nodes.Items[0]) } - checkNatOutgoingRoutes(f, namespaceName, podName, strings.Split(strings.Trim(gwIPv4+","+gwIPv6, ","), ",")) + hops := strings.Split(strings.Trim(gwIPv4+","+gwIPv6, ","), ",") + framework.CheckPodEgressRoutes(namespaceName, podName, f.HasIPv4(), f.HasIPv6(), 2, hops) ginkgo.By("Change subnet spec field enableEcmp to true") modifiedSubnet := subnet.DeepCopy() diff --git a/test/e2e/kube-ovn/underlay/underlay.go b/test/e2e/kube-ovn/underlay/underlay.go index d616e61040c..b4e43c46138 100644 --- a/test/e2e/kube-ovn/underlay/underlay.go +++ b/test/e2e/kube-ovn/underlay/underlay.go @@ -12,7 +12,6 @@ import ( dockernetwork "github.com/docker/docker/api/types/network" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" @@ -208,7 +207,7 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { routeMap[node.ID] = append(routeMap[node.ID], r) } } - framework.ExpectHaveKey(linkMap, node.ID) + framework.ExpectHaveKey(routeMap, node.ID) linkMap[node.Name()] = linkMap[node.ID] routeMap[node.Name()] = routeMap[node.ID] @@ -350,7 +349,7 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { vpcClient.DeleteSync(vpcName) ginkgo.By("Deleting vlan " + vlanName) - vlanClient.Delete(vlanName, metav1.DeleteOptions{}) + vlanClient.Delete(vlanName) ginkgo.By("Deleting provider network " + providerNetworkName) providerNetworkClient.DeleteSync(providerNetworkName) diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 4fed16067e9..bd7c22b095a 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -427,9 +427,9 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { subnetClient.DeleteSync(underlayExtraSubnetName) ginkgo.By("Deleting vlan " + vlanName) - vlanClient.Delete(vlanName, metav1.DeleteOptions{}) + vlanClient.Delete(vlanName) ginkgo.By("Deleting extra vlan " + vlanExtraName) - vlanClient.Delete(vlanExtraName, metav1.DeleteOptions{}) + vlanClient.Delete(vlanExtraName) ginkgo.By("Deleting provider network " + providerNetworkName) providerNetworkClient.DeleteSync(providerNetworkName) diff --git a/test/e2e/vpc-egress-gateway/e2e_test.go b/test/e2e/vpc-egress-gateway/e2e_test.go new file mode 100644 index 00000000000..0c1e7cd0024 --- /dev/null +++ b/test/e2e/vpc-egress-gateway/e2e_test.go @@ -0,0 +1,520 @@ +package multus + +import ( + "context" + "flag" + "fmt" + "math/rand/v2" + "net" + "reflect" + "slices" + "strconv" + "strings" + "testing" + "time" + + dockernetwork "github.com/docker/docker/api/types/network" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/component-base/logs" + "k8s.io/klog/v2" + commontest "k8s.io/kubernetes/test/e2e/common" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" + "github.com/kubeovn/kube-ovn/test/e2e/framework/docker" + "github.com/kubeovn/kube-ovn/test/e2e/framework/iproute" + "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" +) + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + + logs.InitLogs() + defer logs.FlushLogs() + klog.EnableContextualLogging(true) + + gomega.RegisterFailHandler(k8sframework.Fail) + + // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins + suiteConfig, reporterConfig := k8sframework.CreateGinkgoConfig() + klog.Infof("Starting e2e run %q on Ginkgo node %d", k8sframework.RunID, suiteConfig.ParallelProcess) + ginkgo.RunSpecs(t, "Kube-OVN e2e suite", suiteConfig, reporterConfig) +} + +const ( + kindNetwork = "kind" + + controlPlaneLabel = "node-role.kubernetes.io/control-plane" +) + +var clusterName string + +var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { + // Reference common test to make the import valid. + commontest.CurrentSuite = commontest.E2E + + cs, err := k8sframework.LoadClientset() + framework.ExpectNoError(err) + + ginkgo.By("Getting k8s nodes") + k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) + framework.ExpectNoError(err) + + var ok bool + if clusterName, ok = kind.IsKindProvided(k8sNodes.Items[0].Spec.ProviderID); !ok { + ginkgo.Fail("vpc-egress-gateway spec only runs on kind clusters") + } + + return []byte(clusterName) +}, func(data []byte) { + clusterName = string(data) +}) + +var _ = framework.Describe("[group:veg]", func() { + f := framework.NewDefaultFramework("veg") + + var vpcClient *framework.VpcClient + var subnetClient *framework.SubnetClient + var nadClient *framework.NetworkAttachmentDefinitionClient + var nadName, externalSubnetName, namespaceName string + var nodes, schedulableNodes []corev1.Node + var replicas int32 + ginkgo.BeforeEach(func() { + namespaceName = f.Namespace.Name + nadName = "nad-" + framework.RandomSuffix() + externalSubnetName = "ext-" + framework.RandomSuffix() + vpcClient = f.VpcClient() + subnetClient = f.SubnetClient() + nadClient = f.NetworkAttachmentDefinitionClient() + + nodeList, err := e2enode.GetReadyNodesIncludingTainted(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(nodeList.Items) + nodes = nodeList.Items + + nodeList, err = e2enode.GetReadySchedulableNodes(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(nodeList.Items) + schedulableNodes = nodeList.Items + + replicas = min(int32(len(schedulableNodes)), 3) + }) + + framework.ConformanceIt("should be able to create vpc-egress-gateway with underlay subnet", func() { + provider := fmt.Sprintf("%s.%s.%s", nadName, namespaceName, util.OvnProvider) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeOVNNetworkAttachmentDefinition(nadName, namespaceName, provider, nil) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting network attachment definition " + nadName) + nadClient.Delete(nadName) + }) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + dockerNetworkName := "net-" + framework.RandomSuffix() + ginkgo.By("Creating docker network " + dockerNetworkName) + dockerNetwork, err := docker.NetworkCreate(dockerNetworkName, true, true) + framework.ExpectNoError(err, "creating docker network "+dockerNetworkName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting docker network " + dockerNetworkName) + err = docker.NetworkRemove(dockerNetworkName) + framework.ExpectNoError(err, "removing docker network "+dockerNetworkName) + }) + + ginkgo.By("Getting kind nodes") + kindNodes, err := kind.ListNodes(clusterName, "") + framework.ExpectNoError(err, "getting nodes in kind cluster") + framework.ExpectNotEmpty(nodes) + + ginkgo.By("Connecting nodes to the docker network") + err = kind.NetworkConnect(dockerNetwork.ID, kindNodes) + framework.ExpectNoError(err, "connecting nodes to network "+dockerNetworkName) + ginkgo.DeferCleanup(func() { + err = kind.NetworkDisconnect(dockerNetwork.ID, kindNodes) + framework.ExpectNoError(err, "disconnecting nodes from network "+dockerNetworkName) + }) + + ginkgo.By("Getting node links that belong to the docker network") + kindNodes, err = kind.ListNodes(clusterName, "") + framework.ExpectNoError(err, "getting nodes in kind cluster") + linkMap := make(map[string]*iproute.Link, len(nodes)) + for _, node := range kindNodes { + links, err := node.ListLinks() + framework.ExpectNoError(err, "failed to list links on node %s: %v", node.Name(), err) + + for _, link := range links { + if link.Address == node.NetworkSettings.Networks[dockerNetworkName].MacAddress { + linkMap[node.Name()] = &link + break + } + } + framework.ExpectHaveKey(linkMap, node.Name()) + } + + providerNetworkName := "pn-" + framework.RandomSuffix() + ginkgo.By("Creating provider network " + providerNetworkName) + providerNetworkClient := f.ProviderNetworkClient() + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting provider network " + providerNetworkName) + providerNetworkClient.DeleteSync(providerNetworkName) + }) + var defaultInterface string + customInterfaces := make(map[string][]string, 0) + for node, link := range linkMap { + if defaultInterface == "" { + defaultInterface = link.IfName + } else if link.IfName != defaultInterface { + customInterfaces[link.IfName] = append(customInterfaces[link.IfName], node) + } + } + pn := framework.MakeProviderNetwork(providerNetworkName, false, defaultInterface, customInterfaces, nil) + _ = providerNetworkClient.CreateSync(pn) + + vlanName := "vlan-" + framework.RandomSuffix() + ginkgo.By("Creating vlan " + vlanName) + vlanClient := f.VlanClient() + vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) + _ = vlanClient.Create(vlan) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting vlan " + vlanName) + vlanClient.Delete(vlanName) + }) + + ginkgo.By("Getting docker network " + dockerNetworkName) + network, err := docker.NetworkInspect(dockerNetworkName) + framework.ExpectNoError(err, "getting docker network "+dockerNetworkName) + + externalSubnet := generateSubnetFromDockerNetwork(externalSubnetName, network, f.HasIPv4(), f.HasIPv6()) + externalSubnet.Spec.Provider = provider + externalSubnet.Spec.Vlan = vlanName + + ginkgo.By("Creating underlay subnet " + externalSubnetName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting external subnet " + externalSubnetName) + subnetClient.DeleteSync(externalSubnetName) + }) + _ = subnetClient.CreateSync(externalSubnet) + + vpcName := "ovn-cluster" + cidr := framework.RandomCIDR(f.ClusterIPFamily) + bfdIP := framework.RandomIPs(cidr, ";", 1) + ginkgo.By("Enabling BFD Port with IP " + bfdIP + " for VPC " + vpcName) + vpc := vpcClient.Get(vpcName) + patchedVpc := vpc.DeepCopy() + patchedVpc.Spec.BFDPort = &apiv1.BFDPort{ + Enabled: true, + IP: bfdIP, + NodeSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: controlPlaneLabel, + Operator: metav1.LabelSelectorOpExists, + }}, + }, + } + updatedVpc := vpcClient.PatchSync(vpc, patchedVpc, 10*time.Second) + ginkgo.DeferCleanup(func() { + ginkgo.By("Disabling BFD Port for VPC " + vpcName) + patchedVpc := updatedVpc.DeepCopy() + patchedVpc.Spec.BFDPort = nil + updatedVpc := vpcClient.PatchSync(updatedVpc, patchedVpc, 10*time.Second) + framework.ExpectEmpty(updatedVpc.Status.BFDPort.Name) + framework.ExpectEmpty(updatedVpc.Status.BFDPort.Nodes) + }) + + framework.ExpectNotEmpty(updatedVpc.Status.BFDPort.Name) + for _, node := range nodes { + if slices.Contains(updatedVpc.Status.BFDPort.Nodes, node.Name) { + framework.ExpectHaveKey(node.Labels, controlPlaneLabel) + } else { + framework.ExpectNotHaveKey(node.Labels, controlPlaneLabel) + } + } + + // TODO: check ovn LRP + + vegTest(f, true, provider, nadName, vpcName, vpc.Status.DefaultLogicalSwitch, externalSubnetName, replicas) + }) + + framework.ConformanceIt("should be able to create vpc-egress-gateway with macvlan", func() { + provider := fmt.Sprintf("%s.%s", nadName, namespaceName) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting network attachment definition " + nadName) + nadClient.Delete(nadName) + }) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + vpcName := "vpc-" + framework.RandomSuffix() + ginkgo.By("Creating vpc " + vpcName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting vpc " + vpcName) + vpcClient.DeleteSync(vpcName) + }) + vpc := &apiv1.Vpc{ObjectMeta: metav1.ObjectMeta{Name: vpcName}} + vpc = vpcClient.CreateSync(vpc) + framework.ExpectEmpty(vpc.Status.BFDPort.Name) + framework.ExpectEmpty(vpc.Status.BFDPort.IP) + framework.ExpectEmpty(vpc.Status.BFDPort.Nodes) + + internalSubnetName := "int-" + framework.RandomSuffix() + ginkgo.By("Creating internal subnet " + internalSubnetName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting internal subnet " + internalSubnetName) + subnetClient.DeleteSync(internalSubnetName) + }) + cidr := framework.RandomCIDR(f.ClusterIPFamily) + internalSubnet := framework.MakeSubnet(internalSubnetName, "", cidr, "", vpcName, "", nil, nil, nil) + _ = subnetClient.CreateSync(internalSubnet) + + ginkgo.By("Getting docker network " + kindNetwork) + network, err := docker.NetworkInspect(kindNetwork) + framework.ExpectNoError(err, "getting docker network "+kindNetwork) + + externalSubnet := generateSubnetFromDockerNetwork(externalSubnetName, network, f.HasIPv4(), f.HasIPv6()) + externalSubnet.Spec.Provider = provider + + ginkgo.By("Creating macvlan subnet " + externalSubnetName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting external subnet " + externalSubnetName) + subnetClient.DeleteSync(externalSubnetName) + }) + _ = subnetClient.CreateSync(externalSubnet) + + vegTest(f, false, provider, nadName, vpcName, internalSubnetName, externalSubnetName, replicas) + }) +}) + +func generateSubnetFromDockerNetwork(subnetName string, network *dockernetwork.Inspect, ipv4, ipv6 bool) *apiv1.Subnet { + ginkgo.GinkgoHelper() + + ginkgo.By("Generating subnet configuration from docker network " + network.Name) + var cidrV4, cidrV6, gatewayV4, gatewayV6 string + for _, config := range network.IPAM.Config { + switch util.CheckProtocol(config.Subnet) { + case apiv1.ProtocolIPv4: + if ipv4 { + cidrV4 = config.Subnet + gatewayV4 = config.Gateway + } + case apiv1.ProtocolIPv6: + if ipv6 { + cidrV6 = config.Subnet + if gatewayV6 = config.Gateway; gatewayV6 == "" { + var err error + gatewayV6, err = util.FirstIP(cidrV6) + framework.ExpectNoError(err) + } + } + } + } + + cidr := make([]string, 0, 2) + gateway := make([]string, 0, 2) + if ipv4 { + cidr = append(cidr, cidrV4) + gateway = append(gateway, gatewayV4) + } + if ipv6 { + cidr = append(cidr, cidrV6) + gateway = append(gateway, gatewayV6) + } + + excludeIPs := make([]string, 0, len(network.Containers)*2) + for _, container := range network.Containers { + if container.IPv4Address != "" && ipv4 { + excludeIPs = append(excludeIPs, strings.Split(container.IPv4Address, "/")[0]) + } + if container.IPv6Address != "" && ipv6 { + excludeIPs = append(excludeIPs, strings.Split(container.IPv6Address, "/")[0]) + } + } + + return framework.MakeSubnet(subnetName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", "", excludeIPs, nil, nil) +} + +func checkEgressAccess(f *framework.Framework, namespaceName, svrPodName, image, svrPort string, svrIPs, intIPs, extIPs []string, subnetName string, snat bool) { + podName := "pod-" + framework.RandomSuffix() + ginkgo.By("Creating client pod " + podName + " within subnet " + subnetName) + annotations := map[string]string{util.LogicalSwitchAnnotation: subnetName} + pod := framework.MakePod(namespaceName, podName, nil, annotations, image, []string{"sleep", "infinity"}, nil) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting pod " + podName) + f.PodClient().DeleteSync(podName) + }) + pod = f.PodClient().CreateSync(pod) + + framework.CheckPodEgressRoutes(pod.Namespace, pod.Name, f.HasIPv4(), f.HasIPv6(), 2, intIPs) + + if !snat { + podIPv4, podIPv6 := util.SplitIpsByProtocol(util.PodIPs(*pod)) + hopsIPv4, hopsIPv6 := util.SplitIpsByProtocol(extIPs) + addEcmpRoutes(namespaceName, svrPodName, podIPv4, hopsIPv4) + addEcmpRoutes(namespaceName, svrPodName, podIPv6, hopsIPv6) + } + + expectedClientIPs := extIPs + if !snat { + expectedClientIPs = util.PodIPs(*pod) + } + for _, svrIP := range svrIPs { + protocol := strings.ToLower(util.CheckProtocol(svrIP)) + ginkgo.By("Checking connection from " + pod.Name + " to " + svrIP + " via " + protocol) + cmd := fmt.Sprintf("curl -q -s --connect-timeout 2 --max-time 2 %s/clientip", net.JoinHostPort(svrIP, svrPort)) + ginkgo.By(fmt.Sprintf(`Executing %q in pod %s`, cmd, pod.Name)) + output := e2epodoutput.RunHostCmdOrDie(pod.Namespace, pod.Name, cmd) + clientIP, _, err := net.SplitHostPort(strings.TrimSpace(output)) + framework.ExpectNoError(err) + framework.ExpectContainElement(expectedClientIPs, clientIP) + } +} + +func addEcmpRoutes(namespaceName, podName string, destinations, nextHops []string) { + ginkgo.GinkgoHelper() + + if len(destinations) == 0 || len(nextHops) == 0 { + return + } + + var args string + if len(nextHops) == 1 { + args = " via " + nextHops[0] + } else { + for _, ip := range nextHops { + args += fmt.Sprintf(" nexthop via %s dev net1 weight 1", ip) + } + } + for _, dst := range destinations { + cmd := fmt.Sprintf("ip route add %s%s", dst, args) + output, err := e2epodoutput.RunHostCmd(namespaceName, podName, cmd) + framework.ExpectNoError(err, output) + } +} + +func vegTest(f *framework.Framework, bfd bool, provider, nadName, vpcName, internalSubnetName, externalSubnetName string, replicas int32) { + ginkgo.GinkgoHelper() + + namespaceName := f.Namespace.Name + snatSubnetName := "snat-" + framework.RandomSuffix() + forwardSubnetName := "forward-" + framework.RandomSuffix() + subnetClient := f.SubnetClient() + vegClient := f.VpcEgressGatewayClient() + deployClient := f.DeploymentClient() + podClient := f.PodClient() + + var forwardSubnet *apiv1.Subnet + for _, subnetName := range []string{snatSubnetName, forwardSubnetName} { + ginkgo.By("Creating subnet " + subnetName) + cidr := framework.RandomCIDR(f.ClusterIPFamily) + subnet := framework.MakeSubnet(subnetName, "", cidr, "", vpcName, "", nil, nil, nil) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting subnet " + subnetName) + subnetClient.DeleteSync(subnetName) + }) + _ = subnetClient.CreateSync(subnet) + if subnetName == forwardSubnetName { + forwardSubnet = subnet + } + } + + vegName := "veg-" + framework.RandomSuffix() + ginkgo.By("Creating vpc egress gateway " + vegName) + veg := framework.MakeVpcEgressGateway(namespaceName, vegName, vpcName, replicas, internalSubnetName, externalSubnetName) + if rand.Int32N(2) == 0 { + veg.Spec.Prefix = fmt.Sprintf("e2e-%s-", framework.RandomSuffix()) + } + veg.Spec.BFD.Enabled = bfd + veg.Spec.Policies = []apiv1.VpcEgressGatewayPolicy{{ + SNAT: true, + Subnets: []string{snatSubnetName}, + }, { + SNAT: false, + IPBlocks: strings.Split(forwardSubnet.Spec.CIDRBlock, ","), + }} + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting vpc egress gateway " + vegName) + vegClient.DeleteSync(vegName) + }) + veg = vegClient.CreateSync(veg) + + ginkgo.By("Validating vpc egress gateway status") + framework.ExpectTrue(veg.Status.Ready) + framework.ExpectEqual(veg.Status.Phase, apiv1.PhaseCompleted) + framework.ExpectHaveLen(veg.Status.InternalIPs, int(replicas)) + framework.ExpectHaveLen(veg.Status.ExternalIPs, int(replicas)) + + ginkgo.By("Validating vpc egress gateway workload") + framework.ExpectEqual(veg.Status.Workload.Name, veg.Spec.Prefix+veg.Name) + deploy := deployClient.Get(veg.Status.Workload.Name) + framework.ExpectEqual(deploy.Status.Replicas, replicas) + framework.ExpectEqual(deploy.Status.ReadyReplicas, replicas) + gvk := appsv1.SchemeGroupVersion.WithKind(reflect.TypeOf(deploy).Elem().Name()) + framework.ExpectEqual(veg.Status.Workload.APIVersion, gvk.GroupVersion().String()) + framework.ExpectEqual(veg.Status.Workload.Kind, gvk.Kind) + framework.ExpectHaveLen(veg.Status.Workload.Nodes, int(replicas)) + workloadPods, err := deployClient.GetPods(deploy) + framework.ExpectNoError(err) + framework.ExpectHaveLen(workloadPods.Items, int(replicas)) + podNodes := make([]string, 0, len(workloadPods.Items)) + for _, pod := range workloadPods.Items { + framework.ExpectNotContainElement(podNodes, pod.Spec.NodeName) + podNodes = append(podNodes, pod.Spec.NodeName) + } + framework.ExpectConsistOf(veg.Status.Workload.Nodes, podNodes) + + svrPodName := "svr-" + framework.RandomSuffix() + ginkgo.By("Creating netexec server pod " + svrPodName) + routes := util.NewPodRoutes() + dstV4, dstV6 := util.SplitStringIP(forwardSubnet.Spec.CIDRBlock) + gwV4, gwV6 := util.SplitStringIP(veg.Status.ExternalIPs[0]) + routes.Add(provider, dstV4, gwV4) + routes.Add(provider, dstV6, gwV6) + annotations, err := routes.ToAnnotations() + framework.ExpectNoError(err) + attachmentNetworkName := fmt.Sprintf("%s/%s", namespaceName, nadName) + annotations[util.AttachmentNetworkAnnotation] = attachmentNetworkName + port := strconv.Itoa(8000 + rand.IntN(1000)) + args := []string{"netexec", "--http-port", port} + svrPod := framework.MakePrivilegedPod(namespaceName, svrPodName, nil, annotations, framework.AgnhostImage, nil, args) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting pod " + svrPodName) + podClient.DeleteSync(svrPodName) + }) + svrPod = podClient.CreateSync(svrPod) + svrIPs, err := util.PodAttachmentIPs(svrPod, attachmentNetworkName) + framework.ExpectNoError(err) + + image := workloadPods.Items[0].Spec.Containers[0].Image + intIPs := make([]string, 0, len(veg.Status.InternalIPs)*2) + extIPs := make([]string, 0, len(veg.Status.ExternalIPs)*2) + for _, ip := range veg.Status.InternalIPs { + intIPs = append(intIPs, strings.Split(ip, ",")...) + } + for _, ips := range veg.Status.ExternalIPs { + extIPs = append(extIPs, strings.Split(ips, ",")...) + } + checkEgressAccess(f, namespaceName, svrPodName, image, port, svrIPs, intIPs, extIPs, snatSubnetName, true) + checkEgressAccess(f, namespaceName, svrPodName, image, port, svrIPs, intIPs, extIPs, forwardSubnetName, false) +} diff --git a/yamls/audit-policy.yaml b/yamls/audit-policy.yaml index 1f196f593c8..8576c6f8190 100644 --- a/yamls/audit-policy.yaml +++ b/yamls/audit-policy.yaml @@ -39,6 +39,8 @@ rules: - vpcs/status - vpc-nat-gateways - vpc-nat-gateways/status + - vpc-egress-gateways + - vpc-egress-gateways/status - vpc-dnses - vpc-dnses/status - subnets From 0cd56627589a5dd9af6b60b23d9b9d4f10880bc3 Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Thu, 28 Nov 2024 09:45:36 +0000 Subject: [PATCH 2/4] add label selector to deployment informer Signed-off-by: zhangzujian --- pkg/controller/controller.go | 16 ++++++++++++++-- pkg/controller/vpc_egress_gateway.go | 2 +- pkg/util/const.go | 6 +++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 14e7a1886a1..50d5405f852 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -271,6 +271,7 @@ type Controller struct { recorder record.EventRecorder informerFactory kubeinformers.SharedInformerFactory cmInformerFactory kubeinformers.SharedInformerFactory + deployInformerFactory kubeinformers.SharedInformerFactory kubeovnInformerFactory kubeovninformer.SharedInformerFactory anpInformerFactory anpinformer.SharedInformerFactory } @@ -294,6 +295,11 @@ func Run(ctx context.Context, config *Configuration) { &workqueue.TypedBucketRateLimiter[string]{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, ) + selector, err := labels.Parse(util.VpcEgressGatewayLabel) + if err != nil { + util.LogFatalAndExit(err, "failed to create label selector for vpc egress gateway workload") + } + informerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeFactoryClient, 0, kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { listOption.AllowWatchBookmarks = true @@ -302,6 +308,11 @@ func Run(ctx context.Context, config *Configuration) { kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { listOption.AllowWatchBookmarks = true }), kubeinformers.WithNamespace(config.PodNamespace)) + deployInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeFactoryClient, 0, + kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { + listOption.AllowWatchBookmarks = true + listOption.LabelSelector = selector.String() + })) kubeovnInformerFactory := kubeovninformer.NewSharedInformerFactoryWithOptions(config.KubeOvnFactoryClient, 0, kubeovninformer.WithTweakListOptions(func(listOption *metav1.ListOptions) { listOption.AllowWatchBookmarks = true @@ -330,7 +341,7 @@ func Run(ctx context.Context, config *Configuration) { nodeInformer := informerFactory.Core().V1().Nodes() serviceInformer := informerFactory.Core().V1().Services() endpointInformer := informerFactory.Core().V1().Endpoints() - deploymentInformer := informerFactory.Apps().V1().Deployments() + deploymentInformer := deployInformerFactory.Apps().V1().Deployments() qosPolicyInformer := kubeovnInformerFactory.Kubeovn().V1().QoSPolicies() configMapInformer := cmInformerFactory.Core().V1().ConfigMaps() npInformer := informerFactory.Networking().V1().NetworkPolicies() @@ -531,11 +542,11 @@ func Run(ctx context.Context, config *Configuration) { recorder: recorder, informerFactory: informerFactory, cmInformerFactory: cmInformerFactory, + deployInformerFactory: deployInformerFactory, kubeovnInformerFactory: kubeovnInformerFactory, anpInformerFactory: anpInformerFactory, } - var err error if controller.OVNNbClient, err = ovs.NewOvnNbClient( config.OvnNbAddr, config.OvnTimeout, @@ -609,6 +620,7 @@ func Run(ctx context.Context, config *Configuration) { // Wait for the caches to be synced before starting workers controller.informerFactory.Start(ctx.Done()) controller.cmInformerFactory.Start(ctx.Done()) + controller.deployInformerFactory.Start(ctx.Done()) controller.kubeovnInformerFactory.Start(ctx.Done()) controller.anpInformerFactory.Start(ctx.Done()) diff --git a/pkg/controller/vpc_egress_gateway.go b/pkg/controller/vpc_egress_gateway.go index 0dc8c89d27a..6a1a46a851b 100644 --- a/pkg/controller/vpc_egress_gateway.go +++ b/pkg/controller/vpc_egress_gateway.go @@ -360,7 +360,7 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa initEnv = append(initEnv, ipv6Env...) // generate workload - labels := map[string]string{util.VpcEgressGatewayAnnotation: gw.Name} + labels := map[string]string{util.VpcEgressGatewayLabel: gw.Name} deploy := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: gw.Spec.Prefix + gw.Name, diff --git a/pkg/util/const.go b/pkg/util/const.go index 1b9974e8488..b3e4063c3b2 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -48,9 +48,6 @@ const ( LogicalRouterAnnotation = "ovn.kubernetes.io/logical_router" VpcAnnotation = "ovn.kubernetes.io/vpc" - VpcEgressGatewayAnnotation = "ovn.kubernetes.io/vpc_egress_gateway" - GenerateHashAnnotation = "ovn.kubernetes.io/generate-hash" - Layer2ForwardAnnotationTemplate = "%s.kubernetes.io/layer2_forward" PortSecurityAnnotationTemplate = "%s.kubernetes.io/port_security" PortVipAnnotationTemplate = "%s.kubernetes.io/port_vips" @@ -114,6 +111,9 @@ const ( NetworkPolicyLogAnnotation = "ovn.kubernetes.io/enable_log" ACLActionsLogAnnotation = "ovn.kubernetes.io/log_acl_actions" + VpcEgressGatewayLabel = "ovn.kubernetes.io/vpc-egress-gateway" + GenerateHashAnnotation = "ovn.kubernetes.io/generate-hash" + VpcLastName = "ovn.kubernetes.io/last_vpc_name" VpcLastPolicies = "ovn.kubernetes.io/last_policies" From 052b99dbc54d06974a0c0752f64c31ba07ca6ca9 Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Thu, 28 Nov 2024 07:36:04 +0000 Subject: [PATCH 3/4] redefine pod routes Signed-off-by: zhangzujian --- pkg/util/pod_routes.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/util/pod_routes.go b/pkg/util/pod_routes.go index d1544f536d0..88e75698af8 100644 --- a/pkg/util/pod_routes.go +++ b/pkg/util/pod_routes.go @@ -9,7 +9,11 @@ import ( "github.com/kubeovn/kube-ovn/pkg/request" ) -type PodRoutes map[string]map[string][]string +// PodProviderRoutes represents configured routes for a provider/interface +type PodProviderRoutes map[string][]string // gateway -> destinations + +// PodRoutes represents configured routes for all providers/interfaces +type PodRoutes map[string]PodProviderRoutes // provider -> PodProviderRoutes func NewPodRoutes() PodRoutes { return make(PodRoutes) @@ -20,8 +24,8 @@ func (r PodRoutes) Add(provider, destination, gateway string) { return } - if _, ok := r[provider]; !ok { - r[provider] = make(map[string][]string) + if r[provider] == nil { + r[provider] = make(PodProviderRoutes) } r[provider][gateway] = append(r[provider][gateway], destination) } From 85d3f533b81e3a0fdbb405adafdee5e47fd62a90 Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Thu, 28 Nov 2024 08:29:03 +0000 Subject: [PATCH 4/4] add comments Signed-off-by: zhangzujian --- pkg/apis/kubeovn/v1/vpc-egress-gateway.go | 75 ++++++++++++++++------- pkg/apis/kubeovn/v1/vpc.go | 12 +++- pkg/controller/controller.go | 1 + pkg/controller/deployment.go | 13 ++-- pkg/controller/vpc_egress_gateway.go | 25 ++++++++ pkg/util/pod_routes.go | 1 + 6 files changed, 94 insertions(+), 33 deletions(-) diff --git a/pkg/apis/kubeovn/v1/vpc-egress-gateway.go b/pkg/apis/kubeovn/v1/vpc-egress-gateway.go index b879ae5be87..b4752dacb99 100644 --- a/pkg/apis/kubeovn/v1/vpc-egress-gateway.go +++ b/pkg/apis/kubeovn/v1/vpc-egress-gateway.go @@ -9,9 +9,12 @@ import ( type Phase string const ( - PhasePending Phase = "Pending" + // PhasePending means the resource is pending and not processed yet + PhasePending Phase = "Pending" + // PhaseProcessing means the resource is being processed PhaseProcessing Phase = "Processing" - PhaseCompleted Phase = "Completed" + // PhaseCompleted means the resource has been processed successfully + PhaseCompleted Phase = "Completed" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -25,6 +28,7 @@ type VpcEgressGatewayList struct { // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +resourceName=vpc-egress-gateways +// vpc egress gateway is used to forward the egress traffic from the VPC to the external network type VpcEgressGateway struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -33,34 +37,59 @@ type VpcEgressGateway struct { Status VpcEgressGatewayStatus `json:"status,omitempty"` } +// Ready returns true if the VpcEgressGateway has been processed successfully and is ready to serve traffic func (g *VpcEgressGateway) Ready() bool { return g.Status.Ready && g.Status.Conditions.IsReady(g.Generation) } type VpcEgressGatewaySpec struct { - VPC string `json:"vpc,omitempty"` - Replicas int32 `json:"replicas,omitempty"` - Prefix string `json:"prefix,omitempty"` - Image string `json:"image,omitempty"` - InternalSubnet string `json:"internalSubnet,omitempty"` - ExternalSubnet string `json:"externalSubnet"` - InternalIPs []string `json:"internalIPs,omitempty"` - ExternalIPs []string `json:"externalIPs,omitempty"` - - BFD VpcEgressGatewayBFDConfig `json:"bfd,omitempty"` - Policies []VpcEgressGatewayPolicy `json:"policies,omitempty"` + // optional VPC name + // if not specified, the default VPC will be used + VPC string `json:"vpc,omitempty"` + // workload replicas + Replicas int32 `json:"replicas,omitempty"` + // optional name prefix used to generate the workload + // the workload name will be generated as + Prefix string `json:"prefix,omitempty"` + // optional image used by the workload + // if not specified, the default image passed in by kube-ovn-controller will be used + Image string `json:"image,omitempty"` + // optional internal subnet used to create the workload + // if not specified, the workload will be created in the default subnet of the VPC + InternalSubnet string `json:"internalSubnet,omitempty"` + // external subnet used to create the workload + ExternalSubnet string `json:"externalSubnet"` + // optional internal/external IPs used to create the workload + // these IPs must be in the internal/external subnet + // the IPs count must NOT be less than the replicas count + InternalIPs []string `json:"internalIPs,omitempty"` + ExternalIPs []string `json:"externalIPs,omitempty"` + + // BFD configuration + BFD VpcEgressGatewayBFDConfig `json:"bfd,omitempty"` + // egress policies + // at least one policy must be specified + Policies []VpcEgressGatewayPolicy `json:"policies,omitempty"` + // optional node selector used to select the nodes where the workload will be running NodeSelector []VpcEgressGatewayNodeSelector `json:"nodeSelector,omitempty"` } type VpcEgressGatewayBFDConfig struct { - Enabled bool `json:"enabled"` + // whether to enable BFD + // if set to true, the egress gateway will establish BFD session(s) with the VPC BFD LRP + // the VPC's .spec.bfd.enabled must be set to true to enable BFD + Enabled bool `json:"enabled"` + // optional BFD minRX/minTX/multiplier MinRX int32 `json:"minRX,omitempty"` MinTX int32 `json:"minTX,omitempty"` Multiplier int32 `json:"multiplier,omitempty"` } type VpcEgressGatewayPolicy struct { - SNAT bool `json:"snat"` + // whether to enable SNAT/MASQUERADE for the egress traffic + SNAT bool `json:"snat"` + // CIDRs/subnets targeted by the egress traffic policy + // packets whose source address is in the CIDRs/subnets will be forwarded to the egress gateway IPBlocks []string `json:"ipBlocks,omitempty"` Subnets []string `json:"subnets,omitempty"` } @@ -72,18 +101,22 @@ type VpcEgressGatewayNodeSelector struct { } type VpcEgressGatewayStatus struct { - Ready bool `json:"ready"` - Phase Phase `json:"phase"` + // whether the egress gateway is ready + Ready bool `json:"ready"` + Phase Phase `json:"phase"` + // internal/external IPs used by the workload InternalIPs []string `json:"internalIPs,omitempty"` ExternalIPs []string `json:"externalIPs,omitempty"` Conditions Conditions `json:"conditions,omitempty"` + // workload information Workload VpcEgressWorkload `json:"workload,omitempty"` } type VpcEgressWorkload struct { - APIVersion string `json:"apiVersion,omitempty"` - Kind string `json:"kind,omitempty"` - Name string `json:"name,omitempty"` - Nodes []string `json:"nodes,omitempty"` + APIVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + // nodes where the workload is running + Nodes []string `json:"nodes,omitempty"` } diff --git a/pkg/apis/kubeovn/v1/vpc.go b/pkg/apis/kubeovn/v1/vpc.go index ad502056824..c6eecfa4e3f 100644 --- a/pkg/apis/kubeovn/v1/vpc.go +++ b/pkg/apis/kubeovn/v1/vpc.go @@ -53,13 +53,19 @@ type VpcSpec struct { EnableExternal bool `json:"enableExternal,omitempty"` ExtraExternalSubnets []string `json:"extraExternalSubnets,omitempty"` EnableBfd bool `json:"enableBfd,omitempty"` - BFDPort *BFDPort `json:"bfdPort"` + + // optional BFD LRP configuration + // currently the LRP is used for vpc external gateway only + BFDPort *BFDPort `json:"bfdPort"` } type BFDPort struct { - Enabled bool `json:"enabled"` - IP string `json:"ip,omitempty"` + Enabled bool `json:"enabled"` + // ip address(es) of the BFD port + IP string `json:"ip,omitempty"` + // optional node selector used to select the nodes where the BFD LRP will be hosted + // if not specified, at most 3 nodes will be selected NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty"` } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 50d5405f852..05840397c4f 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -308,6 +308,7 @@ func Run(ctx context.Context, config *Configuration) { kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { listOption.AllowWatchBookmarks = true }), kubeinformers.WithNamespace(config.PodNamespace)) + // deployment informer used to list/watch vpc egress gateway workloads deployInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeFactoryClient, 0, kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { listOption.AllowWatchBookmarks = true diff --git a/pkg/controller/deployment.go b/pkg/controller/deployment.go index 6286782beff..5a11cf78eb4 100644 --- a/pkg/controller/deployment.go +++ b/pkg/controller/deployment.go @@ -31,19 +31,14 @@ func init() { func (c *Controller) enqueueAddDeployment(obj interface{}) { deploy := obj.(*appsv1.Deployment) - var vegName string for _, ref := range deploy.OwnerReferences { if ref.APIVersion == vpcEgressGatewayGroupVersion && ref.Kind == vpcEgressGatewayKind { - vegName = ref.Name - break + key := fmt.Sprintf("%s/%s", deploy.Namespace, ref.Name) + klog.V(3).Infof("enqueue update vpc-egress-gateway %s", key) + c.addOrUpdateVpcEgressGatewayQueue.Add(key) + return } } - - if vegName != "" { - key := fmt.Sprintf("%s/%s", deploy.Namespace, vegName) - klog.V(3).Infof("enqueue update vpc-egress-gateway %s", key) - c.addOrUpdateVpcEgressGatewayQueue.Add(key) - } } func (c *Controller) enqueueUpdateDeployment(_, newObj interface{}) { diff --git a/pkg/controller/vpc_egress_gateway.go b/pkg/controller/vpc_egress_gateway.go index 6a1a46a851b..b10faccbfe8 100644 --- a/pkg/controller/vpc_egress_gateway.go +++ b/pkg/controller/vpc_egress_gateway.go @@ -122,6 +122,7 @@ func (c *Controller) handleAddOrUpdateVpcEgressGateway(key string) error { bfdIPv4, bfdIPv6 = util.SplitStringIP(bfdIP) } + // reconcile the vpc egress gateway workload and get the route sources for later OVN resources reconciliation attachmentNetworkName, ipv4Src, ipv6Src, deploy, err := c.reconcileVpcEgressGatewayWorkload(gw, vpc, bfdIP, bfdIPv4, bfdIPv6) if err != nil { klog.Error(err) @@ -142,6 +143,7 @@ func (c *Controller) handleAddOrUpdateVpcEgressGateway(key string) error { msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) gw.Status.Conditions.SetCondition(kubeovnv1.Ready, corev1.ConditionFalse, "Processing", msg, gw.Generation) } else { + // get the pods of the deployment to collect the pod IPs podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) if err != nil { err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) @@ -156,6 +158,7 @@ func (c *Controller) handleAddOrUpdateVpcEgressGateway(key string) error { return err } + // update gateway status including the internal/external IPs and the nodes where the pods are running gw.Status.Workload.Nodes = make([]string, 0, len(pods)) for _, pod := range pods { gw.Status.Workload.Nodes = append(gw.Status.Workload.Nodes, pod.Spec.NodeName) @@ -176,9 +179,11 @@ func (c *Controller) handleAddOrUpdateVpcEgressGateway(key string) error { return err } if len(gw.Status.Workload.Nodes) == 0 { + // the workload is not ready yet return nil } + // reconcile OVN routes nextHopsIPv4, hextHopsIPv6 := util.SplitIpsByProtocol(podIPs) if err = c.reconcileVpcEgressGatewayOVNRoutes(gw, 4, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv4, set.New(nextHopsIPv4...), ipv4Src); err != nil { klog.Error(err) @@ -227,6 +232,7 @@ func (c *Controller) updateVpcEgressGatewayStatus(gw *kubeovnv1.VpcEgressGateway return updateGateway, nil } +// create or update vpc egress gateway workload func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGateway, vpc *kubeovnv1.Vpc, bfdIP, bfdIPv4, bfdIPv6 string) (string, set.Set[string], set.Set[string], *appsv1.Deployment, error) { image := c.config.Image if gw.Spec.Image != "" { @@ -282,15 +288,19 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa } attachmentNetworkName := fmt.Sprintf("%s/%s", nadNamespace, nadName) + // generate route annotations used to configure routes in the pod routes := util.NewPodRoutes() intCIDRIPv4, intCIDRIPv6 := util.SplitStringIP(intSubnet.Spec.CIDRBlock) intGatewayIPv4, intGatewayIPv6 := util.SplitStringIP(intSubnet.Spec.Gateway) extGatewayIPv4, extGatewayIPv6 := util.SplitStringIP(extSubnet.Spec.Gateway) + // add routes for the VPC BFD Port so that the egress gateway can establish BFD session(s) with it routes.Add(util.OvnProvider, bfdIPv4, intGatewayIPv4) routes.Add(util.OvnProvider, bfdIPv6, intGatewayIPv6) + // add default routes to forward traffic to the external network routes.Add(extSubnet.Spec.Provider, "0.0.0.0/0", extGatewayIPv4) routes.Add(extSubnet.Spec.Provider, "::/0", extGatewayIPv6) + // generate pod annotations annotations, err := routes.ToAnnotations() if err != nil { klog.Error(err) @@ -299,12 +309,15 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa annotations[util.AttachmentNetworkAnnotation] = attachmentNetworkName annotations[util.LogicalSwitchAnnotation] = intSubnet.Name if len(gw.Spec.InternalIPs) != 0 { + // set internal IPs annotations[util.IPPoolAnnotation] = strings.Join(gw.Spec.InternalIPs, ";") } if len(gw.Spec.ExternalIPs) != 0 { + // set external IPs annotations[fmt.Sprintf(util.IPPoolAnnotationTemplate, extSubnet.Spec.Provider)] = strings.Join(gw.Spec.ExternalIPs, ";") } + // collect egress polcies ipv4ForwardSrc, ipv6ForwardSrc := set.New[string](), set.New[string]() ipv4SNATSrc, ipv6SNATSrc := set.New[string](), set.New[string]() for _, policy := range gw.Spec.Policies { @@ -339,6 +352,7 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa } } + // calculate internal route destinations and SNAT source CIDR blocks intRouteDstIPv4, intRouteDstIPv6 := ipv4ForwardSrc.Union(ipv4SNATSrc), ipv6ForwardSrc.Union(ipv6SNATSrc) intRouteDstIPv4.Insert(bfdIPv4) intRouteDstIPv6.Insert(bfdIPv6) @@ -347,6 +361,8 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa ipv4SNATSrc.Delete("") ipv6SNATSrc.Delete("") + // generate init container environment variables + // the init container is responsible for adding routes and SNAT rules to the pod network namespace initEnv, err := vpcEgressGatewayInitContainerEnv(4, intGatewayIPv4, intCIDRIPv4, intRouteDstIPv4, ipv4SNATSrc) if err != nil { klog.Error(err) @@ -440,16 +456,19 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa }, }, } + // set owner reference so that the workload will be deleted automatically when the vpc egress gateway is deleted if err = util.SetOwnerReference(gw, deploy); err != nil { klog.Error(err) return attachmentNetworkName, nil, nil, nil, err } if bfdIP != "" { + // run BFD in the gateway container to establish BFD session(s) with the VPC BFD LRP container := vpcEgressGatewayContainerBFDD(image, bfdIP, gw.Spec.BFD.MinTX, gw.Spec.BFD.MinRX, gw.Spec.BFD.Multiplier) deploy.Spec.Template.Spec.Containers[0] = container } + // generate hash for the workload to determine whether to update the existing workload or not hash, err := util.Sha256HashObject(deploy) if err != nil { err = fmt.Errorf("failed to hash generated deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) @@ -458,6 +477,7 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa } hash = hash[:12] + // replicas and the hash annotation should be excluded from hash calculation deploy.Spec.Replicas = ptr.To(gw.Spec.Replicas) deploy.Annotations = map[string]string{util.GenerateHashAnnotation: hash} if currentDeploy, err := c.deploymentsLister.Deployments(gw.Namespace).Get(deploy.Name); err != nil { @@ -474,6 +494,7 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa } } else if !reflect.DeepEqual(currentDeploy.Spec.Replicas, deploy.Spec.Replicas) || currentDeploy.Annotations[util.GenerateHashAnnotation] != hash { + // update the deployment if replicas or hash annotation is changed if deploy, err = c.config.KubeClient.AppsV1().Deployments(gw.Namespace). Update(context.Background(), deploy, metav1.UpdateOptions{}); err != nil { err = fmt.Errorf("failed to update deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) @@ -481,9 +502,11 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa return attachmentNetworkName, nil, nil, nil, err } } else { + // no need to create or update the deployment deploy = currentDeploy } + // return the source CIDR blocks excluding BFD LRP IPs for later OVN resources reconciliation intRouteDstIPv4.Delete(bfdIPv4) intRouteDstIPv6.Delete(bfdIPv6) deploy.APIVersion, deploy.Kind = deploymentGroupVersion, deploymentKind @@ -503,6 +526,7 @@ func (c *Controller) reconcileVpcEgressGatewayOVNRoutes(gw *kubeovnv1.VpcEgressG return err } + // reconcile OVN BFD entries bfdIDs := set.New[string]() bfdDstIPs := nextHops.Clone() bfdMap := make(map[string]*string, nextHops.Len()) @@ -715,6 +739,7 @@ func vpcEgressGatewayContainerBFDD(image, bfdIP string, minTX, minRX, multiplier Name: "BFD_MULTI", Value: strconv.Itoa(int(multiplier)), }}, + // wait for the BFD process to be running and initialize the BFD configuration StartupProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ diff --git a/pkg/util/pod_routes.go b/pkg/util/pod_routes.go index 88e75698af8..55274baee2d 100644 --- a/pkg/util/pod_routes.go +++ b/pkg/util/pod_routes.go @@ -13,6 +13,7 @@ import ( type PodProviderRoutes map[string][]string // gateway -> destinations // PodRoutes represents configured routes for all providers/interfaces +// This type is used to generate annotations needed by kube-ovn-cni to configure routes in the pod type PodRoutes map[string]PodProviderRoutes // provider -> PodProviderRoutes func NewPodRoutes() PodRoutes {