From 6427688b5723e2bef59d8258aac30da5c8a800eb Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Thu, 19 Jan 2023 10:54:27 +0800 Subject: [PATCH] ovn db: add support for listening on pod ip (#2235) --- .github/workflows/build-x86-image.yaml | 111 +++++++++++++++++++++++-- Makefile | 13 ++- dist/images/Dockerfile.base | 2 + dist/images/install.sh | 12 +++ dist/images/ovn-is-leader.sh | 14 ++-- dist/images/start-db.sh | 36 +++++--- yamls/ovn-dpdk.yaml | 4 + yamls/ovn-ha.yaml | 4 + yamls/ovn.yaml | 4 + 9 files changed, 177 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 40168f264ff..44d3370f38b 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -584,8 +584,8 @@ jobs: run: sh dist/images/cleanup.sh underlay-logical-gateway-installation-test: - needs: build-kube-ovn name: Underlay Logical Gateway Installation Test + needs: build-kube-ovn runs-on: ubuntu-22.04 timeout-minutes: 30 steps: @@ -620,8 +620,8 @@ jobs: run: sh dist/images/cleanup.sh no-ovn-lb-test: - needs: build-kube-ovn name: Disable OVN LB Test + needs: build-kube-ovn runs-on: ubuntu-22.04 timeout-minutes: 30 steps: @@ -658,8 +658,8 @@ jobs: run: sh dist/images/cleanup.sh no-np-test: - needs: build-kube-ovn name: Disable Network Policy Test + needs: build-kube-ovn runs-on: ubuntu-22.04 timeout-minutes: 30 steps: @@ -696,8 +696,8 @@ jobs: run: sh dist/images/cleanup.sh installation-compatibility-test: - needs: build-kube-ovn name: Installation Compatibility Test + needs: build-kube-ovn runs-on: ubuntu-22.04 timeout-minutes: 10 steps: @@ -732,8 +732,8 @@ jobs: run: sh dist/images/cleanup.sh cilium-chaining-e2e: - needs: build-kube-ovn name: Cilium Chaining E2E + needs: build-kube-ovn runs-on: ubuntu-22.04 timeout-minutes: 30 steps: @@ -819,7 +819,106 @@ jobs: - name: Cleanup run: sh dist/images/cleanup.sh + kube-ovn-security-e2e: + name: Kube-OVN Security E2E + needs: build-kube-ovn + runs-on: ubuntu-22.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + ssl: + - "true" + - "false" + bind-local: + - "true" + - "false" + ip-family: + - ipv4 + - ipv6 + - dual + steps: + - uses: actions/checkout@v3 + + - name: Create the default branch directory + run: mkdir -p test/e2e/source + + - name: Check out the default branch + uses: actions/checkout@v3 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + path: test/e2e/source + + - name: Export E2E directory + run: | + if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then + echo "E2E_DIR=." >> "$GITHUB_ENV" + else + echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" + fi + + - uses: actions/setup-go@v3 + with: + go-version: '${{ env.GO_VERSION }}' + check-latest: true + id: go + + - name: Export Go full version + run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" + + - name: Go cache + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles('${{ env.E2E_DIR }}/**/go.sum') }} + restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- + + - name: Build e2e binaries + working-directory: ${{ env.E2E_DIR }} + run: make e2e-compile + + - name: Install kind + run: | + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + + - name: Download image + uses: actions/download-artifact@v3 + with: + name: kube-ovn + + - name: Load image + run: docker load --input kube-ovn.tar + + - name: Create kind cluster + run: | + sudo pip3 install j2cli + sudo pip3 install "j2cli[yaml]" + sudo PATH=~/.local/bin:$PATH make kind-init-ha-${{ matrix.ip-family }} + sudo cp -r /root/.kube/ ~/.kube/ + sudo chown -R $(id -un). ~/.kube/ + + - name: Install Kube-OVN + run: | + sudo ENABLE_SSL=${{ matrix.ssl }} ENABLE_BIND_LOCAL_IP=${{ matrix.bind-local }} \ + make kind-install-${{ matrix.ip-family }} + + - name: Run E2E + working-directory: ${{ env.E2E_DIR }} + env: + E2E_BRANCH: ${{ github.base_ref || github.ref_name }} + E2E_IP_FAMILY: ${{ matrix.ip-family }} + run: make kube-ovn-security-e2e + + - name: Cleanup + run: sh dist/images/cleanup.sh + push: + name: Push Images needs: - k8s-conformance-e2e # - k8s-netpol-e2e @@ -833,7 +932,7 @@ jobs: - no-ovn-lb-test - no-np-test - cilium-chaining-e2e - name: push + - kube-ovn-security-e2e runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 diff --git a/Makefile b/Makefile index 5c94024ee6c..8d09f865ed1 100644 --- a/Makefile +++ b/Makefile @@ -209,9 +209,20 @@ kind-init-iptables: @kube_proxy_mode=iptables $(MAKE) kind-init .PHONY: kind-init-ha -kind-init-ha: +kind-init-ha: kind-init-ha-ipv4 + +.PHONY: kind-init-ha-ipv4 +kind-init-ha-ipv4: @ha=true $(MAKE) kind-init +.PHONY: kind-init-ha-ipv6 +kind-init-ha-ipv6: + @ip_family=ipv6 $(MAKE) kind-init-ha + +.PHONY: kind-init-ha-dual +kind-init-ha-dual: + @ip_family=dual $(MAKE) kind-init-ha + .PHONY: kind-init-single kind-init-single: @single=true $(MAKE) kind-init diff --git a/dist/images/Dockerfile.base b/dist/images/Dockerfile.base index 9baa28487ca..4522abc18b9 100644 --- a/dist/images/Dockerfile.base +++ b/dist/images/Dockerfile.base @@ -35,6 +35,8 @@ RUN dpkg -i /usr/src/python3-openvswitch*.deb /usr/src/libopenvswitch*.deb RUN cd /usr/src/ && git clone -b branch-21.06 --depth=1 https://github.com/ovn-org/ovn.git && \ cd ovn && \ + # fix ssl listen address + curl -s https://github.com/kubeovn/ovn/commit/62d4969877712c26fe425698d898b440f91b44bf.patch | git apply && \ # expr.c: Use expr_destroy and expr_clone instead of free and xmemdup. curl -s https://github.com/ovn-org/ovn/commit/4b4cadcfabbc79f7d69c213be2b37e5e8634201c.patch | git apply && \ # treewide: bump ovs and fix problematic loops diff --git a/dist/images/install.sh b/dist/images/install.sh index fb8779b5343..6557b8441ae 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -1158,6 +1158,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + - name: ENABLE_BIND_LOCAL_IP + value: "$ENABLE_BIND_LOCAL_IP" resources: requests: cpu: 300m @@ -1635,6 +1641,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + - name: ENABLE_BIND_LOCAL_IP + value: "$ENABLE_BIND_LOCAL_IP" resources: requests: cpu: 300m diff --git a/dist/images/ovn-is-leader.sh b/dist/images/ovn-is-leader.sh index 6bcf1cdfc79..43ea52a8abd 100755 --- a/dist/images/ovn-is-leader.sh +++ b/dist/images/ovn-is-leader.sh @@ -8,12 +8,14 @@ ovn-ctl status_northd ovn-ctl status_ovnnb ovn-ctl status_ovnsb +BIND_LOCAL_ADDR=[${POD_IP:-127.0.0.1}] + # For data consistency, only store leader address in endpoint # Store ovn-nb leader to svc kube-system/ovn-nb if [[ "$ENABLE_SSL" == "false" ]]; then - nb_leader=$(ovsdb-client query tcp:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") + nb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") else - nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") + nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") fi if [[ $nb_leader =~ "true" ]] @@ -34,9 +36,9 @@ fi # Store ovn-sb leader to svc kube-system/ovn-sb if [[ "$ENABLE_SSL" == "false" ]]; then - sb_leader=$(ovsdb-client query tcp:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") + sb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") else - sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") + sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") fi if [[ $sb_leader =~ "true" ]] @@ -51,9 +53,9 @@ then if [ "$northd_leader" == "" ]; then # no available northd leader try to release the lock if [[ "$ENABLE_SSL" == "false" ]]; then - ovsdb-client -v -t 1 steal tcp:127.0.0.1:6642 ovn_northd + ovsdb-client -v -t 1 steal tcp:$BIND_LOCAL_ADDR:6642 ovn_northd else - ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:127.0.0.1:6642 ovn_northd + ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:$BIND_LOCAL_ADDR:6642 ovn_northd fi fi fi diff --git a/dist/images/start-db.sh b/dist/images/start-db.sh index d072e8f46c5..68f87b3e836 100755 --- a/dist/images/start-db.sh +++ b/dist/images/start-db.sh @@ -25,6 +25,14 @@ DB_NB_PORT=${DB_NB_PORT:-6641} DB_SB_ADDR=${DB_SB_ADDR:-::} DB_SB_PORT=${DB_SB_PORT:-6642} ENABLE_SSL=${ENABLE_SSL:-false} +ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-false} +BIND_LOCAL_ADDR=[::] +if [[ $ENABLE_BIND_LOCAL_IP == "true" ]]; then + POD_IPS_LIST=(${POD_IPS//,/ }) + if [[ ${#POD_IPS_LIST[@]} == 1 ]]; then + BIND_LOCAL_ADDR="[${POD_IP}]" + fi +fi . /usr/share/openvswitch/scripts/ovs-lib || exit 1 @@ -177,8 +185,10 @@ if [[ "$ENABLE_SSL" == "false" ]]; then --db-sb-create-insecure-remote=yes \ --db-nb-cluster-local-addr="[${POD_IP}]" \ --db-sb-cluster-local-addr="[${POD_IP}]" \ - --db-nb-addr=[::] \ - --db-sb-addr=[::] \ + --db-nb-addr=$BIND_LOCAL_ADDR \ + --db-sb-addr=$BIND_LOCAL_ADDR \ + --db-nb-use-remote-in-db=no \ + --db-sb-use-remote-in-db=no \ --ovn-northd-nb-db="$(gen_conn_str 6641)" \ --ovn-northd-sb-db="$(gen_conn_str 6642)" \ start_northd @@ -222,8 +232,10 @@ if [[ "$ENABLE_SSL" == "false" ]]; then --db-sb-cluster-local-addr="[${POD_IP}]" \ --db-nb-cluster-remote-addr="[${nb_leader_ip}]" \ --db-sb-cluster-remote-addr="[${sb_leader_ip}]" \ - --db-nb-addr=[::] \ - --db-sb-addr=[::] \ + --db-nb-addr=$BIND_LOCAL_ADDR \ + --db-sb-addr=$BIND_LOCAL_ADDR \ + --db-nb-use-remote-in-db=no \ + --db-sb-use-remote-in-db=no \ --ovn-northd-nb-db="$(gen_conn_str 6641)" \ --ovn-northd-sb-db="$(gen_conn_str 6642)" \ start_northd @@ -277,16 +289,18 @@ else --ovn-northd-ssl-ca-cert=/var/run/tls/cacert \ --db-nb-cluster-local-addr="[${POD_IP}]" \ --db-sb-cluster-local-addr="[${POD_IP}]" \ - --db-nb-addr=[::] \ - --db-sb-addr=[::] \ + --db-nb-addr=$BIND_LOCAL_ADDR \ + --db-sb-addr=$BIND_LOCAL_ADDR \ + --db-nb-use-remote-in-db=no \ + --db-sb-use-remote-in-db=no \ --ovn-northd-nb-db="$(gen_conn_str 6641)" \ --ovn-northd-sb-db="$(gen_conn_str 6642)" \ start_northd - ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":[::] + ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":["${DB_NB_ADDR}"] ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000 ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set NB_Global . options:use_logical_dp_groups=true - ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":[::] + ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":["${DB_SB_ADDR}"] ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000 else # get leader if cluster exists @@ -328,8 +342,10 @@ else --db-sb-cluster-local-addr="[${POD_IP}]" \ --db-nb-cluster-remote-addr="[${nb_leader_ip}]" \ --db-sb-cluster-remote-addr="[${sb_leader_ip}]" \ - --db-nb-addr=[::] \ - --db-sb-addr=[::] \ + --db-nb-addr=$BIND_LOCAL_ADDR \ + --db-sb-addr=$BIND_LOCAL_ADDR \ + --db-nb-use-remote-in-db=no \ + --db-sb-use-remote-in-db=no \ --ovn-northd-nb-db="$(gen_conn_str 6641)" \ --ovn-northd-sb-db="$(gen_conn_str 6642)" \ start_northd diff --git a/yamls/ovn-dpdk.yaml b/yamls/ovn-dpdk.yaml index 3b79672cbe9..d6a3d0e1995 100644 --- a/yamls/ovn-dpdk.yaml +++ b/yamls/ovn-dpdk.yaml @@ -244,6 +244,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs resources: requests: cpu: 500m diff --git a/yamls/ovn-ha.yaml b/yamls/ovn-ha.yaml index 268b2fc45c0..5adecaf3144 100644 --- a/yamls/ovn-ha.yaml +++ b/yamls/ovn-ha.yaml @@ -263,6 +263,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs resources: requests: cpu: 500m diff --git a/yamls/ovn.yaml b/yamls/ovn.yaml index ad50ccb02f3..1302659a916 100644 --- a/yamls/ovn.yaml +++ b/yamls/ovn.yaml @@ -283,6 +283,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs resources: requests: cpu: 500m