From 796947b90d879210ba26a145fdc4ca828e60d71a Mon Sep 17 00:00:00 2001 From: npolshakova Date: Mon, 17 Jun 2024 14:13:48 -0400 Subject: [PATCH 1/3] wip: remove old istio tests --- .../composite-actions/istio-tests/action.yaml | 45 - .github/workflows/istio-tests.yaml | 65 -- .github/workflows/regression-tests.yaml | 2 +- .../v1.18.0-beta1/remove-old-istio-tests.yaml | 4 + ci/kind/setup-kind.sh | 15 - test/kube2e/glooctl/glooctl_suite_test.go | 2 - test/kube2e/glooctl/istio_test.go | 246 ----- test/kube2e/istio/README.md | 287 ------ .../kube2e/istio/artifacts/automtls-helm.yaml | 40 - test/kube2e/istio/artifacts/gateway.yaml | 14 - .../istio/artifacts/gloo-gateway-helm.yaml | 47 - test/kube2e/istio/artifacts/helm.yaml | 47 - test/kube2e/istio/artifacts/httpbin.yaml | 60 -- .../istio/artifacts/peerauth_disable.yaml | 8 - .../istio/artifacts/peerauth_permissive.yaml | 8 - .../istio/artifacts/peerauth_strict.yaml | 8 - test/kube2e/istio/istio_integration_test.go | 951 ------------------ test/kube2e/istio/istio_suite_test.go | 263 ----- 18 files changed, 5 insertions(+), 2107 deletions(-) delete mode 100644 .github/workflows/composite-actions/istio-tests/action.yaml delete mode 100644 .github/workflows/istio-tests.yaml create mode 100644 changelog/v1.18.0-beta1/remove-old-istio-tests.yaml delete mode 100644 test/kube2e/glooctl/istio_test.go delete mode 100644 test/kube2e/istio/README.md delete mode 100644 test/kube2e/istio/artifacts/automtls-helm.yaml delete mode 100644 test/kube2e/istio/artifacts/gateway.yaml delete mode 100644 test/kube2e/istio/artifacts/gloo-gateway-helm.yaml delete mode 100644 test/kube2e/istio/artifacts/helm.yaml delete mode 100644 test/kube2e/istio/artifacts/httpbin.yaml delete mode 100644 test/kube2e/istio/artifacts/peerauth_disable.yaml delete mode 100644 test/kube2e/istio/artifacts/peerauth_permissive.yaml delete mode 100644 test/kube2e/istio/artifacts/peerauth_strict.yaml delete mode 100644 test/kube2e/istio/istio_integration_test.go delete mode 100644 test/kube2e/istio/istio_suite_test.go diff --git a/.github/workflows/composite-actions/istio-tests/action.yaml b/.github/workflows/composite-actions/istio-tests/action.yaml deleted file mode 100644 index 94602edb206..00000000000 --- a/.github/workflows/composite-actions/istio-tests/action.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Gloo Istio Regression Tests - -description: Tests which run Gloo Edge in a Kubernetes cluster with Istio - -runs: - using: "composite" - steps: - - name: Prep Go Runner - uses: ./.github/workflows/composite-actions/prep-go-runner - - name: Install kind - uses: helm/kind-action@v1.10.0 - with: - install_only: true - version: ${{ matrix.kube-version.kind }} - - uses: azure/setup-kubectl@v4 - id: kubectl - with: - version: ${{ matrix.kube-version.kubectl }} - - uses: azure/setup-helm@v4 - with: - version: ${{ matrix.kube-version.helm }} - - name: Setup test env - shell: bash - env: - KUBE2E_TESTS: istio - CLUSTER_NAME: 'kind' - CLUSTER_NODE_VERSION: ${{ matrix.kube-version.node }} - run: ./ci/kind/setup-kind.sh - - name: Testing - kube e2e regression tests - env: - KUBE2E_TESTS: istio - shell: bash - run: | - if [[ "${{ matrix.kube-e2e-test-type }}" == "gloo-gateway" ]]; then - export GLOO_GATEWAY_SETUP=true - elif [[ "${{ matrix.kube-e2e-test-type }}" == "istio-auto" ]]; then - export ISTIO_AUTO_MTLS=true - fi - make install-test-tools run-kube-e2e-tests - - uses: actions/upload-artifact@v4 - if: ${{ failure() }} - with: - name: ${{matrix.kube-e2e-test-type}}@k8s${{matrix.kube-version.kubectl}}-kube-dump - path: "_output/kube2e-artifacts" - if-no-files-found: warn \ No newline at end of file diff --git a/.github/workflows/istio-tests.yaml b/.github/workflows/istio-tests.yaml deleted file mode 100644 index f13f0a20941..00000000000 --- a/.github/workflows/istio-tests.yaml +++ /dev/null @@ -1,65 +0,0 @@ -name: Istio Tests -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -env: - VERSION: '1.0.0-ci1' - GITHUB_TOKEN: ${{ github.token }} # necessary to pass upgrade tests - -jobs: - prepare_env: - name: Prepare Environment - runs-on: ubuntu-22.04 - timeout-minutes: 5 - outputs: - should-auto-succeed-istio-tests: ${{ steps.run-strategy.outputs.auto_succeed }} - steps: - - name: Cancel Previous Actions - uses: styfle/cancel-workflow-action@0.12.1 - with: - access_token: ${{ github.token }} - - id: checkout-code - uses: actions/checkout@v4 - with: - # We require gathering the branch and tag history since we rely on a `git diff` - # which compares the state of two branches - fetch-depth: 0 - - id: process-skip-directives - uses: ./.github/workflows/composite-actions/process-skip-directives - with: - base-ref: ${{ github.base_ref }} - - id: run-strategy - name: Determine Test Run Strategy - run: | - skip_kube_tests=${{ steps.process-skip-directives.outputs.skip-kube-tests }} - if [[ ! -z $skip_kube_tests && $skip_kube_tests = true ]] ; then - echo "auto_succeed=true" >> $GITHUB_OUTPUT - fi - - # TODO(npolshak): Add multiple istio versions to test matrix - istio_tests: - name: istio integration tests - needs: prepare_env - runs-on: ubuntu-22.04 - timeout-minutes: 60 - if: ${{ !github.event.pull_request.draft }} - strategy: - fail-fast: false - matrix: - kube-e2e-test-type: [ 'istio-auto', 'gloo-gateway' ] - kube-version: [ { node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245', kubectl: 'v1.29.2', kind: 'v0.20.0', helm: 'v3.14.4' } ] - merge-to-main: - - ${{ github.event.pull_request.base.ref == 'main' }} - exclude: - - merge-to-main: true - kube-e2e-test-type: upgrade - steps: - - uses: actions/checkout@v4 - - id: run-tests - if: needs.prepare_env.outputs.should-auto-succeed-istio-tests != 'true' - uses: ./.github/workflows/composite-actions/istio-tests - - id: auto-succeed-tests - if: needs.prepare_env.outputs.should-auto-succeed-istio-tests == 'true' - run: | - echo "Istio tests auto-succeeded" \ No newline at end of file diff --git a/.github/workflows/regression-tests.yaml b/.github/workflows/regression-tests.yaml index 367e8d7da12..5ad77987100 100644 --- a/.github/workflows/regression-tests.yaml +++ b/.github/workflows/regression-tests.yaml @@ -49,7 +49,7 @@ jobs: # upgrade tests are run on LTS but not on main branch, for main they are run nightly # ingress will be deprecated from 1.17. Ref: https://solo-io-corp.slack.com/archives/G01EERAK3KJ/p1716389614777799 # this is the github action version of ternary op - kube-e2e-test-type: [ 'gateway', 'gloo', 'ingress', 'helm', 'gloomtls', 'glooctl', 'upgrade', 'istio' ] + kube-e2e-test-type: [ 'gateway', 'gloo', 'ingress', 'helm', 'gloomtls', 'glooctl', 'upgrade' ] kube-version: [ { node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245', kubectl: 'v1.29.2', kind: 'v0.20.0', helm: 'v3.14.4' } ] image-variant: - distroless diff --git a/changelog/v1.18.0-beta1/remove-old-istio-tests.yaml b/changelog/v1.18.0-beta1/remove-old-istio-tests.yaml new file mode 100644 index 00000000000..a0c06f2696a --- /dev/null +++ b/changelog/v1.18.0-beta1/remove-old-istio-tests.yaml @@ -0,0 +1,4 @@ +changelog: + - type: NON_USER_FACING + description: >- + Remove old Istio integration regression e2e tests and glooctl Istio integration e2e tests. \ No newline at end of file diff --git a/ci/kind/setup-kind.sh b/ci/kind/setup-kind.sh index 4b126944ab6..ca0b224d96c 100755 --- a/ci/kind/setup-kind.sh +++ b/ci/kind/setup-kind.sh @@ -15,8 +15,6 @@ SKIP_DOCKER="${SKIP_DOCKER:-false}" JUST_KIND="${JUST_KIND:-false}" # Offer a default value for type of installation KUBE2E_TESTS="${KUBE2E_TESTS:-gateway}" # If 'KUBE2E_TESTS' not set or null, use 'gateway'. -# The version of istio to install for glooctl tests. This should get set by the 'setup-kind-cluster' github action, where it is a required input. -ISTIO_VERSION="${ISTIO_VERSION:-1.22.0}" # Set the default image variant to standard IMAGE_VARIANT="${IMAGE_VARIANT:-standard}" # If true, run extra steps to set up k8s gateway api conformance test environment @@ -103,16 +101,3 @@ if [[ $CONFORMANCE == "true" ]]; then - address-pool EOF fi - -# 7. Install additional resources used for particular KUBE2E tests -if [[ $KUBE2E_TESTS = "glooctl" || $KUBE2E_TESTS = "istio" ]]; then - TARGET_ARCH=x86_64 - if [[ $ARCH == 'arm64' ]]; then - TARGET_ARCH=arm64 - fi - echo "Downloading Istio $ISTIO_VERSION" - curl -L https://istio.io/downloadIstio | ISTIO_VERSION=$ISTIO_VERSION TARGET_ARCH=$TARGET_ARCH sh - - - echo "Installing Istio" - yes | "./istio-$ISTIO_VERSION/bin/istioctl" install --set profile=minimal -fi diff --git a/test/kube2e/glooctl/glooctl_suite_test.go b/test/kube2e/glooctl/glooctl_suite_test.go index 1ca4a6005c2..3a0fbb5bcb9 100644 --- a/test/kube2e/glooctl/glooctl_suite_test.go +++ b/test/kube2e/glooctl/glooctl_suite_test.go @@ -52,8 +52,6 @@ func StartTestHelper() { var err error testHelper, err = kube2e.GetTestHelper(ctx, namespace) Expect(err).NotTo(HaveOccurred()) - // Register additional fail handlers - skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, metav1.ObjectMeta{Namespace: "istio-system"}, metav1.ObjectMeta{Namespace: testHelper.InstallNamespace})) if !testutils.ShouldSkipInstall() { installGloo() diff --git a/test/kube2e/glooctl/istio_test.go b/test/kube2e/glooctl/istio_test.go deleted file mode 100644 index aee1e68ef01..00000000000 --- a/test/kube2e/glooctl/istio_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package glooctl_test - -import ( - "net/http" - "time" - - "github.com/solo-io/gloo/projects/gateway/pkg/defaults" - "github.com/solo-io/gloo/test/kube2e/helper" - "github.com/solo-io/go-utils/testutils/exec" - "github.com/solo-io/skv2/codegen/util" - "github.com/solo-io/solo-kit/pkg/api/v1/clients" - - "path/filepath" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var ( - petstoreYaml = filepath.Join(util.GetModuleRoot(), "example", "petstore", "petstore.yaml") - petstoreCurlOpts = helper.CurlOpts{ - Protocol: "http", - Path: "/api/pets", - Method: http.MethodGet, - Host: defaults.GatewayProxyName, - Service: defaults.GatewayProxyName, - Verbose: true, - Port: 80, - ConnectionTimeout: 1, - WithoutStats: true, - Headers: map[string]string{ - "Cache-Control": "no-cache", - }, - } - petstoreSuccessfulResponse = `[{"id":1,"name":"Dog","status":"available"},{"id":2,"name":"Cat","status":"pending"}]` -) - -var _ = Describe("Istio", Ordered, func() { - - BeforeEach(func() { - // These tests are known to be inconsistent, and cause toil in our CI pipeline - // https://github.com/solo-io/solo-projects/issues/6048 tracks the work to re-enable these tests - Skip("These tests are inconsistent. Temporarily disabling") - }) - - // Tests for: `glooctl istio [..]` - // These tests assume that Gloo and Istio are pre-installed in the cluster - - BeforeAll(func() { - err := exec.RunCommand(testHelper.RootDir, false, "kubectl", "apply", "-f", petstoreYaml) - Expect(err).NotTo(HaveOccurred(), "should be able to install petstore") - - _, err = GlooctlOut("add", "route", "--name", "petstore", "--namespace", testHelper.InstallNamespace, "--path-prefix", "/", "--dest-name", "default-petstore-8080", "--dest-namespace", testHelper.InstallNamespace) - Expect(err).NotTo(HaveOccurred(), "should be able to add gloo route to petstore") - - err = exec.RunCommand(testHelper.RootDir, false, "kubectl", "label", "namespace", "default", "istio-injection=enabled", "--overwrite") - Expect(err).NotTo(HaveOccurred(), "should be able to add a label to enable istio injection") - }) - - AfterAll(func() { - err := exec.RunCommand(testHelper.RootDir, false, "kubectl", "label", "namespace", "default", "istio-injection-") - Expect(err).NotTo(HaveOccurred(), "should be able to remove the istio injection label") - - err = exec.RunCommand(testHelper.RootDir, false, "kubectl", "delete", "-f", petstoreYaml) - Expect(err).NotTo(HaveOccurred(), "should be able to uninstall petstore") - - err = exec.RunCommand(testHelper.RootDir, false, "kubectl", "delete", "vs", "petstore", "-n", testHelper.InstallNamespace) - Expect(err).NotTo(HaveOccurred(), "should be able to delete the petstore VS") - - Eventually(func(g Gomega) { - virtualServices, err := resourceClientset.VirtualServiceClient().List(testHelper.InstallNamespace, clients.ListOpts{ - Ctx: ctx, - }) - g.Expect(err).NotTo(HaveOccurred(), "should be able to list virtual services") - g.Expect(virtualServices).To(BeEmpty(), "should have no virtual services") - }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) - }) - - EventuallyIstioInjected := func() { - trueOffset := 1 - EventuallyWithOffset(trueOffset, func(g Gomega) { - // Check for sds sidecar - sdsContainer, err := exec.RunCommandOutput(testHelper.RootDir, false, "kubectl", "get", "-n", testHelper.InstallNamespace, "deployments", "gateway-proxy", "-o", `jsonpath='{.spec.template.spec.containers[?(@.name == "sds")].name}'`) - g.Expect(sdsContainer).To(Equal("'sds'"), "sds container should be present after injection") - g.Expect(err).NotTo(HaveOccurred(), "should be able to kubectl get the gateway-proxy containers") - - // Check for istio-proxy sidecar - istioContainer, err := exec.RunCommandOutput(testHelper.RootDir, false, "kubectl", "get", "-n", testHelper.InstallNamespace, "deployments", "gateway-proxy", "-o", `jsonpath='{.spec.template.spec.containers[?(@.name == "istio-proxy")].name}'`) - g.Expect(istioContainer).To(Equal("'istio-proxy'"), "istio-proxy container should be present after injection") - g.Expect(err).NotTo(HaveOccurred()) - - // Check for configMap changes - configMapEnvoyYAML, err := exec.RunCommandOutput(testHelper.RootDir, false, "kubectl", "get", "-n", testHelper.InstallNamespace, "configmaps", "gateway-proxy-envoy-config", "-o", `jsonpath='{.data}'`) - g.Expect(configMapEnvoyYAML).To(ContainSubstring("clusterName: gateway_proxy_sds")) - g.Expect(err).NotTo(HaveOccurred(), "should be able to kubectl get the gateway-proxy containers") - }, time.Second*10, time.Second).ShouldNot(HaveOccurred(), "eventually istio injected") - } - - EventuallyIstioUninjected := func() { - trueOffset := 1 - EventuallyWithOffset(trueOffset, func(g Gomega) { - // Check for sds sidecar - sdsContainer, err := exec.RunCommandOutput(testHelper.RootDir, false, "kubectl", "get", "-n", testHelper.InstallNamespace, "deployments", "gateway-proxy", "-o", `jsonpath='{.spec.template.spec.containers[?(@.name == "sds")].name}'`) - g.Expect(sdsContainer).To(Equal("''"), "sds container should be removed after uninjection") - g.Expect(err).NotTo(HaveOccurred(), "should be able to kubectl get the gateway-proxy containers") - - // Check for istio-proxy sidecar - istioContainer, err := exec.RunCommandOutput(testHelper.RootDir, false, "kubectl", "get", "-n", testHelper.InstallNamespace, "deployments", "gateway-proxy", "-o", `jsonpath='{.spec.template.spec.containers[?(@.name == "istio-proxy")].name}'`) - g.Expect(istioContainer).To(Equal("''"), "istio-proxy container should be removed after uninjection") - g.Expect(err).NotTo(HaveOccurred()) - - // Check for configMap changes - configMapEnvoyYAML, err := exec.RunCommandOutput(testHelper.RootDir, false, "kubectl", "get", "-n", testHelper.InstallNamespace, "configmaps", "gateway-proxy-envoy-config", "-o", `jsonpath='{.data}'`) - g.Expect(configMapEnvoyYAML).NotTo(ContainSubstring("clusterName: gateway_proxy_sds"), "gateway_proxy_sds cluster should be removed after uninject") - g.Expect(err).NotTo(HaveOccurred(), "should be able to kubectl get the gateway-proxy containers") - }, time.Second*10, time.Second).ShouldNot(HaveOccurred(), "eventually istio uninjected") - - } - - Context("inject", func() { - - BeforeEach(func() { - // We are assuming to be working from a clean slate, so there is no need to set anything up - }) - - AfterEach(func() { - _, err := GlooctlOut("istio", "uninject", "--namespace", testHelper.InstallNamespace, "--include-upstreams=true") - Expect(err).NotTo(HaveOccurred(), "should be able to run 'glooctl istio uninject' without errors") - - EventuallyIstioUninjected() - }) - - It("works on gateway-pod", func() { - testHelper.CurlEventuallyShouldRespond(petstoreCurlOpts, petstoreSuccessfulResponse, 0, 60*time.Second, 1*time.Second) - - _, err := GlooctlOut("istio", "inject", "--namespace", testHelper.InstallNamespace) - Expect(err).NotTo(HaveOccurred(), "should be able to run 'glooctl istio inject' without errors") - - EventuallyIstioInjected() - - _, err = GlooctlOut("istio", "enable-mtls", "--upstream", "default-petstore-8080", "-n", testHelper.InstallNamespace) - Expect(err).NotTo(HaveOccurred(), "should be able to enable mtls on the petstore upstream via sslConfig") - - err = toggleStictModePetstore(true) - Expect(err).NotTo(HaveOccurred(), "should be able to enable mtls strict mode on the petstore app") - - testHelper.CurlEventuallyShouldRespond(petstoreCurlOpts, petstoreSuccessfulResponse, 0, 60*time.Second, 1*time.Second) - }) - - }) - - Context("uninject (success)", func() { - - BeforeEach(func() { - testHelper.CurlEventuallyShouldRespond(petstoreCurlOpts, petstoreSuccessfulResponse, 1, 10*time.Second, 1*time.Second) - - _, err := GlooctlOut("istio", "inject", "--namespace", testHelper.InstallNamespace) - Expect(err).NotTo(HaveOccurred(), "should be able to run 'glooctl istio inject' without errors") - - EventuallyIstioInjected() - - err = toggleStictModePetstore(false) - Expect(err).NotTo(HaveOccurred(), "should be able to disable mtls strict mode on the petstore app") - }) - - AfterEach(func() { - // We are assuming each test to uninject correctly, so there is nothing to clean up - }) - - When("no upstreams contain sds configuration", func() { - - It("succeeds", func() { - _, err := GlooctlOut("istio", "uninject", "--namespace", testHelper.InstallNamespace) - Expect(err).NotTo(HaveOccurred(), "should be able to run 'glooctl istio uninject' without errors") - - EventuallyIstioUninjected() - - // Expect it to work - testHelper.CurlEventuallyShouldRespond(petstoreCurlOpts, petstoreSuccessfulResponse, 1, 60*time.Second, 1*time.Second) - }) - - }) - - When("upstreams contain sds configuration and --include-upstreams=true", func() { - - It("succeeds", func() { - _, err := GlooctlOut("istio", "enable-mtls", "--upstream", "default-petstore-8080", "-n", testHelper.InstallNamespace) - Expect(err).NotTo(HaveOccurred(), "should be able to enable mtls on the petstore upstream via sslConfig") - - _, err = GlooctlOut("istio", "uninject", "--namespace", testHelper.InstallNamespace, "--include-upstreams=true") - Expect(err).NotTo(HaveOccurred(), "should be able to run 'glooctl istio uninject' without errors") - - EventuallyIstioUninjected() - - // Expect it to work - testHelper.CurlEventuallyShouldRespond(petstoreCurlOpts, petstoreSuccessfulResponse, 1, 60*time.Second, 1*time.Second) - }) - }) - - }) - - Context("uninject (failure)", func() { - - BeforeEach(func() { - testHelper.CurlEventuallyShouldRespond(petstoreCurlOpts, petstoreSuccessfulResponse, 1, 10*time.Second, 1*time.Second) - - _, err := GlooctlOut("istio", "inject", "--namespace", testHelper.InstallNamespace) - Expect(err).NotTo(HaveOccurred(), "should be able to run 'glooctl istio inject' without errors") - - EventuallyIstioInjected() - - err = toggleStictModePetstore(false) - Expect(err).NotTo(HaveOccurred(), "should be able to enable mtls strict mode on the petstore app") - }) - - AfterEach(func() { - _, err := GlooctlOut("istio", "uninject", "--namespace", testHelper.InstallNamespace, "--include-upstreams=true") - Expect(err).NotTo(HaveOccurred(), "should be able to run 'glooctl istio uninject' without errors") - - EventuallyIstioUninjected() - }) - - When("upstreams contain sds configuration and --include-upstreams=false", func() { - - It("fails", func() { - _, err := GlooctlOut("istio", "enable-mtls", "--upstream", "default-petstore-8080", "-n", testHelper.InstallNamespace) - Expect(err).NotTo(HaveOccurred(), "should be able to enable mtls on the petstore upstream via sslConfig") - - _, err = GlooctlOut("istio", "uninject", "--namespace", testHelper.InstallNamespace, "--include-upstreams=false") - Expect(err).To(HaveOccurred(), "should not be able to run 'glooctl istio uninject' without errors") - }) - - }) - - }) - -}) - -func toggleStictModePetstore(strictModeEnabled bool) error { - yamlPath := testHelper.RootDir + "/test/kube2e/glooctl/petstore_peerauth_permissive.yaml" - if strictModeEnabled { - yamlPath = testHelper.RootDir + "/test/kube2e/glooctl/petstore_peerauth_strict.yaml" - } - return exec.RunCommand(testHelper.RootDir, false, "kubectl", "apply", "-f", yamlPath) - -} diff --git a/test/kube2e/istio/README.md b/test/kube2e/istio/README.md deleted file mode 100644 index c4cdd64f7c1..00000000000 --- a/test/kube2e/istio/README.md +++ /dev/null @@ -1,287 +0,0 @@ -An overview of some key concepts related to this test suite: - -# mTLS -[mTLS explained](https://www.cloudflare.com/learning/access-management/what-is-mutual-tls/) - -[mTLS in Envoy](https://www.envoyproxy.io/docs/envoy/latest/start/quick-start/securing#use-mutual-tls-mtls-to-enforce-client-certificate-authentication) - -# Gloo mTLS -[Background](https://docs.solo.io/gloo-edge/latest/guides/security/tls/mtls/) - -In Gloo Edge, services that receive their configuration from the xDS server, communicate using the xDS protocol, which is done in plaintext. Since the information being provided contains sensitive information (secrets), it is preferable to encrypt this traffic. This applies to the following services: -- Gateway-proxy -- Ext-auth-service -- Rate-limiter - -## How does this work? -Each component involved in the xDS communication will contain an SDS sidecar and potentially an Envoy sidecar as well (for our gateway-proxy pod, which already contains an envoy container, this Envoy sidecar is not necessary) - -**Envoy Sidecar**: Responsible for TLS termination, and outgoing encryption. Receives it’s secret information via SDS, talking to the SDS sidecar. The sidecar bootstrap configuration is defined in a ConfigMap for each component. - -**SDS Sidecar**: Watches the gloo-mtls-certs TLS secret in kubernetes, and serves the contents via the SDS API to the Envoy sidecar. The SDS code can be found [here](https://github.com/solo-io/gloo/tree/main/projects/sds). Anytime the secret is modified, the contents will be updated dynamically. To automate the updating of secrets, run the [CertGen Job](https://github.com/solo-io/gloo/blob/main/install/helm/gloo/templates/19-gloo-mtls-certgen-job.yaml) - - -### Example: Gloo <-> ExtAuth -[ExtAuth Envoy Sidecar](https://github.com/solo-io/solo-projects/blob/main/install/helm/gloo-ee/templates/24-extauth-sidecar-config.yaml) - -- ExtAuth pod initiates xDS request to Gloo -- ExtAuth.EnvoySidecar hijacks request, and sends encrypted request to Gloo -- Gloo.EnvoySidecar terminates TLS request from ExtAuth -- Gloo.EnvoySidecar directs request to Gloo xDS port - -# Istio mTLS - -[Background](https://docs.solo.io/gloo-edge/latest/guides/integrations/service_mesh/istio/) - -When services are deployed within the Istio Service Mesh, it is possible to require that all traffic between those services is encrypted with mTLS. This is handled dynamically by Istio managing certs for services within the Mesh. However, it poses a challenge when Gloo Edge (not in the Mesh) tries to route traffic to services within the Mesh. - -This challenge is that Edge encrypts traffic using the certificates that were generated by IstioD and used within the Mesh. To accomplish this, Edge uses a similar paradigm as the Gloo mTLS solution, leveraging sidecars to encrypt traffic. - -## How does this work? -To keep the mTLS communication transparent to the Edge configuration, we again leverage sidecars to handle the encryption. - -[Istio](https://istio.io/latest/docs/ops/deployment/architecture/) is logically split into a data plane and a control plane. The data plane is composed of a set of intelligent proxies (Envoy) deployed as sidecars. The control plane (istiod) provides service discovery, configuration and certificate management. -Istiod acts as a Certificate Authority (CA) and generates certificates to allow secure mTLS communication in the data plane. - -**Istio-Proxy Sidecar**: This is responsible for generating the certificates used for mTLS communication. These certificates are mounted to a volume, which are then provided to the gateway-proxy configuration via SDS. The gateway-proxy, with these certificates, is now able to establish mTLS communication with an upstream in the Mesh. -The Istio-Proxy Sidecar usually runs both the istio-agent and envoy. The [istio-agent](https://github.com/istio/istio/blob/master/architecture/security/istio-agent.md) is responsible for generating the certificates, and the envoy is responsible for terminating TLS and establishing mTLS communication with other services in the Mesh. -By default, the istio-proxy will run an extended version of the Envoy proxy. However, we don't need the istio-proxy Envoy functionality, we only need the istio-agent to create the CSR request to istiod and handle rotating certificates near expiration. -To avoid running the istio-proxy Envoy, we can set the `DISABLE_ENVOY` environment variable. This will cause the istio-proxy to run in proxyless mode and not start the Envoy process. - - - -**SDS Sidecar**: Watches the Istio certs, generated by the istio-proxy sidecar, and serves them up via SDS to the gateway-proxy. The SDS code can be found [here](https://github.com/solo-io/gloo/tree/main/projects/sds). Any time the volume is modified, the contents will be updated dynamically. This is done because Istio rotates its certificates by default, every 24 hours (can be controlled by env variable). Without this SDS sidecar, the proxies would use outdated certificates, and when new ones are rolled out by Istio, our gateway-proxy wouldn’t be able to communicate with services using the new certificates. - -# SDS - -Envoy provides an API for dynamically updating TLS certificates. Historically, this required restarting the proxies with the new certificates, but the [SDS API](https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret) allows a zero-downtime approach to this. - -In Gloo Edge, we build an SDS server which implements this API. Its sole responsibility is to implement the SDS API, and serve up certificates via xDS. We configure our Envoy proxies as the client to this server, and the proxies are [configured to get their certificates via the SDS API](https://github.com/solo-io/gloo/blob/main/install/helm/gloo/templates/9-gateway-proxy-configmap.yaml#L195) - -While the component itself has a single responsibility, implement the SDS API, it can be used in Gloo Edge in multiple ways: -- Serve certificates to establish mTLS communication between Gloo components (Gloo mTLS) -- Serve certificates to establish mLTS communication between Gateway-Proxy and Application running in Service Mesh (Istio mTLS) - -_As a result, we have `glooMtls.enabled` to enable the former, and `istioIntegration.enabled` to enable the latter._ - -Note, the current Gloo SDS does not reach out to Istiod. The istio-agent is responsible for sending the CSR to Istiod. -The SDS server then reads the certs from a file written by the istio-agent and then SDS serves the certificates to the Gloo Envoy proxy. - -# Validating mTLS Traffic - -Istio leverages the [`x-forwarded-client-cert`](https://istio.io/latest/docs/ops/configuration/traffic-management/network-topologies/#forwarding-external-client-attributes-ip-address-certificate-info-to-destination-workloads) header to identify encrypted traffic - -If the application that we’re running can logs requests that it receives, we could search the logs for the existence of that header - -# Testing automtls - -The istio e2e integration tests automtls functionality with Gloo Edge "classic" APIs and k8s Gateway API resources. This -can be manually tested by following the steps below on a kind cluster: - -1. Setup environment and kind cluster - -```shell -ci/kind/setup-kind.sh; make kind-build-and-load -``` - -2. Install Istio - -```shell -./projects/gateway2/istio.sh -``` - -```shell -cat < Date: Mon, 17 Jun 2024 15:41:08 -0400 Subject: [PATCH 2/3] add regression test, add edge gw api port routing test --- .github/workflows/pr-kubernetes-tests.yaml | 2 +- .../port_routing/edge_gateway_suite.go | 177 ++++++++++++++++++ .../{suite.go => k8s_gateway_suite.go} | 18 +- .../port_routing/testdata/edge-routing.yaml | 16 ++ .../invalid-port-and-invalid-targetport.yaml | 11 ++ .../invalid-port-and-valid-targetport.yaml | 11 ++ .../edge/invalid-port-without-targetport.yaml | 11 ++ .../match-pod-port-without-targetport.yaml | 11 ++ .../edge/match-port-and-targetport.yaml | 11 ++ .../port_routing/testdata/k8s-gw.yaml | 13 ++ .../invalid-port-and-invalid-targetport.yaml | 13 -- .../invalid-port-and-valid-targetport.yaml | 12 -- .../invalid-port-without-targetport.yaml | 11 -- .../match-pod-port-without-targetport.yaml | 11 -- .../{ => k8s}/match-port-and-targetport.yaml | 12 -- .../features/port_routing/testdata/setup.yaml | 14 -- .../invalid-port-and-invalid-targetport.yaml | 12 ++ .../invalid-port-and-valid-targetport.yaml | 11 ++ .../svc/invalid-port-without-targetport.yaml | 10 + .../match-pod-port-without-targetport.yaml | 10 + .../svc/match-port-and-targetport.yaml | 11 ++ .../e2e/features/port_routing/types.go | 32 +++- .../tests/automtls_istio_edge_api_tests.go | 2 + .../e2e/tests/automtls_istio_tests.go | 2 +- test/kubernetes/e2e/tests/edge_gw_tests.go | 2 + .../e2e/tests/istio_edge_api_tests.go | 2 + .../e2e/tests/istio_regression_test.go | 73 ++++++++ .../e2e/tests/istio_regression_tests.go | 18 ++ .../{istio_test.go => k8s_gw_istio_test.go} | 0 .../{istio_tests.go => k8s_gw_istio_tests.go} | 2 +- .../e2e/tests/k8s_gw_no_validation_tests.go | 2 +- test/kubernetes/e2e/tests/k8s_gw_tests.go | 2 +- .../manifests/istio-regression-helm.yaml | 47 +++++ 33 files changed, 502 insertions(+), 90 deletions(-) create mode 100644 test/kubernetes/e2e/features/port_routing/edge_gateway_suite.go rename test/kubernetes/e2e/features/port_routing/{suite.go => k8s_gateway_suite.go} (80%) create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/edge-routing.yaml create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-and-invalid-targetport.yaml create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-and-valid-targetport.yaml create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-without-targetport.yaml create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/edge/match-pod-port-without-targetport.yaml create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/edge/match-port-and-targetport.yaml create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/k8s-gw.yaml rename test/kubernetes/e2e/features/port_routing/testdata/{ => k8s}/invalid-port-and-invalid-targetport.yaml (50%) rename test/kubernetes/e2e/features/port_routing/testdata/{ => k8s}/invalid-port-and-valid-targetport.yaml (56%) rename test/kubernetes/e2e/features/port_routing/testdata/{ => k8s}/invalid-port-without-targetport.yaml (55%) rename test/kubernetes/e2e/features/port_routing/testdata/{ => k8s}/match-pod-port-without-targetport.yaml (53%) rename test/kubernetes/e2e/features/port_routing/testdata/{ => k8s}/match-port-and-targetport.yaml (52%) create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-and-invalid-targetport.yaml create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-and-valid-targetport.yaml create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-without-targetport.yaml create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/svc/match-pod-port-without-targetport.yaml create mode 100644 test/kubernetes/e2e/features/port_routing/testdata/svc/match-port-and-targetport.yaml create mode 100644 test/kubernetes/e2e/tests/istio_regression_test.go create mode 100644 test/kubernetes/e2e/tests/istio_regression_tests.go rename test/kubernetes/e2e/tests/{istio_test.go => k8s_gw_istio_test.go} (100%) rename test/kubernetes/e2e/tests/{istio_tests.go => k8s_gw_istio_tests.go} (89%) create mode 100644 test/kubernetes/e2e/tests/manifests/istio-regression-helm.yaml diff --git a/.github/workflows/pr-kubernetes-tests.yaml b/.github/workflows/pr-kubernetes-tests.yaml index c0cbbed272c..1ecc62f1c9d 100644 --- a/.github/workflows/pr-kubernetes-tests.yaml +++ b/.github/workflows/pr-kubernetes-tests.yaml @@ -64,7 +64,7 @@ jobs: # May 14th: ~ minutes execution time (see load_balancing_tests.md) - cluster-name: 'cluster-three' go-test-args: '-v -timeout=25m' - go-test-run-regex: '(^TestK8sGatewayIstioAutoMtls$$|^TestAutomtlsIstioEdgeApisGateway$$|^TestIstioEdgeApiGateway$$)' + go-test-run-regex: '(^TestK8sGatewayIstioAutoMtls$$|^TestAutomtlsIstioEdgeApisGateway$$|^TestIstioEdgeApiGateway$$|^TestIstioRegression$$)' # May 14th: ~ minutes execution time (see load_balancing_tests.md) - cluster-name: 'cluster-four' diff --git a/test/kubernetes/e2e/features/port_routing/edge_gateway_suite.go b/test/kubernetes/e2e/features/port_routing/edge_gateway_suite.go new file mode 100644 index 00000000000..d429a7a0ecd --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/edge_gateway_suite.go @@ -0,0 +1,177 @@ +package port_routing + +import ( + "context" + + "github.com/solo-io/gloo/projects/gateway/pkg/defaults" + "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/solo-io/gloo/pkg/utils/kubeutils" + "github.com/solo-io/gloo/pkg/utils/requestutils/curl" + "github.com/solo-io/gloo/test/kubernetes/e2e" +) + +// glooGatewayPortRoutingTestingSuite is the entire Suite of tests for the "PortRouting" cases +type glooGatewayPortRoutingTestingSuite struct { + suite.Suite + + ctx context.Context + + // testInstallation contains all the metadata/utilities necessary to execute a series of tests + // against an installation of Gloo Gateway + testInstallation *e2e.TestInstallation + + // maps test name to a list of manifests to apply before the test + manifests map[string][]testManifest +} + +/* +The port routing suite sets up in the following order + +SetupSuite: + 1. Create k8s Gateway + 2. Proxy provisioned + +Each port routing test: + 1. Attach HttpRoute with different port/targetport definition per test + 2. Remove HttpRoute, proxy still exists without any routes + +TearDownSuite: + 1. Deletes the k8s Gateway + 2. Proxy de-provisioned +*/ +func NewEdgeGatewayApiTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { + return &glooGatewayPortRoutingTestingSuite{ + ctx: ctx, + testInstallation: testInst, + manifests: map[string][]testManifest{ + "TestInvalidPortAndValidTargetport": { + {manifestFile: upstreamInvalidPortAndValidTargetportManifest, extraArgs: []string{"-n", testInst.Metadata.InstallNamespace}}, + {manifestFile: svcInvalidPortAndValidTargetportManifest}, + }, + "TestMatchPortAndTargetport": { + {manifestFile: upstreamMatchPortandTargetportManifest, extraArgs: []string{"-n", testInst.Metadata.InstallNamespace}}, + {manifestFile: svcMatchPortandTargetportManifest}, + }, + "TestMatchPodPortWithoutTargetport": { + {manifestFile: upstreamMatchPodPortWithoutTargetportManifest, extraArgs: []string{"-n", testInst.Metadata.InstallNamespace}}, + {manifestFile: svcMatchPodPortWithoutTargetportManifest}, + }, + "TestInvalidPortWithoutTargetport": { + {manifestFile: upstreamInvalidPortWithoutTargetportManifest, extraArgs: []string{"-n", testInst.Metadata.InstallNamespace}}, + {manifestFile: svcInvalidPortWithoutTargetportManifest}, + }, + "TestInvalidPortAndInvalidTargetportManifest": { + {manifestFile: upstreamInvalidPortAndInvalidTargetportManifest, extraArgs: []string{"-n", testInst.Metadata.InstallNamespace}}, + {manifestFile: svcInvalidPortAndInvalidTargetportManifest}, + }, + }, + } +} + +func (s *glooGatewayPortRoutingTestingSuite) SetupSuite() { + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest) + s.NoError(err, "can apply setup manifest") + + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupEdgeManifest, "-n", s.testInstallation.Metadata.InstallNamespace) + s.NoError(err, "can apply edge routing manifest") +} + +func (s *glooGatewayPortRoutingTestingSuite) TearDownSuite() { + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupManifest) + s.NoError(err, "can delete setup manifest") + + err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupEdgeManifest, "-n", s.testInstallation.Metadata.InstallNamespace) + s.NoError(err, "can delete edge routing manifest") +} + +func (s *glooGatewayPortRoutingTestingSuite) BeforeTest(suiteName, testName string) { + manifests, ok := s.manifests[testName] + if !ok { + s.FailNow("no manifests found for %s, manifest map contents: %v", testName, s.manifests) + } + + for _, manifest := range manifests { + // apply gloo gateway resources to gloo installation namespace + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, manifest.manifestFile, manifest.extraArgs...) + s.NoError(err, "can apply "+manifest.manifestFile) + } +} + +func (s *glooGatewayPortRoutingTestingSuite) AfterTest(suiteName, testName string) { + manifests, ok := s.manifests[testName] + if !ok { + s.FailNow("no manifests found for " + testName) + } + + for _, manifest := range manifests { + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, manifest.manifestFile, manifest.extraArgs...) + s.NoError(err, "can delete "+manifest.manifestFile) + } +} + +func (s *glooGatewayPortRoutingTestingSuite) TestInvalidPortAndValidTargetport() { + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + // The host header must match the domain in the VirtualService + curl.WithHostHeader("example.com"), + curl.WithPort(80), + }, + expectedHealthyResponse) +} + +func (s *glooGatewayPortRoutingTestingSuite) TestMatchPortAndTargetport() { + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + // The host header must match the domain in the VirtualService + curl.WithHostHeader("example.com"), + curl.WithPort(80), + }, + expectedHealthyResponse) +} + +func (s *glooGatewayPortRoutingTestingSuite) TestMatchPodPortWithoutTargetport() { + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + // The host header must match the domain in the VirtualService + curl.WithHostHeader("example.com"), + curl.WithPort(80), + }, + expectedHealthyResponse) +} + +func (s *glooGatewayPortRoutingTestingSuite) TestInvalidPortWithoutTargetport() { + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + // The host header must match the domain in the VirtualService + curl.WithHostHeader("example.com"), + curl.WithPort(80), + }, + expectedServiceUnavailableResponse) +} + +func (s *glooGatewayPortRoutingTestingSuite) TestInvalidPortAndInvalidTargetportManifest() { + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + // The host header must match the domain in the VirtualService + curl.WithHostHeader("example.com"), + curl.WithPort(80), + }, + expectedServiceUnavailableResponse) +} diff --git a/test/kubernetes/e2e/features/port_routing/suite.go b/test/kubernetes/e2e/features/port_routing/k8s_gateway_suite.go similarity index 80% rename from test/kubernetes/e2e/features/port_routing/suite.go rename to test/kubernetes/e2e/features/port_routing/k8s_gateway_suite.go index 8b914714a15..08abb45a83c 100644 --- a/test/kubernetes/e2e/features/port_routing/suite.go +++ b/test/kubernetes/e2e/features/port_routing/k8s_gateway_suite.go @@ -11,7 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var _ e2e.NewSuiteFunc = NewTestingSuite +var _ e2e.NewSuiteFunc = NewK8sGatewayTestingSuite // portRoutingTestingSuite is the entire Suite of tests for the "PortRouting" cases type portRoutingTestingSuite struct { @@ -42,16 +42,16 @@ TearDownSuite: 1. Deletes the k8s Gateway 2. Proxy de-provisioned */ -func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { +func NewK8sGatewayTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { return &portRoutingTestingSuite{ ctx: ctx, testInstallation: testInst, manifests: map[string][]string{ - "TestInvalidPortAndValidTargetport": {invalidPortAndValidTargetportManifest}, - "TestMatchPortAndTargetport": {matchPortandTargetportManifest}, - "TestMatchPodPortWithoutTargetport": {matchPodPortWithoutTargetportManifest}, - "TestInvalidPortWithoutTargetport": {invalidPortWithoutTargetportManifest}, - "TestInvalidPortAndInvalidTargetport": {invalidPortAndInvalidTargetportManifest}, + "TestInvalidPortAndValidTargetport": {svcInvalidPortAndValidTargetportManifest, invalidPortAndValidTargetportManifest}, + "TestMatchPortAndTargetport": {svcMatchPortandTargetportManifest, matchPortandTargetportManifest}, + "TestMatchPodPortWithoutTargetport": {svcMatchPodPortWithoutTargetportManifest, matchPodPortWithoutTargetportManifest}, + "TestInvalidPortWithoutTargetport": {svcInvalidPortWithoutTargetportManifest, invalidPortWithoutTargetportManifest}, + "TestInvalidPortAndInvalidTargetport": {svcInvalidPortAndInvalidTargetportManifest, invalidPortAndInvalidTargetportManifest}, }, } } @@ -59,6 +59,8 @@ func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite. func (s *portRoutingTestingSuite) SetupSuite() { err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest) s.NoError(err, "can apply setup manifest") + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupK8sManifest) + s.NoError(err, "can apply setup k8s gateway manifest") s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment) // Check that test resources are running s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, proxyDeployment.ObjectMeta.GetNamespace(), @@ -68,6 +70,8 @@ func (s *portRoutingTestingSuite) SetupSuite() { func (s *portRoutingTestingSuite) TearDownSuite() { err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupManifest) s.NoError(err, "can delete setup manifest") + err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupK8sManifest) + s.NoError(err, "can delete setup k8s gateway manifest") s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, proxyService, proxyDeployment) } diff --git a/test/kubernetes/e2e/features/port_routing/testdata/edge-routing.yaml b/test/kubernetes/e2e/features/port_routing/testdata/edge-routing.yaml new file mode 100644 index 00000000000..bc6525cf8de --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/edge-routing.yaml @@ -0,0 +1,16 @@ +# Don't set namespace, apply to test installation namespace +apiVersion: gateway.solo.io/v1 +kind: VirtualService +metadata: + name: example-vs +spec: + virtualHost: + domains: + - "example.com" + routes: + - matchers: + - prefix: / + routeAction: + single: + upstream: + name: nginx-upstream \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-and-invalid-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-and-invalid-targetport.yaml new file mode 100644 index 00000000000..903d7bdf0e4 --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-and-invalid-targetport.yaml @@ -0,0 +1,11 @@ +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: nginx-upstream +spec: + kube: + selector: + app.kubernetes.io/name: nginx + serviceName: example-svc + serviceNamespace: default + servicePort: 8000 \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-and-valid-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-and-valid-targetport.yaml new file mode 100644 index 00000000000..c8a34b48c23 --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-and-valid-targetport.yaml @@ -0,0 +1,11 @@ +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: nginx-upstream +spec: + kube: + selector: + app.kubernetes.io/name: nginx + serviceName: example-svc + serviceNamespace: default + servicePort: 81 \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-without-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-without-targetport.yaml new file mode 100644 index 00000000000..c8a34b48c23 --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/edge/invalid-port-without-targetport.yaml @@ -0,0 +1,11 @@ +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: nginx-upstream +spec: + kube: + selector: + app.kubernetes.io/name: nginx + serviceName: example-svc + serviceNamespace: default + servicePort: 81 \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/testdata/edge/match-pod-port-without-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/edge/match-pod-port-without-targetport.yaml new file mode 100644 index 00000000000..d12ca18d280 --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/edge/match-pod-port-without-targetport.yaml @@ -0,0 +1,11 @@ +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: nginx-upstream +spec: + kube: + selector: + app.kubernetes.io/name: nginx + serviceName: example-svc + serviceNamespace: default + servicePort: 80 \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/testdata/edge/match-port-and-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/edge/match-port-and-targetport.yaml new file mode 100644 index 00000000000..d12ca18d280 --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/edge/match-port-and-targetport.yaml @@ -0,0 +1,11 @@ +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: nginx-upstream +spec: + kube: + selector: + app.kubernetes.io/name: nginx + serviceName: example-svc + serviceNamespace: default + servicePort: 80 \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/testdata/k8s-gw.yaml b/test/kubernetes/e2e/features/port_routing/testdata/k8s-gw.yaml new file mode 100644 index 00000000000..213b95003ad --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/k8s-gw.yaml @@ -0,0 +1,13 @@ +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: gw +spec: + gatewayClassName: gloo-gateway + listeners: + - protocol: HTTP + port: 8080 + name: http + allowedRoutes: + namespaces: + from: Same \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/testdata/invalid-port-and-invalid-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/k8s/invalid-port-and-invalid-targetport.yaml similarity index 50% rename from test/kubernetes/e2e/features/port_routing/testdata/invalid-port-and-invalid-targetport.yaml rename to test/kubernetes/e2e/features/port_routing/testdata/k8s/invalid-port-and-invalid-targetport.yaml index 8960de0fec3..48ecaaa3e19 100644 --- a/test/kubernetes/e2e/features/port_routing/testdata/invalid-port-and-invalid-targetport.yaml +++ b/test/kubernetes/e2e/features/port_routing/testdata/k8s/invalid-port-and-invalid-targetport.yaml @@ -1,16 +1,3 @@ -apiVersion: v1 -kind: Service -metadata: - name: example-svc -spec: - selector: - app.kubernetes.io/name: nginx - ports: - - protocol: TCP - port: 8000 - targetPort: 81 # pointing to wrong target port - name: http ---- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: diff --git a/test/kubernetes/e2e/features/port_routing/testdata/invalid-port-and-valid-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/k8s/invalid-port-and-valid-targetport.yaml similarity index 56% rename from test/kubernetes/e2e/features/port_routing/testdata/invalid-port-and-valid-targetport.yaml rename to test/kubernetes/e2e/features/port_routing/testdata/k8s/invalid-port-and-valid-targetport.yaml index 89611f08d88..f6d8fb1daca 100644 --- a/test/kubernetes/e2e/features/port_routing/testdata/invalid-port-and-valid-targetport.yaml +++ b/test/kubernetes/e2e/features/port_routing/testdata/k8s/invalid-port-and-valid-targetport.yaml @@ -1,15 +1,3 @@ -apiVersion: v1 -kind: Service -metadata: - name: example-svc -spec: - selector: - app.kubernetes.io/name: nginx - ports: - - protocol: TCP - port: 81 - targetPort: 80 ---- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: diff --git a/test/kubernetes/e2e/features/port_routing/testdata/invalid-port-without-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/k8s/invalid-port-without-targetport.yaml similarity index 55% rename from test/kubernetes/e2e/features/port_routing/testdata/invalid-port-without-targetport.yaml rename to test/kubernetes/e2e/features/port_routing/testdata/k8s/invalid-port-without-targetport.yaml index e8c8adb7411..f6d8fb1daca 100644 --- a/test/kubernetes/e2e/features/port_routing/testdata/invalid-port-without-targetport.yaml +++ b/test/kubernetes/e2e/features/port_routing/testdata/k8s/invalid-port-without-targetport.yaml @@ -1,14 +1,3 @@ -apiVersion: v1 -kind: Service -metadata: - name: example-svc -spec: - selector: - app.kubernetes.io/name: nginx - ports: - - protocol: TCP - port: 81 # not matching app's port ---- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: diff --git a/test/kubernetes/e2e/features/port_routing/testdata/match-pod-port-without-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/k8s/match-pod-port-without-targetport.yaml similarity index 53% rename from test/kubernetes/e2e/features/port_routing/testdata/match-pod-port-without-targetport.yaml rename to test/kubernetes/e2e/features/port_routing/testdata/k8s/match-pod-port-without-targetport.yaml index 478a8e9acf5..b8b9017ede1 100644 --- a/test/kubernetes/e2e/features/port_routing/testdata/match-pod-port-without-targetport.yaml +++ b/test/kubernetes/e2e/features/port_routing/testdata/k8s/match-pod-port-without-targetport.yaml @@ -1,14 +1,3 @@ -apiVersion: v1 -kind: Service -metadata: - name: example-svc -spec: - selector: - app.kubernetes.io/name: nginx - ports: - - protocol: TCP - port: 80 # without target port, but matching port ---- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: diff --git a/test/kubernetes/e2e/features/port_routing/testdata/match-port-and-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/k8s/match-port-and-targetport.yaml similarity index 52% rename from test/kubernetes/e2e/features/port_routing/testdata/match-port-and-targetport.yaml rename to test/kubernetes/e2e/features/port_routing/testdata/k8s/match-port-and-targetport.yaml index 84c6d2f8744..b8b9017ede1 100644 --- a/test/kubernetes/e2e/features/port_routing/testdata/match-port-and-targetport.yaml +++ b/test/kubernetes/e2e/features/port_routing/testdata/k8s/match-port-and-targetport.yaml @@ -1,15 +1,3 @@ -apiVersion: v1 -kind: Service -metadata: - name: example-svc -spec: - selector: - app.kubernetes.io/name: nginx - ports: - - protocol: TCP - port: 80 # match port and targetport - targetPort: 80 ---- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: diff --git a/test/kubernetes/e2e/features/port_routing/testdata/setup.yaml b/test/kubernetes/e2e/features/port_routing/testdata/setup.yaml index 601ab9fdb45..1ddc88d4b21 100644 --- a/test/kubernetes/e2e/features/port_routing/testdata/setup.yaml +++ b/test/kubernetes/e2e/features/port_routing/testdata/setup.yaml @@ -1,17 +1,3 @@ -kind: Gateway -apiVersion: gateway.networking.k8s.io/v1 -metadata: - name: gw -spec: - gatewayClassName: gloo-gateway - listeners: - - protocol: HTTP - port: 8080 - name: http - allowedRoutes: - namespaces: - from: Same ---- apiVersion: v1 kind: Pod metadata: diff --git a/test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-and-invalid-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-and-invalid-targetport.yaml new file mode 100644 index 00000000000..194c8a562e5 --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-and-invalid-targetport.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: example-svc +spec: + selector: + app.kubernetes.io/name: nginx + ports: + - protocol: TCP + port: 8000 + targetPort: 81 # pointing to wrong target port + name: http \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-and-valid-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-and-valid-targetport.yaml new file mode 100644 index 00000000000..8c270e44e72 --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-and-valid-targetport.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: example-svc +spec: + selector: + app.kubernetes.io/name: nginx + ports: + - protocol: TCP + port: 81 + targetPort: 80 \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-without-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-without-targetport.yaml new file mode 100644 index 00000000000..020289b1cc8 --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/svc/invalid-port-without-targetport.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: example-svc +spec: + selector: + app.kubernetes.io/name: nginx + ports: + - protocol: TCP + port: 81 # not matching app's port \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/testdata/svc/match-pod-port-without-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/svc/match-pod-port-without-targetport.yaml new file mode 100644 index 00000000000..8222542a4aa --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/svc/match-pod-port-without-targetport.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: example-svc +spec: + selector: + app.kubernetes.io/name: nginx + ports: + - protocol: TCP + port: 80 # without target port, but matching port \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/testdata/svc/match-port-and-targetport.yaml b/test/kubernetes/e2e/features/port_routing/testdata/svc/match-port-and-targetport.yaml new file mode 100644 index 00000000000..23095bced63 --- /dev/null +++ b/test/kubernetes/e2e/features/port_routing/testdata/svc/match-port-and-targetport.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: example-svc +spec: + selector: + app.kubernetes.io/name: nginx + ports: + - protocol: TCP + port: 80 # match port and targetport + targetPort: 80 \ No newline at end of file diff --git a/test/kubernetes/e2e/features/port_routing/types.go b/test/kubernetes/e2e/features/port_routing/types.go index bcf43d81351..a45be00fe5c 100644 --- a/test/kubernetes/e2e/features/port_routing/types.go +++ b/test/kubernetes/e2e/features/port_routing/types.go @@ -15,14 +15,36 @@ import ( testmatchers "github.com/solo-io/gloo/test/gomega/matchers" ) +type testManifest struct { + manifestFile string + extraArgs []string +} + var ( setupManifest = filepath.Join(util.MustGetThisDir(), "testdata/setup.yaml") - invalidPortAndValidTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata/invalid-port-and-valid-targetport.yaml") - invalidPortAndInvalidTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata/invalid-port-and-invalid-targetport.yaml") - matchPodPortWithoutTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata/match-pod-port-without-targetport.yaml") - matchPortandTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata/match-port-and-targetport.yaml") - invalidPortWithoutTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata/invalid-port-without-targetport.yaml") + // Shared Resources + svcInvalidPortAndValidTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "svc", "invalid-port-and-valid-targetport.yaml") + svcInvalidPortAndInvalidTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "svc", "invalid-port-and-invalid-targetport.yaml") + svcMatchPodPortWithoutTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "svc", "match-pod-port-without-targetport.yaml") + svcMatchPortandTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "svc", "match-port-and-targetport.yaml") + svcInvalidPortWithoutTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "svc", "invalid-port-without-targetport.yaml") + + // K8s Resources + setupK8sManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s-gw.yaml") + invalidPortAndValidTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s", "invalid-port-and-valid-targetport.yaml") + invalidPortAndInvalidTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s", "invalid-port-and-invalid-targetport.yaml") + matchPodPortWithoutTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s", "match-pod-port-without-targetport.yaml") + matchPortandTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s", "match-port-and-targetport.yaml") + invalidPortWithoutTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s", "invalid-port-without-targetport.yaml") + + // Gloo Edge Resources + setupEdgeManifest = filepath.Join(util.MustGetThisDir(), "testdata", "edge-routing.yaml") + upstreamInvalidPortAndValidTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "edge", "invalid-port-and-valid-targetport.yaml") + upstreamInvalidPortAndInvalidTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "edge", "invalid-port-and-invalid-targetport.yaml") + upstreamMatchPodPortWithoutTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "edge", "match-pod-port-without-targetport.yaml") + upstreamMatchPortandTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "edge", "match-port-and-targetport.yaml") + upstreamInvalidPortWithoutTargetportManifest = filepath.Join(util.MustGetThisDir(), "testdata", "edge", "invalid-port-without-targetport.yaml") // When we apply the setup.yaml file, we expect resources to be created with this metadata glooProxyObjectMeta = metav1.ObjectMeta{ diff --git a/test/kubernetes/e2e/tests/automtls_istio_edge_api_tests.go b/test/kubernetes/e2e/tests/automtls_istio_edge_api_tests.go index c0951003fa5..3f04e40f82d 100644 --- a/test/kubernetes/e2e/tests/automtls_istio_edge_api_tests.go +++ b/test/kubernetes/e2e/tests/automtls_istio_edge_api_tests.go @@ -4,12 +4,14 @@ import ( "github.com/solo-io/gloo/test/kubernetes/e2e" "github.com/solo-io/gloo/test/kubernetes/e2e/features/headless_svc" "github.com/solo-io/gloo/test/kubernetes/e2e/features/istio" + "github.com/solo-io/gloo/test/kubernetes/e2e/features/port_routing" ) func AutomtlsIstioEdgeApiSuiteRunner() e2e.SuiteRunner { automtlsIstioEdgeApiSuiteRunner := e2e.NewSuiteRunner(false) automtlsIstioEdgeApiSuiteRunner.Register("HeadlessSvc", headless_svc.NewEdgeGatewayHeadlessSvcSuite) + automtlsIstioEdgeApiSuiteRunner.Register("PortRouting", port_routing.NewEdgeGatewayApiTestingSuite) automtlsIstioEdgeApiSuiteRunner.Register("IstioIntegrationAutoMtls", istio.NewGlooIstioAutoMtlsSuite) return automtlsIstioEdgeApiSuiteRunner diff --git a/test/kubernetes/e2e/tests/automtls_istio_tests.go b/test/kubernetes/e2e/tests/automtls_istio_tests.go index 450c7500a36..94b0b17ce67 100644 --- a/test/kubernetes/e2e/tests/automtls_istio_tests.go +++ b/test/kubernetes/e2e/tests/automtls_istio_tests.go @@ -10,7 +10,7 @@ import ( func AutomtlsIstioSuiteRunner() e2e.SuiteRunner { automtlsIstioSuiteRunner := e2e.NewSuiteRunner(false) - automtlsIstioSuiteRunner.Register("PortRouting", port_routing.NewTestingSuite) + automtlsIstioSuiteRunner.Register("PortRouting", port_routing.NewK8sGatewayTestingSuite) automtlsIstioSuiteRunner.Register("HeadlessSvc", headless_svc.NewK8sGatewayHeadlessSvcSuite) automtlsIstioSuiteRunner.Register("IstioIntegrationAutoMtls", istio.NewIstioAutoMtlsSuite) diff --git a/test/kubernetes/e2e/tests/edge_gw_tests.go b/test/kubernetes/e2e/tests/edge_gw_tests.go index 0fe5328311e..da0ba6f3c40 100644 --- a/test/kubernetes/e2e/tests/edge_gw_tests.go +++ b/test/kubernetes/e2e/tests/edge_gw_tests.go @@ -3,12 +3,14 @@ package tests import ( "github.com/solo-io/gloo/test/kubernetes/e2e" "github.com/solo-io/gloo/test/kubernetes/e2e/features/headless_svc" + "github.com/solo-io/gloo/test/kubernetes/e2e/features/port_routing" ) func EdgeGwSuiteRunner() e2e.SuiteRunner { edgeGwSuiteRunner := e2e.NewSuiteRunner(false) edgeGwSuiteRunner.Register("HeadlessSvc", headless_svc.NewEdgeGatewayHeadlessSvcSuite) + edgeGwSuiteRunner.Register("PortRouting", port_routing.NewEdgeGatewayApiTestingSuite) return edgeGwSuiteRunner } diff --git a/test/kubernetes/e2e/tests/istio_edge_api_tests.go b/test/kubernetes/e2e/tests/istio_edge_api_tests.go index f72e6574b59..d59bf7a40bc 100644 --- a/test/kubernetes/e2e/tests/istio_edge_api_tests.go +++ b/test/kubernetes/e2e/tests/istio_edge_api_tests.go @@ -4,12 +4,14 @@ import ( "github.com/solo-io/gloo/test/kubernetes/e2e" "github.com/solo-io/gloo/test/kubernetes/e2e/features/headless_svc" "github.com/solo-io/gloo/test/kubernetes/e2e/features/istio" + "github.com/solo-io/gloo/test/kubernetes/e2e/features/port_routing" ) func IstioEdgeApiSuiteRunner() e2e.SuiteRunner { istioEdgeApiSuiteRunner := e2e.NewSuiteRunner(false) istioEdgeApiSuiteRunner.Register("HeadlessSvc", headless_svc.NewEdgeGatewayHeadlessSvcSuite) + istioEdgeApiSuiteRunner.Register("PortRouting", port_routing.NewEdgeGatewayApiTestingSuite) istioEdgeApiSuiteRunner.Register("IstioIntegration", istio.NewGlooTestingSuite) return istioEdgeApiSuiteRunner diff --git a/test/kubernetes/e2e/tests/istio_regression_test.go b/test/kubernetes/e2e/tests/istio_regression_test.go new file mode 100644 index 00000000000..11ad0a7f3da --- /dev/null +++ b/test/kubernetes/e2e/tests/istio_regression_test.go @@ -0,0 +1,73 @@ +package tests_test + +import ( + "context" + "log" + "path/filepath" + "testing" + "time" + + "github.com/solo-io/gloo/test/kube2e/helper" + "github.com/solo-io/gloo/test/kubernetes/e2e" + . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" + "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + + "github.com/solo-io/skv2/codegen/util" +) + +// TestIstioRegression is the function which executes a series of tests against a given installation where +// the k8s Gateway controller is disabled and the deprecated Istio integration values are used to check for regressions +func TestIstioRegression(t *testing.T) { + ctx := context.Background() + testInstallation := e2e.CreateTestInstallation( + t, + &gloogateway.Context{ + InstallNamespace: "istio-regression-test", + ValuesManifestFile: filepath.Join(util.MustGetThisDir(), "manifests", "istio-regression-helm.yaml"), + }, + ) + + testHelper := e2e.MustTestHelper(ctx, testInstallation) + + err := testInstallation.AddIstioctl(ctx) + if err != nil { + log.Printf("failed to add istioctl: %v\n", err) + t.Fail() + } + + // We register the cleanup function _before_ we actually perform the installation. + // This allows us to uninstall Gloo Gateway, in case the original installation only completed partially + t.Cleanup(func() { + if t.Failed() { + testInstallation.PreFailHandler(ctx) + + // Generate istioctl bug report + testInstallation.CreateIstioBugReport(ctx) + } + + testInstallation.UninstallGlooGateway(ctx, func(ctx context.Context) error { + return testHelper.UninstallGlooAll() + }) + + // Uninstall Istio + err = testInstallation.UninstallIstio() + if err != nil { + log.Printf("failed to uninstall: %v\n", err) + t.Fail() + } + }) + + // Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy + err = testInstallation.InstallMinimalIstio(ctx) + if err != nil { + log.Printf("failed to install: %v\n", err) + t.Fail() + } + + // Install Gloo Gateway with only Edge APIs enabled + testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { + return testHelper.InstallGloo(ctx, helper.GATEWAY, 5*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + }) + + IstioRegressionSuiteRunner().Run(ctx, t, testInstallation) +} diff --git a/test/kubernetes/e2e/tests/istio_regression_tests.go b/test/kubernetes/e2e/tests/istio_regression_tests.go new file mode 100644 index 00000000000..7d431684d93 --- /dev/null +++ b/test/kubernetes/e2e/tests/istio_regression_tests.go @@ -0,0 +1,18 @@ +package tests + +import ( + "github.com/solo-io/gloo/test/kubernetes/e2e" + "github.com/solo-io/gloo/test/kubernetes/e2e/features/headless_svc" + "github.com/solo-io/gloo/test/kubernetes/e2e/features/istio" + "github.com/solo-io/gloo/test/kubernetes/e2e/features/port_routing" +) + +func IstioRegressionSuiteRunner() e2e.SuiteRunner { + istioEdgeApiSuiteRunner := e2e.NewSuiteRunner(false) + + istioEdgeApiSuiteRunner.Register("HeadlessSvc", headless_svc.NewEdgeGatewayHeadlessSvcSuite) + istioEdgeApiSuiteRunner.Register("PortRouting", port_routing.NewEdgeGatewayApiTestingSuite) + istioEdgeApiSuiteRunner.Register("IstioIntegration", istio.NewGlooTestingSuite) + + return istioEdgeApiSuiteRunner +} diff --git a/test/kubernetes/e2e/tests/istio_test.go b/test/kubernetes/e2e/tests/k8s_gw_istio_test.go similarity index 100% rename from test/kubernetes/e2e/tests/istio_test.go rename to test/kubernetes/e2e/tests/k8s_gw_istio_test.go diff --git a/test/kubernetes/e2e/tests/istio_tests.go b/test/kubernetes/e2e/tests/k8s_gw_istio_tests.go similarity index 89% rename from test/kubernetes/e2e/tests/istio_tests.go rename to test/kubernetes/e2e/tests/k8s_gw_istio_tests.go index 2502e1840e8..9e84ccf0109 100644 --- a/test/kubernetes/e2e/tests/istio_tests.go +++ b/test/kubernetes/e2e/tests/k8s_gw_istio_tests.go @@ -11,7 +11,7 @@ import ( func IstioSuiteRunner() e2e.SuiteRunner { istioSuiteRunner := e2e.NewSuiteRunner(false) - istioSuiteRunner.Register("PortRouting", port_routing.NewTestingSuite) + istioSuiteRunner.Register("PortRouting", port_routing.NewK8sGatewayTestingSuite) istioSuiteRunner.Register("HeadlessSvc", headless_svc.NewK8sGatewayHeadlessSvcSuite) istioSuiteRunner.Register("IstioIntegration", istio.NewTestingSuite) istioSuiteRunner.Register("IstioGatewayParameters", deployer.NewIstioIntegrationTestingSuite) diff --git a/test/kubernetes/e2e/tests/k8s_gw_no_validation_tests.go b/test/kubernetes/e2e/tests/k8s_gw_no_validation_tests.go index 74090f6c162..d89def8144a 100644 --- a/test/kubernetes/e2e/tests/k8s_gw_no_validation_tests.go +++ b/test/kubernetes/e2e/tests/k8s_gw_no_validation_tests.go @@ -14,7 +14,7 @@ func KubeGatewayNoValidationSuiteRunner() e2e.SuiteRunner { kubeGatewayNoValidationSuiteRunner.Register("ListenerOptions", listener_options.NewTestingSuite) kubeGatewayNoValidationSuiteRunner.Register("RouteOptions", route_options.NewTestingSuite) kubeGatewayNoValidationSuiteRunner.Register("VirtualHostOptions", virtualhost_options.NewTestingSuite) - kubeGatewayNoValidationSuiteRunner.Register("PortRouting", port_routing.NewTestingSuite) + kubeGatewayNoValidationSuiteRunner.Register("PortRouting", port_routing.NewK8sGatewayTestingSuite) return kubeGatewayNoValidationSuiteRunner } diff --git a/test/kubernetes/e2e/tests/k8s_gw_tests.go b/test/kubernetes/e2e/tests/k8s_gw_tests.go index 0f88fcf7bea..78a0f7630c7 100644 --- a/test/kubernetes/e2e/tests/k8s_gw_tests.go +++ b/test/kubernetes/e2e/tests/k8s_gw_tests.go @@ -29,7 +29,7 @@ func KubeGatewaySuiteRunner() e2e.SuiteRunner { kubeGatewaySuiteRunner.Register("Upstreams", upstreams.NewTestingSuite) kubeGatewaySuiteRunner.Register("Services", services.NewTestingSuite) kubeGatewaySuiteRunner.Register("HeadlessSvc", headless_svc.NewK8sGatewayHeadlessSvcSuite) - kubeGatewaySuiteRunner.Register("PortRouting", port_routing.NewTestingSuite) + kubeGatewaySuiteRunner.Register("PortRouting", port_routing.NewK8sGatewayTestingSuite) kubeGatewaySuiteRunner.Register("RouteDelegation", route_delegation.NewTestingSuite) kubeGatewaySuiteRunner.Register("Glooctl", newGlooctlTestingSuite) diff --git a/test/kubernetes/e2e/tests/manifests/istio-regression-helm.yaml b/test/kubernetes/e2e/tests/manifests/istio-regression-helm.yaml new file mode 100644 index 00000000000..920ddb70cc5 --- /dev/null +++ b/test/kubernetes/e2e/tests/manifests/istio-regression-helm.yaml @@ -0,0 +1,47 @@ +global: + # Set up gloo with istio integration enabled (through `enableIstioSidecarOnGateway`) + istioIntegration: + enableIstioSidecarOnGateway: true + disableAutoinjection: true # We do not want Gloo components to be included in the mesh + istioSDS: + enabled: true + glooMtls: + istioProxy: + image: + repository: proxyv2 + registry: docker.io/istio + tag: 1.22.0 # This tag has to match the version of Istio being used in the test + podSecurityStandards: + container: + enableRestrictedContainerDefaults: true +gloo: + logLevel: info + disableLeaderElection: true + deployment: + # We have limited GitHub action resources which can cause containers to not create + # therefore we lessen the cpu resource requests values from the default (500m) to 100m. + resources: + requests: + cpu: 100m + memory: 256Mi +gatewayProxies: + gatewayProxy: + podTemplate: + resources: + requests: + cpu: 100m + memory: 256Mi + healthyPanicThreshold: 0 + +# These values are recommended production values and are not expected to impact tested behavior for the Istio suite +settings: + invalidConfigPolicy: + replaceInvalidRoutes: true + invalidRouteResponseCode: 404 + invalidRouteResponseBody: Gloo Gateway has invalid configuration. +gateway: + persistProxySpec: true + logLevel: info + validation: + allowWarnings: true + alwaysAcceptResources: false \ No newline at end of file From d53df48f0929626b17091cd8dc4c6f24a3522b81 Mon Sep 17 00:00:00 2001 From: npolshakova Date: Mon, 17 Jun 2024 17:40:51 -0400 Subject: [PATCH 3/3] pr feedback --- devel/architecture/istio-integration.md | 287 ++++++++++++++++++ .../port_routing/edge_gateway_suite.go | 12 +- .../port_routing/k8s_gateway_suite.go | 10 +- 3 files changed, 299 insertions(+), 10 deletions(-) create mode 100644 devel/architecture/istio-integration.md diff --git a/devel/architecture/istio-integration.md b/devel/architecture/istio-integration.md new file mode 100644 index 00000000000..84168e19bfd --- /dev/null +++ b/devel/architecture/istio-integration.md @@ -0,0 +1,287 @@ +An overview of some key concepts related to this test suite: + +# mTLS +[mTLS explained](https://www.cloudflare.com/learning/access-management/what-is-mutual-tls/) + +[mTLS in Envoy](https://www.envoyproxy.io/docs/envoy/latest/start/quick-start/securing#use-mutual-tls-mtls-to-enforce-client-certificate-authentication) + +# Gloo mTLS +[Background](https://docs.solo.io/gloo-edge/latest/guides/security/tls/mtls/) + +In Gloo Edge, services that receive their configuration from the xDS server, communicate using the xDS protocol, which is done in plaintext. Since the information being provided contains sensitive information (secrets), it is preferable to encrypt this traffic. This applies to the following services: +- Gateway-proxy +- Ext-auth-service +- Rate-limiter + +## How does this work? +Each component involved in the xDS communication will contain an SDS sidecar and potentially an Envoy sidecar as well (for our gateway-proxy pod, which already contains an envoy container, this Envoy sidecar is not necessary) + +**Envoy Sidecar**: Responsible for TLS termination, and outgoing encryption. Receives it’s secret information via SDS, talking to the SDS sidecar. The sidecar bootstrap configuration is defined in a ConfigMap for each component. + +**SDS Sidecar**: Watches the gloo-mtls-certs TLS secret in kubernetes, and serves the contents via the SDS API to the Envoy sidecar. The SDS code can be found [here](https://github.com/solo-io/gloo/tree/main/projects/sds). Anytime the secret is modified, the contents will be updated dynamically. To automate the updating of secrets, run the [CertGen Job](https://github.com/solo-io/gloo/blob/main/install/helm/gloo/templates/19-gloo-mtls-certgen-job.yaml) + + +### Example: Gloo <-> ExtAuth +[ExtAuth Envoy Sidecar](https://github.com/solo-io/solo-projects/blob/main/install/helm/gloo-ee/templates/24-extauth-sidecar-config.yaml) + +- ExtAuth pod initiates xDS request to Gloo +- ExtAuth.EnvoySidecar hijacks request, and sends encrypted request to Gloo +- Gloo.EnvoySidecar terminates TLS request from ExtAuth +- Gloo.EnvoySidecar directs request to Gloo xDS port + +# Istio mTLS + +[Background](https://docs.solo.io/gloo-edge/latest/guides/integrations/service_mesh/istio/) + +When services are deployed within the Istio Service Mesh, it is possible to require that all traffic between those services is encrypted with mTLS. This is handled dynamically by Istio managing certs for services within the Mesh. However, it poses a challenge when Gloo Edge (not in the Mesh) tries to route traffic to services within the Mesh. + +This challenge is that Edge encrypts traffic using the certificates that were generated by IstioD and used within the Mesh. To accomplish this, Edge uses a similar paradigm as the Gloo mTLS solution, leveraging sidecars to encrypt traffic. + +## How does this work? +To keep the mTLS communication transparent to the Edge configuration, we again leverage sidecars to handle the encryption. + +[Istio](https://istio.io/latest/docs/ops/deployment/architecture/) is logically split into a data plane and a control plane. The data plane is composed of a set of intelligent proxies (Envoy) deployed as sidecars. The control plane (istiod) provides service discovery, configuration and certificate management. +Istiod acts as a Certificate Authority (CA) and generates certificates to allow secure mTLS communication in the data plane. + +**Istio-Proxy Sidecar**: This is responsible for generating the certificates used for mTLS communication. These certificates are mounted to a volume, which are then provided to the gateway-proxy configuration via SDS. The gateway-proxy, with these certificates, is now able to establish mTLS communication with an upstream in the Mesh. +The Istio-Proxy Sidecar usually runs both the istio-agent and envoy. The [istio-agent](https://github.com/istio/istio/blob/master/architecture/security/istio-agent.md) is responsible for generating the certificates, and the envoy is responsible for terminating TLS and establishing mTLS communication with other services in the Mesh. +By default, the istio-proxy will run an extended version of the Envoy proxy. However, we don't need the istio-proxy Envoy functionality, we only need the istio-agent to create the CSR request to istiod and handle rotating certificates near expiration. +To avoid running the istio-proxy Envoy, we can set the `DISABLE_ENVOY` environment variable. This will cause the istio-proxy to run in proxyless mode and not start the Envoy process. + + + +**SDS Sidecar**: Watches the Istio certs, generated by the istio-proxy sidecar, and serves them up via SDS to the gateway-proxy. The SDS code can be found [here](https://github.com/solo-io/gloo/tree/main/projects/sds). Any time the volume is modified, the contents will be updated dynamically. This is done because Istio rotates its certificates by default, every 24 hours (can be controlled by env variable). Without this SDS sidecar, the proxies would use outdated certificates, and when new ones are rolled out by Istio, our gateway-proxy wouldn’t be able to communicate with services using the new certificates. + +# SDS + +Envoy provides an API for dynamically updating TLS certificates. Historically, this required restarting the proxies with the new certificates, but the [SDS API](https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret) allows a zero-downtime approach to this. + +In Gloo Edge, we build an SDS server which implements this API. Its sole responsibility is to implement the SDS API, and serve up certificates via xDS. We configure our Envoy proxies as the client to this server, and the proxies are [configured to get their certificates via the SDS API](https://github.com/solo-io/gloo/blob/main/install/helm/gloo/templates/9-gateway-proxy-configmap.yaml#L195) + +While the component itself has a single responsibility, implement the SDS API, it can be used in Gloo Edge in multiple ways: +- Serve certificates to establish mTLS communication between Gloo components (Gloo mTLS) +- Serve certificates to establish mLTS communication between Gateway-Proxy and Application running in Service Mesh (Istio mTLS) + +_As a result, we have `glooMtls.enabled` to enable the former, and `istioIntegration.enabled` to enable the latter._ + +Note, the current Gloo SDS does not reach out to Istiod. The istio-agent is responsible for sending the CSR to Istiod. +The SDS server then reads the certs from a file written by the istio-agent and then SDS serves the certificates to the Gloo Envoy proxy. + +# Validating mTLS Traffic + +Istio leverages the [`x-forwarded-client-cert`](https://istio.io/latest/docs/ops/configuration/traffic-management/network-topologies/#forwarding-external-client-attributes-ip-address-certificate-info-to-destination-workloads) header to identify encrypted traffic + +If the application that we’re running can logs requests that it receives, we could search the logs for the existence of that header + +# Testing automtls + +The istio e2e integration tests automtls functionality with Gloo Edge "classic" APIs and k8s Gateway API resources. This +can be manually tested by following the steps below on a kind cluster: + +1. Setup environment and kind cluster + +```shell +ci/kind/setup-kind.sh; make kind-build-and-load +``` + +2. Install Istio + +```shell +./projects/gateway2/istio.sh +``` + +```shell +cat <