From 80cea6d2ff2e698b8b35d0e50549d1f7a7eeed76 Mon Sep 17 00:00:00 2001 From: Dave Stanke Date: Sun, 7 Apr 2019 14:34:52 -0400 Subject: [PATCH 1/7] WIP on gke-per-test --- cloudbuild.gke-per-test.yaml | 125 +++++++++++++++++++++++++++++++++++ cloudbuild.gke.yaml | 8 ++- 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 cloudbuild.gke-per-test.yaml diff --git a/cloudbuild.gke-per-test.yaml b/cloudbuild.gke-per-test.yaml new file mode 100644 index 0000000..8eae2c8 --- /dev/null +++ b/cloudbuild.gke-per-test.yaml @@ -0,0 +1,125 @@ +steps: + +- id: 'create cluster' + name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'bash' + args: + - '-c' + - | + gcloud container clusters create test-$$(date +%s) + waitFor: ['-'] # start immediately + +- id: 'compile web app' + name: 'gcr.io/cloud-builders/npm' + dir: 'web' + args: ['install'] + waitFor: ['-'] + +- id: 'web app unit tests' + name: 'gcr.io/cloud-builders/npm' + dir: 'web' + args: ['test'] + waitFor: ['compile web app'] + +- id: 'build web' + name: 'gcr.io/cloud-builders/docker' + args: [ + 'build', + '--tag=web', + '--tag=gcr.io/$PROJECT_ID/web', + '--cache-from', 'gcr.io/$PROJECT_ID/web:latest', + 'web/.', + ] + waitFor: ['web app unit tests'] # start immediately + +- id: 'build db' + name: 'gcr.io/cloud-builders/docker' + args: [ + 'build', + '--tag=mysql', + '--tag=gcr.io/$PROJECT_ID/mysql', + '--cache-from', 'gcr.io/$PROJECT_ID/mysql:latest', + 'mysql/.', + ] + env: + - "MYSQL_ROOT_PASSWORD=password" + waitFor: ['-'] # start immediately + +- id: 'push web' + name: 'gcr.io/cloud-builders/docker' + args: ['push', 'gcr.io/$PROJECT_ID/web'] + waitFor: ['build web'] + +- id: 'push db' + name: 'gcr.io/cloud-builders/docker' + args: ['push', 'gcr.io/$PROJECT_ID/mysql'] + waitFor: ['build db'] + +- id: 'deploy to staging' + name: 'gcr.io/cloud-builders/kubectl' + args: ['apply', '-f', 'k8s/',] + waitFor: ['push web','push db','create cluster'] + +- id: 'get endpoint' + name: 'gcr.io/cloud-builders/kubectl' + entrypoint: 'bash' + args: + - '-c' + - | + # determine which node port the service was exposed on + get_nodeport() { + kubectl get service cookieshop-web -o=jsonpath='{.spec.ports[0].nodePort}' + } + + until [[ -n "$(get_nodeport)" ]]; do + echo "querying for nodeport" + sleep 3 + done + + echo "$(get_nodeport)" > _nodeport # save port for use in next step + + # grab the public IP of a random node + # (why do this instead of a load balancer? b/c it's faster to instantiate) + # (and doesn't require additional billable resources) + # (we can reasonably expect the node to be durable enough to last until the test completes) + get_nodeip() { + kubectl get nodes --output jsonpath='{.items[0].status.addresses[?(@.type=="ExternalIP")].address}' + } + + until [[ -n "$(get_nodeip)" ]]; do + echo "querying for nodeip" + sleep 3 + done + + echo $(get_nodeip) > _nodeip # save ip for use in next step + + waitFor: ['deploy to staging'] + +- id: 'integration tests' + name: 'gcr.io/cloud-builders/curl' + entrypoint: '/bin/bash' + args: + - '-c' + - | + ### -r = retries; -i = interval; -k = keyword to search for ### + ./test/test-connection.sh -r 20 -i 3 -u http://$(cat _nodeip):$(cat _nodeport) + ./test/test-content.sh -r 20 -i 3 -u http://$(cat _nodeip):$(cat _nodeport) -k 'Chocolate Chip' + waitFor: ['get endpoint'] + +- id: 'delete cluster' + name: 'gcr.io/cloud-builders/gcloud' + args: [ + 'container','clusters','delete', + 'test-$BUILD_ID', # unique cluster name + '--quiet', # don't prompt for confirmation + ] + waitFor: ['integration tests'] + +options: + env: + # location/name of GKE cluster (used by all kubectl commands) + - CLOUDSDK_COMPUTE_ZONE=us-central1-a + - CLOUDSDK_CONTAINER_CLUSTER=staging + +timeout: + 3600s \ No newline at end of file diff --git a/cloudbuild.gke.yaml b/cloudbuild.gke.yaml index f8afc1e..72d747f 100644 --- a/cloudbuild.gke.yaml +++ b/cloudbuild.gke.yaml @@ -50,10 +50,16 @@ steps: args: ['push', 'gcr.io/$PROJECT_ID/mysql'] waitFor: ['build db'] +- id: 'patch k8s config' + name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'bash' + args: ['-c','find k8s/ -type f | xargs sed -i "s/PROJECT_ID/$PROJECT_ID/g"'] + waitFor: ['-'] + - id: 'deploy to staging' name: 'gcr.io/cloud-builders/kubectl' args: ['apply', '-f', 'k8s/', '--namespace=test-$BUILD_ID'] - waitFor: ['push web','push db'] + waitFor: ['push web','push db','patch k8s config'] - id: 'get endpoint' name: 'gcr.io/cloud-builders/kubectl' From 0a447421403389bbcce9773c3dcebfd185f37cf2 Mon Sep 17 00:00:00 2001 From: Dave Stanke Date: Sun, 7 Apr 2019 14:40:53 -0700 Subject: [PATCH 2/7] change placeholder for consistency --- k8s/db.yaml | 2 +- k8s/service-web.yaml | 17 ----------------- k8s/web.yaml | 2 +- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/k8s/db.yaml b/k8s/db.yaml index aef979e..bddf095 100644 --- a/k8s/db.yaml +++ b/k8s/db.yaml @@ -33,7 +33,7 @@ spec: containers: - name: mysql # Dev TODO: add your project id here - image: gcr.io/PROJECT-ID/mysql:latest + image: gcr.io/PROJECT_ID/mysql:latest env: - name: MYSQL_ROOT_PASSWORD value: "password" diff --git a/k8s/service-web.yaml b/k8s/service-web.yaml index 66dd50b..f00ea4b 100644 --- a/k8s/service-web.yaml +++ b/k8s/service-web.yaml @@ -11,23 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -# kind: Service -# apiVersion: v1 -# metadata: -# name: cookieshop-web -# labels: -# app: cookieshop -# role: web -# spec: -# type: NodePort -# ports: -# - port: 3000 -# targetPort: 3000 -# selector: -# role: web -# app: cookieshop ---- kind: Service apiVersion: v1 metadata: diff --git a/k8s/web.yaml b/k8s/web.yaml index f16865b..358b7e0 100644 --- a/k8s/web.yaml +++ b/k8s/web.yaml @@ -33,7 +33,7 @@ spec: containers: - name: web # Dev TODO: add your project id here - image: gcr.io/PROJECT-ID/web:latest + image: gcr.io/PROJECT_ID/web:latest env: - name: DB_HOST value: cookieshop-db From 263282db0d77437d7d84494c19513b092816bcab Mon Sep 17 00:00:00 2001 From: Dave Stanke Date: Mon, 8 Apr 2019 14:09:47 -0700 Subject: [PATCH 3/7] working gke-per-test method --- cloudbuild.gke-per-test.yaml | 23 ++++++++++++++++------- cloudbuild.gke.yaml | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/cloudbuild.gke-per-test.yaml b/cloudbuild.gke-per-test.yaml index 8eae2c8..999bf09 100644 --- a/cloudbuild.gke-per-test.yaml +++ b/cloudbuild.gke-per-test.yaml @@ -6,7 +6,10 @@ steps: args: - '-c' - | - gcloud container clusters create test-$$(date +%s) + # capture the cluster name to be created + # (it will be used again for the deletion) + echo "test-$(date +%s)" > _cluster-name + gcloud container clusters create $(cat _cluster-name) waitFor: ['-'] # start immediately - id: 'compile web app' @@ -55,10 +58,16 @@ steps: args: ['push', 'gcr.io/$PROJECT_ID/mysql'] waitFor: ['build db'] +- id: 'patch k8s config' + name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'bash' + args: ['-c','find k8s -type f | xargs sed -i "s/PROJECT_ID/$PROJECT_ID/g"'] + waitFor: ['-'] + - id: 'deploy to staging' name: 'gcr.io/cloud-builders/kubectl' args: ['apply', '-f', 'k8s/',] - waitFor: ['push web','push db','create cluster'] + waitFor: ['push web','push db','create cluster','patch k8s config'] - id: 'get endpoint' name: 'gcr.io/cloud-builders/kubectl' @@ -108,11 +117,11 @@ steps: - id: 'delete cluster' name: 'gcr.io/cloud-builders/gcloud' - args: [ - 'container','clusters','delete', - 'test-$BUILD_ID', # unique cluster name - '--quiet', # don't prompt for confirmation - ] + entrypoint: 'bash' + args: + - '-c' + - | + gcloud container clusters delete $(cat _cluster-name) --quiet # don't prompt for confirmation waitFor: ['integration tests'] options: diff --git a/cloudbuild.gke.yaml b/cloudbuild.gke.yaml index 72d747f..156df91 100644 --- a/cloudbuild.gke.yaml +++ b/cloudbuild.gke.yaml @@ -53,7 +53,7 @@ steps: - id: 'patch k8s config' name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' - args: ['-c','find k8s/ -type f | xargs sed -i "s/PROJECT_ID/$PROJECT_ID/g"'] + args: ['-c','find k8s -type f | xargs sed -i "s/PROJECT_ID/$PROJECT_ID/g"'] waitFor: ['-'] - id: 'deploy to staging' From 45f2a0309b6b64460ea01bbad2f5aeabfe1aff37 Mon Sep 17 00:00:00 2001 From: Dave Stanke Date: Mon, 8 Apr 2019 14:13:26 -0700 Subject: [PATCH 4/7] Add IDs to meta-CI steps. Fixes #46 --- cloudbuild.meta.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cloudbuild.meta.yaml b/cloudbuild.meta.yaml index 1ce564e..fbf13f8 100644 --- a/cloudbuild.meta.yaml +++ b/cloudbuild.meta.yaml @@ -1,7 +1,8 @@ # verify that all three testing methods work steps: -- name: 'gcr.io/cloud-builders/gcloud' +- id: 'Docker Compose' + name: 'gcr.io/cloud-builders/gcloud' args: [ 'builds', 'submit', @@ -10,7 +11,8 @@ steps: ] waitFor: ['-'] -- name: 'gcr.io/cloud-builders/gcloud' +- id: 'GKE staging cluster' + name: 'gcr.io/cloud-builders/gcloud' args: [ 'builds', 'submit', @@ -19,7 +21,8 @@ steps: ] waitFor: ['-'] -- name: 'gcr.io/cloud-builders/gcloud' +- id: 'Single-node Kubernetes' + name: 'gcr.io/cloud-builders/gcloud' args: [ 'builds', 'submit', From 6892b4d0d3204ec364c34d1fe1943de50ee4c989 Mon Sep 17 00:00:00 2001 From: Dave Stanke Date: Mon, 8 Apr 2019 19:29:27 -0700 Subject: [PATCH 5/7] add gke-per-test to meta-ci --- cloudbuild.meta.yaml | 10 ++++++++++ cloudbuild.vm.yaml | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/cloudbuild.meta.yaml b/cloudbuild.meta.yaml index fbf13f8..483e043 100644 --- a/cloudbuild.meta.yaml +++ b/cloudbuild.meta.yaml @@ -31,5 +31,15 @@ steps: ] waitFor: ['-'] +- id: 'GKE cluster per test' + name: 'gcr.io/cloud-builders/gcloud' + args: [ + 'builds', + 'submit', + '--config=cloudbuild.gke-per-test.yaml', + '.' + ] + waitFor: ['-'] + timeout: 3600s diff --git a/cloudbuild.vm.yaml b/cloudbuild.vm.yaml index f3735c8..1e87169 100644 --- a/cloudbuild.vm.yaml +++ b/cloudbuild.vm.yaml @@ -24,6 +24,12 @@ steps: ] waitFor: ['init terraform'] +- id: 'patch k8s config' + name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'bash' + args: ['-c','find k8s -type f | xargs sed -i "s/PROJECT_ID/$PROJECT_ID/g"'] + waitFor: ['-'] + - id: 'patch kubeconfig' name: 'gcr.io/$PROJECT_ID/terraform' entrypoint: '/bin/bash' From ff4aa99c942acad1f41429759ef4f30f63d6a980 Mon Sep 17 00:00:00 2001 From: Dave Stanke Date: Tue, 9 Apr 2019 08:53:48 -0700 Subject: [PATCH 6/7] remove hard-coded number of test methods --- cloudbuild.meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.meta.yaml b/cloudbuild.meta.yaml index 483e043..b41eafe 100644 --- a/cloudbuild.meta.yaml +++ b/cloudbuild.meta.yaml @@ -1,4 +1,4 @@ -# verify that all three testing methods work +# verify that all testing methods work steps: - id: 'Docker Compose' From a0e984f13e888a69cd7b3c5c842eaceb1b0ad60a Mon Sep 17 00:00:00 2001 From: Dave Stanke Date: Tue, 9 Apr 2019 08:55:09 -0700 Subject: [PATCH 7/7] Add Apache license file to cloudbuild configs --- cloudbuild.gke-per-test.yaml | 14 ++++++++++++++ cloudbuild.gke.yaml | 14 ++++++++++++++ cloudbuild.meta.yaml | 14 ++++++++++++++ cloudbuild.vm.yaml | 14 ++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/cloudbuild.gke-per-test.yaml b/cloudbuild.gke-per-test.yaml index 999bf09..8b32c0a 100644 --- a/cloudbuild.gke-per-test.yaml +++ b/cloudbuild.gke-per-test.yaml @@ -1,3 +1,17 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + steps: - id: 'create cluster' diff --git a/cloudbuild.gke.yaml b/cloudbuild.gke.yaml index 156df91..5e5594f 100644 --- a/cloudbuild.gke.yaml +++ b/cloudbuild.gke.yaml @@ -1,3 +1,17 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + steps: - id: 'compile web app' name: 'gcr.io/cloud-builders/npm' diff --git a/cloudbuild.meta.yaml b/cloudbuild.meta.yaml index b41eafe..200d34e 100644 --- a/cloudbuild.meta.yaml +++ b/cloudbuild.meta.yaml @@ -1,3 +1,17 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # verify that all testing methods work steps: diff --git a/cloudbuild.vm.yaml b/cloudbuild.vm.yaml index 1e87169..7d27026 100644 --- a/cloudbuild.vm.yaml +++ b/cloudbuild.vm.yaml @@ -1,3 +1,17 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + steps: # 1. terraform: make [self-destructing] VM [with microk8s]