From 139b929fd173d7ef62d3679592ad003505cb86fa Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 11:42:16 +0100 Subject: [PATCH 01/41] comment ci yaml --- .circleci/config.yml | 150 ++++++++++++++++++++++++++----------------- 1 file changed, 91 insertions(+), 59 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 57370cbee..aa293cbed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,34 +1,40 @@ version: 2.1 + +# Setting up docker executor using casskop-build image. executors: casskop-build: docker: - image: orangeopensource/casskop-build:v0.9.0 + experimental: notify: branches: only: - master +# Define shareable packages of configuration +# elements used into the worflow (= Orbs) orbs: minikube: + # Setting up VM executor for minikube. executors: vm-classic: machine: image: circleci/classic:201808-01 - + # Setting up job list jobs: - e2e-orb: + e2e-orb: # Define job which execute 'docker-e2e-test-fix-arg' makefile job + executor: vm-classic working_directory: /home/circleci/cassandra-k8s-operator + # PARAMETERS parameters: - test_name: + test_nametest_name: description: name of the test in e2e code type: string default: toto - executor: vm-classic - - + # ENVIRONMENTS VARIABLES environment: K8S_VERSION: v1.14.0 KUBECONFIG: /home/circleci/.kube/config @@ -42,32 +48,30 @@ orbs: GO111MODULE: on DOCKER_LATEST: 1 GIN_MODE: release - - + # STEPS steps: - - run: + # Pre-checkout steps + - run: # Install Minikube components to be able to run it. name: Setup minikube command: | wget -O minikube https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ - - run: + - run: # Run minikube with expected parameters name: Start minikube background: true command: | sudo -E minikube start --vm-driver=none --cpus 4 --memory 4096 --kubernetes-version=${K8S_VERSION} - - - run: + - run: # Install kubectl cli name: Setup kubectl command: | wget -O kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ mkdir -p ${HOME}/.kube touch ${HOME}/.kube/config - + # Checkout steps - checkout - - - attach_workspace: + - attach_workspace: # Create artifact from workdir at: ./ - - - run: + # Check environment set up correctly + - run: # Wait until minikube nodes are ready name: Wait for minikube command: | timeout 300s bash \< Date: Thu, 26 Dec 2019 12:07:53 +0100 Subject: [PATCH 02/41] add workpath param for build --- .circleci/config.yml | 40 +++++++--------------------------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aa293cbed..1d0b2a980 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -109,7 +109,12 @@ orbs: jobs: # Build job, which build operator docker image (with operator-sdk build) build: - working_directory: /home/circleci/cassandra-k8s-operator + parameters: + workdir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + working_directory: /home/circleci/<< parameters.workdir >> executor: casskop-build # STEPS steps: @@ -373,35 +378,4 @@ workflows: only: /.*/ branches: # Forked pull requests have CIRCLE_BRANCH set to pull/XXX - ignore: /pull\/[0-9]+/ - - ci-multicasskop-operator: - jobs: - - build: - filters: - tags: - only: /.*/ - - - unit-test: - requires: - - build - filters: - tags: - only: /.*/ - - - docker-dgoss-test: - requires: - - build - filters: - tags: - only: /.*/ - - - report: - requires: - - unit-test - filters: - tags: - only: /.*/ - branches: - # report sonar only from canonical repository - only: /^(?!pull\/).*$/ # https://stackoverflow.com/a/5334825/358804 \ No newline at end of file + ignore: /pull\/[0-9]+/ \ No newline at end of file From f0a896f7dd9ebdacdbdf03438d6568d96f533c26 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 12:13:05 +0100 Subject: [PATCH 03/41] clean & correct --- .circleci/config.yml | 2 +- multi-casskop/multi-casskop.env | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) delete mode 100644 multi-casskop/multi-casskop.env diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d0b2a980..7766f9505 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,7 @@ orbs: working_directory: /home/circleci/cassandra-k8s-operator # PARAMETERS parameters: - test_nametest_name: + test_name: description: name of the test in e2e code type: string default: toto diff --git a/multi-casskop/multi-casskop.env b/multi-casskop/multi-casskop.env deleted file mode 100644 index b808d14f5..000000000 --- a/multi-casskop/multi-casskop.env +++ /dev/null @@ -1,16 +0,0 @@ -KUBERNETES_PORT=tcp://10.55.240.1:443 -KUBERNETES_PORT_443_TCP=tcp://10.55.240.1:443 -KUBERNETES_PORT_443_TCP_ADDR=10.55.240.1 -KUBERNETES_PORT_443_TCP_PORT=443 -KUBERNETES_PORT_443_TCP_PROTO=tcp -KUBERNETES_SERVICE_HOST=10.55.240.1 -KUBERNETES_SERVICE_PORT=443 -KUBERNETES_SERVICE_PORT_HTTPS=443 -OPERATOR_NAME=multi-casskop -POD_NAME=multi-casskop-138af14ba568470ba3cb33d4a92c10f4-6cd54444b6-bm7t6 -TELEPRESENCE_CONTAINER=multi-casskop -TELEPRESENCE_CONTAINER_NAMESPACE=cassandra-demo -TELEPRESENCE_MOUNTS=/var/run/secrets/kubernetes.io/serviceaccount:/var/run/secrets/admiralty.io/serviceaccountimports/gke-slave-west1-c:/var/run/secrets/admiralty.io/serviceaccountimports/gke-slave-west1-d -TELEPRESENCE_POD=multi-casskop-138af14ba568470ba3cb33d4a92c10f4-6cd54444b6-bm7t6 -TELEPRESENCE_ROOT=/tmp/known -WATCH_NAMESPACE=cassandra-demo From 44445b8b8daa9352cdcd9fb2838fb6500950a16e Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 12:41:22 +0100 Subject: [PATCH 04/41] parameterize jobs to reuse them for multicasskop --- .circleci/config.yml | 45 ++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7766f9505..1c81e838b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,9 +1,19 @@ version: 2.1 +# PARAMETERS +parameters: + workDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + operatorName: + type: string + description: operator's name + default: CassKop # Setting up docker executor using casskop-build image. executors: - casskop-build: + operator-build: docker: - image: orangeopensource/casskop-build:v0.9.0 @@ -109,13 +119,8 @@ orbs: jobs: # Build job, which build operator docker image (with operator-sdk build) build: - parameters: - workdir: - type: string - description: relative path where the build must be run - default: cassandra-k8s-operator - working_directory: /home/circleci/<< parameters.workdir >> - executor: casskop-build + working_directory: /home/circleci/<< parameters.workDir >> + executor: operator-build # STEPS steps: # Checkout steps @@ -123,7 +128,7 @@ jobs: - setup_remote_docker # Creates a separate environment for each build for security. - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. keys: # Use checksum of go.sum to version cache. - - build-{{ .Branch }}-{{ checksum "go.sum" }} + - << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} # Build steps - run: name: Download dependencies @@ -132,7 +137,7 @@ jobs: name: Vendor dependencies command: if [ ! -d vendor ]; then go mod vendor; fi - run: - name: Build Cassandra Operator + name: Build Operator command: make build # Create artifacts - persist_to_workspace: @@ -154,7 +159,7 @@ jobs: # Cache configuration - save_cache: name: Save build artefacts in cache - key: build-{{ .Branch }}-{{ checksum "go.sum" }} + key: << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} paths: - build/_output - pkg/apis/db/v1alpha1/zz_generated.deepcopy.go @@ -162,8 +167,8 @@ jobs: # Unit test job, will execute makefile 'unit-test-with-vendor' step unit-test: - working_directory: /home/circleci/cassandra-k8s-operator - executor: casskop-build + working_directory: /home/circleci/<< parameters.workDir >> + executor: operator-build # STEPS steps: # Checkout steps @@ -185,8 +190,8 @@ jobs: # Dgoss tests job, will execute make 'dgoss-bootstrap' docker-dgoss-test: - working_directory: /home/circleci/cassandra-k8s-operator - executor: casskop-build + working_directory: /home/circleci/<< parameters.workDir >> + executor: operator-build # STEPS steps: # Checkout steps @@ -208,8 +213,8 @@ jobs: # Reporting step, will run sonar report from scan. report: - working_directory: /home/circleci/cassandra-k8s-operator - executor: casskop-build + working_directory: /home/circleci/<< parameters.workDir >> + executor: operator-build # STEPS steps: # Checkout steps @@ -236,13 +241,13 @@ jobs: name: Sonarcloud Scan command: | ./sonar-scanner-3.3.0.1492-linux/bin/sonar-scanner \ - -Dsonar.projectName=CassKop \ + -Dsonar.projectName=<< parameters.operatorName >> \ -Dsonar.projectKey=${SONAR_PROJECT} \ -Dsonar.login=${SONAR_TOKEN} \ -Dsonar.branch.name=${CIRCLE_BRANCH} \ -Dsonar.organization=${SONAR_ORGANISATION} \ -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.projectBaseDir=/home/circleci/cassandra-k8s-operator \ + -Dsonar.projectBaseDir=/home/circleci/<< parameters.workDir >> \ -Dsonar.sources=. \ -Dsonar.sources.inclusions="**/**.go" \ -Dsonar.exclusions="**/*_test.go,**/vendor/**,**/sonar-scanner-3.3.0.1492-linux/**,**docs/**,**/**/zz_generated*" \ @@ -264,7 +269,7 @@ jobs: # Workflow definition workflows: version: 2 - + # CassKop operator workflow. ci-casskop-operator: jobs: - build: From 25994fbbf6eccc5308db28c397a80b6dcca7d850 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 12:45:49 +0100 Subject: [PATCH 05/41] parameterize jobs to reuse them for multicasskop --- .circleci/config.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c81e838b..fad58a5cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -108,7 +108,7 @@ orbs: command: | kubectl create namespace cassandra-e2e ./tools/circleci-get-logs.sh& - make docker-e2e-test-fix-arg <> || { + make docker-e2e-test-fix-arg << parameters.test_name >> || { kubectl get events --all-namespaces --sort-by .metadata.creationTimestamp ; echo "Show Operator logs:" cat operator.log @@ -119,7 +119,7 @@ orbs: jobs: # Build job, which build operator docker image (with operator-sdk build) build: - working_directory: /home/circleci/<< parameters.workDir >> + working_directory: /home/circleci/<< pipeline.parameters.workDir >> executor: operator-build # STEPS steps: @@ -128,7 +128,7 @@ jobs: - setup_remote_docker # Creates a separate environment for each build for security. - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. keys: # Use checksum of go.sum to version cache. - - << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} + - << pipeline.parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} # Build steps - run: name: Download dependencies @@ -159,7 +159,7 @@ jobs: # Cache configuration - save_cache: name: Save build artefacts in cache - key: << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} + key: << pipeline.parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} paths: - build/_output - pkg/apis/db/v1alpha1/zz_generated.deepcopy.go @@ -167,7 +167,7 @@ jobs: # Unit test job, will execute makefile 'unit-test-with-vendor' step unit-test: - working_directory: /home/circleci/<< parameters.workDir >> + working_directory: /home/circleci/<< pipeline.parameters.workDir >> executor: operator-build # STEPS steps: @@ -190,7 +190,7 @@ jobs: # Dgoss tests job, will execute make 'dgoss-bootstrap' docker-dgoss-test: - working_directory: /home/circleci/<< parameters.workDir >> + working_directory: /home/circleci/<< pipeline.parameters.workDir >> executor: operator-build # STEPS steps: @@ -213,7 +213,7 @@ jobs: # Reporting step, will run sonar report from scan. report: - working_directory: /home/circleci/<< parameters.workDir >> + working_directory: /home/circleci/<< pipeline.parameters.workDir >> executor: operator-build # STEPS steps: @@ -241,13 +241,13 @@ jobs: name: Sonarcloud Scan command: | ./sonar-scanner-3.3.0.1492-linux/bin/sonar-scanner \ - -Dsonar.projectName=<< parameters.operatorName >> \ + -Dsonar.projectName=<< pipeline.parameters.operatorName >> \ -Dsonar.projectKey=${SONAR_PROJECT} \ -Dsonar.login=${SONAR_TOKEN} \ -Dsonar.branch.name=${CIRCLE_BRANCH} \ -Dsonar.organization=${SONAR_ORGANISATION} \ -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.projectBaseDir=/home/circleci/<< parameters.workDir >> \ + -Dsonar.projectBaseDir=/home/circleci/<< pipeline.parameters.workDir >> \ -Dsonar.sources=. \ -Dsonar.sources.inclusions="**/**.go" \ -Dsonar.exclusions="**/*_test.go,**/vendor/**,**/sonar-scanner-3.3.0.1492-linux/**,**docs/**,**/**/zz_generated*" \ @@ -311,7 +311,7 @@ workflows: only: /.*/ branches: # Forked pull requests have CIRCLE_BRANCH set to pull/XXX - ignore: /pull\/[0-9]+/ + ignore: /pull\/[0-9w]+/ - minikube/e2e-orb: name: e2e-test ScaleUp From 69ebd128688ce23db7f434746debf6a268806695 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 12:50:54 +0100 Subject: [PATCH 06/41] add orb params --- .circleci/config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fad58a5cb..db7cabe9f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,14 +36,18 @@ orbs: # Setting up job list jobs: e2e-orb: # Define job which execute 'docker-e2e-test-fix-arg' makefile job - executor: vm-classic - working_directory: /home/circleci/cassandra-k8s-operator # PARAMETERS parameters: test_name: description: name of the test in e2e code type: string default: toto + workDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + executor: vm-classic + working_directory: /home/circleci/<< parameters.workDir >> # ENVIRONMENTS VARIABLES environment: K8S_VERSION: v1.14.0 From fa5e3b376d4dc244e241fd42496baedaffc6e867 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 13:36:09 +0100 Subject: [PATCH 07/41] add multi-casskop pipeline --- .circleci/config.yml | 98 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index db7cabe9f..309b50ee6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,16 +1,5 @@ version: 2.1 -# PARAMETERS -parameters: - workDir: - type: string - description: relative path where the build must be run - default: cassandra-k8s-operator - operatorName: - type: string - description: operator's name - default: CassKop - # Setting up docker executor using casskop-build image. executors: operator-build: @@ -123,7 +112,17 @@ orbs: jobs: # Build job, which build operator docker image (with operator-sdk build) build: - working_directory: /home/circleci/<< pipeline.parameters.workDir >> + # PARAMETERS + parameters: + workDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + operatorName: + type: string + description: operator's name + default: CassKop + working_directory: /home/circleci/<< parameters.workDir >> executor: operator-build # STEPS steps: @@ -132,7 +131,7 @@ jobs: - setup_remote_docker # Creates a separate environment for each build for security. - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. keys: # Use checksum of go.sum to version cache. - - << pipeline.parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} + - << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} # Build steps - run: name: Download dependencies @@ -163,7 +162,7 @@ jobs: # Cache configuration - save_cache: name: Save build artefacts in cache - key: << pipeline.parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} + key: << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} paths: - build/_output - pkg/apis/db/v1alpha1/zz_generated.deepcopy.go @@ -171,7 +170,17 @@ jobs: # Unit test job, will execute makefile 'unit-test-with-vendor' step unit-test: - working_directory: /home/circleci/<< pipeline.parameters.workDir >> + # PARAMETERS + parameters: + workDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + operatorName: + type: string + description: operator's name + default: CassKop + working_directory: /home/circleci/<< parameters.workDir >> executor: operator-build # STEPS steps: @@ -194,7 +203,17 @@ jobs: # Dgoss tests job, will execute make 'dgoss-bootstrap' docker-dgoss-test: - working_directory: /home/circleci/<< pipeline.parameters.workDir >> + # PARAMETERS + parameters: + workDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + operatorName: + type: string + description: operator's name + default: CassKop + working_directory: /home/circleci/<< parameters.workDir >> executor: operator-build # STEPS steps: @@ -217,7 +236,17 @@ jobs: # Reporting step, will run sonar report from scan. report: - working_directory: /home/circleci/<< pipeline.parameters.workDir >> + # PARAMETERS + parameters: + workDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + operatorName: + type: string + description: operator's name + default: CassKop + working_directory: /home/circleci/<< parameters.workDir >> executor: operator-build # STEPS steps: @@ -245,13 +274,13 @@ jobs: name: Sonarcloud Scan command: | ./sonar-scanner-3.3.0.1492-linux/bin/sonar-scanner \ - -Dsonar.projectName=<< pipeline.parameters.operatorName >> \ + -Dsonar.projectName=<< parameters.operatorName >> \ -Dsonar.projectKey=${SONAR_PROJECT} \ -Dsonar.login=${SONAR_TOKEN} \ -Dsonar.branch.name=${CIRCLE_BRANCH} \ -Dsonar.organization=${SONAR_ORGANISATION} \ -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.projectBaseDir=/home/circleci/<< pipeline.parameters.workDir >> \ + -Dsonar.projectBaseDir=/home/circleci/<< parameters.workDir >> \ -Dsonar.sources=. \ -Dsonar.sources.inclusions="**/**.go" \ -Dsonar.exclusions="**/*_test.go,**/vendor/**,**/sonar-scanner-3.3.0.1492-linux/**,**docs/**,**/**/zz_generated*" \ @@ -387,4 +416,33 @@ workflows: only: /.*/ branches: # Forked pull requests have CIRCLE_BRANCH set to pull/XXX - ignore: /pull\/[0-9]+/ \ No newline at end of file + ignore: /pull\/[0-9]+/ + + # CassKop operator workflow. + ci-multicasskop-operator: + jobs: + - build: + workDir: cassandra-k8s-operator/multi-casskop + operatorName: MultiCassKop + filters: + tags: + only: /.*/ + - unit-test: + workDir: cassandra-k8s-operator/multi-casskop + operatorName: MultiCassKop + requires: + - build + filters: + tags: + only: /.*/ + - report: + workDir: cassandra-k8s-operator/multi-casskop + operatorName: MultiCassKop + requires: + - unit-test + filters: + tags: + only: /.*/ + branches: + # report sonar only from canonical repository + only: /^(?!pull\/).*$/ # https://stackoverflow.com/a/5334825/358804 \ No newline at end of file From ebce2258ab86d9f3b49bacdfdc4088d9ba6cb895 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 13:43:39 +0100 Subject: [PATCH 08/41] debug --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 309b50ee6..857f20b64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -141,7 +141,7 @@ jobs: command: if [ ! -d vendor ]; then go mod vendor; fi - run: name: Build Operator - command: make build + command: echo $pwd && ls -la && make build # Create artifacts - persist_to_workspace: root: ./ From f045e655451a38d179e0eca1b5f4eb3dbcd24a2e Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 14:39:43 +0100 Subject: [PATCH 09/41] debug --- .circleci/config.yml | 47 +++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 857f20b64..a0c940936 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,12 +31,12 @@ orbs: description: name of the test in e2e code type: string default: toto - workDir: + operatorDir: type: string description: relative path where the build must be run default: cassandra-k8s-operator executor: vm-classic - working_directory: /home/circleci/<< parameters.workDir >> + working_directory: /home/circleci/cassandra-k8s-operator # ENVIRONMENTS VARIABLES environment: K8S_VERSION: v1.14.0 @@ -73,6 +73,9 @@ orbs: - checkout - attach_workspace: # Create artifact from workdir at: ./ + - run: + name: Change workdir + command: cd ${HOME}/<< parameters.operatorDir >> # Check environment set up correctly - run: # Wait until minikube nodes are ready name: Wait for minikube @@ -114,7 +117,7 @@ jobs: build: # PARAMETERS parameters: - workDir: + operatorDir: type: string description: relative path where the build must be run default: cassandra-k8s-operator @@ -122,7 +125,7 @@ jobs: type: string description: operator's name default: CassKop - working_directory: /home/circleci/<< parameters.workDir >> + working_directory: /home/circleci/cassandra-k8s-operator executor: operator-build # STEPS steps: @@ -132,6 +135,9 @@ jobs: - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. keys: # Use checksum of go.sum to version cache. - << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} + - run: + name: Change workdir + command: cd ${HOME}/<< parameters.operatorDir >> # Build steps - run: name: Download dependencies @@ -141,7 +147,7 @@ jobs: command: if [ ! -d vendor ]; then go mod vendor; fi - run: name: Build Operator - command: echo $pwd && ls -la && make build + command: ls -la ${CIRCLE_WORKING_DIRECTORY} && ls -la && make build # Create artifacts - persist_to_workspace: root: ./ @@ -172,7 +178,7 @@ jobs: unit-test: # PARAMETERS parameters: - workDir: + operatorDir: type: string description: relative path where the build must be run default: cassandra-k8s-operator @@ -180,7 +186,7 @@ jobs: type: string description: operator's name default: CassKop - working_directory: /home/circleci/<< parameters.workDir >> + working_directory: /home/circleci/cassandra-k8s-operator executor: operator-build # STEPS steps: @@ -188,6 +194,9 @@ jobs: - checkout - attach_workspace: # Create artifact from workdir at: ./ + - run: + name: Change workdir + command: cd ${HOME}/<< parameters.operatorDir >> # Unit tests steps - run: name: Unit Tests @@ -205,7 +214,7 @@ jobs: docker-dgoss-test: # PARAMETERS parameters: - workDir: + operatorDir: type: string description: relative path where the build must be run default: cassandra-k8s-operator @@ -213,7 +222,7 @@ jobs: type: string description: operator's name default: CassKop - working_directory: /home/circleci/<< parameters.workDir >> + working_directory: /home/circleci/cassandra-k8s-operator executor: operator-build # STEPS steps: @@ -222,6 +231,9 @@ jobs: - setup_remote_docker - attach_workspace: # Create artifact from workdir at: ./ + - run: + name: Change workdir + command: cd ${HOME}/<< parameters.operatorDir >> # Dgoss tests steps - run: # Run test wich enable to validate bootstrap image. name: Dgoss tests @@ -238,7 +250,7 @@ jobs: report: # PARAMETERS parameters: - workDir: + operatorDir: type: string description: relative path where the build must be run default: cassandra-k8s-operator @@ -246,7 +258,7 @@ jobs: type: string description: operator's name default: CassKop - working_directory: /home/circleci/<< parameters.workDir >> + working_directory: /home/circleci/cassandra-k8s-operator executor: operator-build # STEPS steps: @@ -258,11 +270,14 @@ jobs: name: Restore SonarQube Scanner keys: - sonar-scanner-3.3.0.1492-linux + - run: + name: Change workdir + command: cd ${HOME}/<< parameters.operatorDir >> # Sonar steps - run: name: Download SonarQube Scanner command: | - chmod +x .circleci/install_sonar-scanner.sh + chmod +x /home/circleci/cassandra-k8s-operator/install_sonar-scanner.sh ./.circleci/install_sonar-scanner.sh - save_cache: # Cache sonar download name: Cache SonarQube Scanner @@ -280,7 +295,7 @@ jobs: -Dsonar.branch.name=${CIRCLE_BRANCH} \ -Dsonar.organization=${SONAR_ORGANISATION} \ -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.projectBaseDir=/home/circleci/<< parameters.workDir >> \ + -Dsonar.projectBaseDir=/home/circleci/<< parameters.operatorDir >> \ -Dsonar.sources=. \ -Dsonar.sources.inclusions="**/**.go" \ -Dsonar.exclusions="**/*_test.go,**/vendor/**,**/sonar-scanner-3.3.0.1492-linux/**,**docs/**,**/**/zz_generated*" \ @@ -422,13 +437,13 @@ workflows: ci-multicasskop-operator: jobs: - build: - workDir: cassandra-k8s-operator/multi-casskop + operatorDir: cassandra-k8s-operator/multi-casskop operatorName: MultiCassKop filters: tags: only: /.*/ - unit-test: - workDir: cassandra-k8s-operator/multi-casskop + operatorDir: cassandra-k8s-operator/multi-casskop operatorName: MultiCassKop requires: - build @@ -436,7 +451,7 @@ workflows: tags: only: /.*/ - report: - workDir: cassandra-k8s-operator/multi-casskop + operatorDir: cassandra-k8s-operator/multi-casskop operatorName: MultiCassKop requires: - unit-test From 240c4bdf6f1526a19df56ed89830c150dc19526c Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 14:41:14 +0100 Subject: [PATCH 10/41] debug --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0c940936..387d2392a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -125,7 +125,7 @@ jobs: type: string description: operator's name default: CassKop - working_directory: /home/circleci/cassandra-k8s-operator + working_directory: /home/circleci/cassandra-k8s-operator executor: operator-build # STEPS steps: From 23152df3dc4318238da046b765fdd65d8f923e16 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 14:43:05 +0100 Subject: [PATCH 11/41] debug --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 387d2392a..5f0845fd7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,7 @@ orbs: at: ./ - run: name: Change workdir - command: cd ${HOME}/<< parameters.operatorDir >> + command: cd /home/circleci/<< parameters.operatorDir >> # Check environment set up correctly - run: # Wait until minikube nodes are ready name: Wait for minikube @@ -137,7 +137,7 @@ jobs: - << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} - run: name: Change workdir - command: cd ${HOME}/<< parameters.operatorDir >> + command: cd /home/circleci/<< parameters.operatorDir >> # Build steps - run: name: Download dependencies @@ -196,7 +196,7 @@ jobs: at: ./ - run: name: Change workdir - command: cd ${HOME}/<< parameters.operatorDir >> + command: cd /home/circleci/<< parameters.operatorDir >> # Unit tests steps - run: name: Unit Tests @@ -233,7 +233,7 @@ jobs: at: ./ - run: name: Change workdir - command: cd ${HOME}/<< parameters.operatorDir >> + command: cd /home/circleci/<< parameters.operatorDir >> # Dgoss tests steps - run: # Run test wich enable to validate bootstrap image. name: Dgoss tests @@ -272,7 +272,7 @@ jobs: - sonar-scanner-3.3.0.1492-linux - run: name: Change workdir - command: cd ${HOME}/<< parameters.operatorDir >> + command: cd /home/circleci/<< parameters.operatorDir >> # Sonar steps - run: name: Download SonarQube Scanner From 2ca7e0b11dd827b089af65cf77c6e8c3a8ca077b Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 14:54:36 +0100 Subject: [PATCH 12/41] debug --- .circleci/config.yml | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f0845fd7..df8c56a0e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,7 +36,7 @@ orbs: description: relative path where the build must be run default: cassandra-k8s-operator executor: vm-classic - working_directory: /home/circleci/cassandra-k8s-operator + working_directory: /home/circleci/<< parameters.operatorDir >> # ENVIRONMENTS VARIABLES environment: K8S_VERSION: v1.14.0 @@ -70,12 +70,10 @@ orbs: mkdir -p ${HOME}/.kube touch ${HOME}/.kube/config # Checkout steps - - checkout + - checkout: + path: /home/circleci/cassandra-k8s-operator - attach_workspace: # Create artifact from workdir at: ./ - - run: - name: Change workdir - command: cd /home/circleci/<< parameters.operatorDir >> # Check environment set up correctly - run: # Wait until minikube nodes are ready name: Wait for minikube @@ -125,19 +123,17 @@ jobs: type: string description: operator's name default: CassKop - working_directory: /home/circleci/cassandra-k8s-operator + working_directory: /home/circleci/<< parameters.operatorDir >> executor: operator-build # STEPS steps: # Checkout steps - - checkout + - checkout: + path: /home/circleci/cassandra-k8s-operator - setup_remote_docker # Creates a separate environment for each build for security. - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. keys: # Use checksum of go.sum to version cache. - << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} - - run: - name: Change workdir - command: cd /home/circleci/<< parameters.operatorDir >> # Build steps - run: name: Download dependencies @@ -186,17 +182,15 @@ jobs: type: string description: operator's name default: CassKop - working_directory: /home/circleci/cassandra-k8s-operator + working_directory: /home/circleci/<< parameters.operatorDir >> executor: operator-build # STEPS steps: # Checkout steps - - checkout + - checkout: + path: /home/circleci/cassandra-k8s-operator - attach_workspace: # Create artifact from workdir at: ./ - - run: - name: Change workdir - command: cd /home/circleci/<< parameters.operatorDir >> # Unit tests steps - run: name: Unit Tests @@ -222,18 +216,16 @@ jobs: type: string description: operator's name default: CassKop - working_directory: /home/circleci/cassandra-k8s-operator + working_directory: /home/circleci/<< parameters.operatorDir >> executor: operator-build # STEPS steps: # Checkout steps - - checkout + - checkout: + path: /home/circleci/cassandra-k8s-operator - setup_remote_docker - attach_workspace: # Create artifact from workdir at: ./ - - run: - name: Change workdir - command: cd /home/circleci/<< parameters.operatorDir >> # Dgoss tests steps - run: # Run test wich enable to validate bootstrap image. name: Dgoss tests @@ -258,21 +250,19 @@ jobs: type: string description: operator's name default: CassKop - working_directory: /home/circleci/cassandra-k8s-operator + working_directory: /home/circleci/<< parameters.operatorDir >> executor: operator-build # STEPS steps: # Checkout steps - - checkout + - checkout: + path: /home/circleci/cassandra-k8s-operator - attach_workspace: at: ./ - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. name: Restore SonarQube Scanner keys: - sonar-scanner-3.3.0.1492-linux - - run: - name: Change workdir - command: cd /home/circleci/<< parameters.operatorDir >> # Sonar steps - run: name: Download SonarQube Scanner From 99774fa11d2ce9df227c3ad3d8bcd85abf80e56f Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 16:08:15 +0100 Subject: [PATCH 13/41] try to restructure pipeline --- .circleci/config.yml | 522 +++++++++++++++++++++++++------------------ Makefile | 7 + 2 files changed, 311 insertions(+), 218 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index df8c56a0e..360853c22 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,11 +1,5 @@ version: 2.1 -# Setting up docker executor using casskop-build image. -executors: - operator-build: - docker: - - image: orangeopensource/casskop-build:v0.9.0 - experimental: notify: @@ -72,8 +66,11 @@ orbs: # Checkout steps - checkout: path: /home/circleci/cassandra-k8s-operator - - attach_workspace: # Create artifact from workdir - at: ./ + - attach_workspace: # Create artifact from workdir + at: /tmp/workspace + - run: + name: Copying artifacts + command: mv /tmp/workspace/* ./ # Check environment set up correctly - run: # Wait until minikube nodes are ready name: Wait for minikube @@ -108,201 +105,257 @@ orbs: cat operator.log false; } + operator: + # Setting up docker executor using casskop-build image. + executors: + operator-build: + docker: + - image: orangeopensource/casskop-build:v0.9.0 + # Define jobs list + jobs: + # Build job, which build operator docker image (with operator-sdk build) + generate: + # PARAMETERS + parameters: + operatorDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + operatorName: + type: string + description: operator's name + default: CassKop + working_directory: /home/circleci/<< parameters.operatorDir >> + executor: operator-build + # STEPS + steps: + # Checkout steps + - checkout: + path: /home/circleci/cassandra-k8s-operator + - setup_remote_docker # Creates a separate environment for each build for security. + - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. + keys: # Use checksum of go.sum to version cache. + - << parameters.operatorName >>-generate-{{ .Branch }}-{{ checksum "go.sum" }} + - attach_workspace: # Create artifact from workdir + at: /tmp/workspace + - run: + name: Copying artifacts + command: mv /tmp/workspace/* ./ + # Build steps + - run: + name: Download dependencies + command: if [ ! -d vendor ]; then go mod download; fi + - run: + name: Vendor dependencies + command: if [ ! -d vendor ]; then go mod vendor; fi + - run: + name: Build Operator + command: make generate + # Create artifacts + - persist_to_workspace: + root: /tmp/workspace/<< parameters.operatorDir >> + paths: + - build/_output + - pkg/apis/db/v1alpha1/zz_generated.deepcopy.go + - vendor -# Define jobs list -jobs: - # Build job, which build operator docker image (with operator-sdk build) - build: - # PARAMETERS - parameters: - operatorDir: - type: string - description: relative path where the build must be run - default: cassandra-k8s-operator - operatorName: - type: string - description: operator's name - default: CassKop - working_directory: /home/circleci/<< parameters.operatorDir >> - executor: operator-build - # STEPS - steps: - # Checkout steps - - checkout: - path: /home/circleci/cassandra-k8s-operator - - setup_remote_docker # Creates a separate environment for each build for security. - - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. - keys: # Use checksum of go.sum to version cache. - - << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} - # Build steps - - run: - name: Download dependencies - command: if [ ! -d vendor ]; then go mod download; fi - - run: - name: Vendor dependencies - command: if [ ! -d vendor ]; then go mod vendor; fi - - run: - name: Build Operator - command: ls -la ${CIRCLE_WORKING_DIRECTORY} && ls -la && make build - # Create artifacts - - persist_to_workspace: - root: ./ - paths: - - build/_output - - pkg/apis/db/v1alpha1/zz_generated.deepcopy.go - - vendor - # Deploy steps - - deploy: - name: Push image to Docker Hub - command: | - if [[ $(echo "$CIRCLE_BRANCH" | grep -c "pull") -gt 0 ]]; then - echo "This is a PR, we don't push to Hub." - else - docker login --username $DOCKERHUB_USER --password $DOCKERHUB_PASSWORD - make push - fi - # Cache configuration - - save_cache: - name: Save build artefacts in cache - key: << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} - paths: - - build/_output - - pkg/apis/db/v1alpha1/zz_generated.deepcopy.go - - vendor + # Build job, which build operator docker image (with operator-sdk build) + build: + # PARAMETERS + parameters: + operatorDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + operatorName: + type: string + description: operator's name + default: CassKop + working_directory: /home/circleci/<< parameters.operatorDir >> + executor: operator-build + # STEPS + steps: + # Checkout steps + - checkout: + path: /home/circleci/cassandra-k8s-operator + - setup_remote_docker # Creates a separate environment for each build for security. + - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. + keys: # Use checksum of go.sum to version cache. + - << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} + - attach_workspace: # Create artifact from workdir + at: /tmp/workspace + - run: + name: Copying artifacts + command: mv /tmp/workspace/* ./ + # Build steps + - run: + name: Build Operator + command: make build + # Create artifacts + - persist_to_workspace: + root: /tmp/workspace/<< parameters.operatorDir >> + paths: + - build/_output + - pkg/apis/db/v1alpha1/zz_generated.deepcopy.go + - vendor + # Deploy steps + - deploy: + name: Push image to Docker Hub + command: | + if [[ $(echo "$CIRCLE_BRANCH" | grep -c "pull") -gt 0 ]]; then + echo "This is a PR, we don't push to Hub." + else + docker login --username $DOCKERHUB_USER --password $DOCKERHUB_PASSWORD + make push + fi + # Cache configuration + - save_cache: + name: Save build artefacts in cache + key: << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} + paths: + - build/_output + - pkg/apis/db/v1alpha1/zz_generated.deepcopy.go + - vendor - # Unit test job, will execute makefile 'unit-test-with-vendor' step - unit-test: - # PARAMETERS - parameters: - operatorDir: - type: string - description: relative path where the build must be run - default: cassandra-k8s-operator - operatorName: - type: string - description: operator's name - default: CassKop - working_directory: /home/circleci/<< parameters.operatorDir >> - executor: operator-build - # STEPS - steps: - # Checkout steps - - checkout: - path: /home/circleci/cassandra-k8s-operator - - attach_workspace: # Create artifact from workdir - at: ./ - # Unit tests steps - - run: - name: Unit Tests - command: make unit-test-with-vendor - # Artifacts definitions - - store_artifacts: # Store unit test coverage reports - path: coverage.html - - persist_to_workspace: # Persist unit tests *.out files - root: ./ - paths: - - coverage.out - - test-report.out + # Unit test job, will execute makefile 'unit-test-with-vendor' step + unit-test: + # PARAMETERS + parameters: + operatorDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + operatorName: + type: string + description: operator's name + default: CassKop + working_directory: /home/circleci/<< parameters.operatorDir >> + executor: operator-build + # STEPS + steps: + # Checkout steps + - checkout: + path: /home/circleci/cassandra-k8s-operator + - attach_workspace: # Create artifact from workdir + at: /tmp/workspace + - run: + name: Copying artifacts + command: mv /tmp/workspace/* ./ + # Unit tests steps + - run: + name: Unit Tests + command: make unit-test-with-vendor + # Artifacts definitions + - store_artifacts: # Store unit test coverage reports + path: coverage.html + - persist_to_workspace: + root: /tmp/workspace/<< parameters.operatorDir >> + paths: + - coverage.out + - test-report.out - # Dgoss tests job, will execute make 'dgoss-bootstrap' - docker-dgoss-test: - # PARAMETERS - parameters: - operatorDir: - type: string - description: relative path where the build must be run - default: cassandra-k8s-operator - operatorName: - type: string - description: operator's name - default: CassKop - working_directory: /home/circleci/<< parameters.operatorDir >> - executor: operator-build - # STEPS - steps: - # Checkout steps - - checkout: - path: /home/circleci/cassandra-k8s-operator - - setup_remote_docker - - attach_workspace: # Create artifact from workdir - at: ./ - # Dgoss tests steps - - run: # Run test wich enable to validate bootstrap image. - name: Dgoss tests - command: | - curl -L https://raw.githubusercontent.com/aelsabbahy/goss/master/extras/dgoss/dgoss -o /usr/local/bin/dgoss - chmod +rx /usr/local/bin/dgoss - curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.6/goss-linux-amd64 -o /usr/local/bin/goss - chmod +rx /usr/local/bin/goss - export GOSS_PATH=/usr/local/bin/goss - export GOSS_FILES_STRATEGY=cp - make dgoss-bootstrap + # Dgoss tests job, will execute make 'dgoss-bootstrap' + docker-dgoss-test: + # PARAMETERS + parameters: + operatorDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + operatorName: + type: string + description: operator's name + default: CassKop + working_directory: /home/circleci/<< parameters.operatorDir >> + executor: operator-build + # STEPS + steps: + # Checkout steps + - checkout: + path: /home/circleci/cassandra-k8s-operator + - setup_remote_docker + - attach_workspace: # Create artifact from workdir + at: /tmp/workspace + - run: + name: Copying artifacts + command: mv /tmp/workspace/* ./ + # Dgoss tests steps + - run: # Run test wich enable to validate bootstrap image. + name: Dgoss tests + command: | + curl -L https://raw.githubusercontent.com/aelsabbahy/goss/master/extras/dgoss/dgoss -o /usr/local/bin/dgoss + chmod +rx /usr/local/bin/dgoss + curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.6/goss-linux-amd64 -o /usr/local/bin/goss + chmod +rx /usr/local/bin/goss + export GOSS_PATH=/usr/local/bin/goss + export GOSS_FILES_STRATEGY=cp + make dgoss-bootstrap - # Reporting step, will run sonar report from scan. - report: - # PARAMETERS - parameters: - operatorDir: - type: string - description: relative path where the build must be run - default: cassandra-k8s-operator - operatorName: - type: string - description: operator's name - default: CassKop - working_directory: /home/circleci/<< parameters.operatorDir >> - executor: operator-build - # STEPS - steps: - # Checkout steps - - checkout: - path: /home/circleci/cassandra-k8s-operator - - attach_workspace: - at: ./ - - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. - name: Restore SonarQube Scanner - keys: - - sonar-scanner-3.3.0.1492-linux - # Sonar steps - - run: - name: Download SonarQube Scanner - command: | - chmod +x /home/circleci/cassandra-k8s-operator/install_sonar-scanner.sh - ./.circleci/install_sonar-scanner.sh - - save_cache: # Cache sonar download - name: Cache SonarQube Scanner - key: sonar-scanner-3.3.0.1492-linux - paths: - - sonar-scanner-3.3.0.1492-linux - - /home/circleci/cassandra-k8s-operator/.sonar - - run: # Run sonar scan - name: Sonarcloud Scan - command: | - ./sonar-scanner-3.3.0.1492-linux/bin/sonar-scanner \ - -Dsonar.projectName=<< parameters.operatorName >> \ - -Dsonar.projectKey=${SONAR_PROJECT} \ - -Dsonar.login=${SONAR_TOKEN} \ - -Dsonar.branch.name=${CIRCLE_BRANCH} \ - -Dsonar.organization=${SONAR_ORGANISATION} \ - -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.projectBaseDir=/home/circleci/<< parameters.operatorDir >> \ - -Dsonar.sources=. \ - -Dsonar.sources.inclusions="**/**.go" \ - -Dsonar.exclusions="**/*_test.go,**/vendor/**,**/sonar-scanner-3.3.0.1492-linux/**,**docs/**,**/**/zz_generated*" \ - -Dsonar.coverage.exclusions="**/vendor/**,**/test/**,**docs/**" \ - -Dsonar.tests=. \ - -Dsonar.language=go \ - -Dsonar.sourceEncoding=UTF-8 \ - -Dsonar.test.inclusions="**/**_test.go" \ - -Dsonar.test.exclusions="**/vendor/**" \ - -Dsonar.go.coverage.reportPaths=coverage.out \ - -Dsonar.go.tests.reportPaths=test-report.out \ - -Dsonar.coverage.dtdVerification=false \ - -Dsonar.log.level=INFO - cat -n .scannerwork/report-task.txt - # Artifacts definitions - - store_test_results: # Store sonar scan reports - path: reports + # Reporting step, will run sonar report from scan. + report: + # PARAMETERS + parameters: + operatorDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + operatorName: + type: string + description: operator's name + default: CassKop + working_directory: /home/circleci/<< parameters.operatorDir >> + executor: operator-build + # STEPS + steps: + # Checkout steps + - checkout: + path: /home/circleci/cassandra-k8s-operator + - attach_workspace: + at: ./ + - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. + name: Restore SonarQube Scanner + keys: + - sonar-scanner-3.3.0.1492-linux + # Sonar steps + - run: + name: Download SonarQube Scanner + command: | + chmod +x /home/circleci/cassandra-k8s-operator/install_sonar-scanner.sh + ./.circleci/install_sonar-scanner.sh + - save_cache: # Cache sonar download + name: Cache SonarQube Scanner + key: sonar-scanner-3.3.0.1492-linux + paths: + - sonar-scanner-3.3.0.1492-linux + - /home/circleci/cassandra-k8s-operator/.sonar + - run: # Run sonar scan + name: Sonarcloud Scan + command: | + ./sonar-scanner-3.3.0.1492-linux/bin/sonar-scanner \ + -Dsonar.projectName=<< parameters.operatorName >> \ + -Dsonar.projectKey=${SONAR_PROJECT} \ + -Dsonar.login=${SONAR_TOKEN} \ + -Dsonar.branch.name=${CIRCLE_BRANCH} \ + -Dsonar.organization=${SONAR_ORGANISATION} \ + -Dsonar.host.url=https://sonarcloud.io \ + -Dsonar.projectBaseDir=/home/circleci/<< parameters.operatorDir >> \ + -Dsonar.sources=. \ + -Dsonar.sources.inclusions="**/**.go" \ + -Dsonar.exclusions="**/*_test.go,**/vendor/**,**/sonar-scanner-3.3.0.1492-linux/**,**docs/**,**/**/zz_generated*" \ + -Dsonar.coverage.exclusions="**/vendor/**,**/test/**,**docs/**" \ + -Dsonar.tests=. \ + -Dsonar.language=go \ + -Dsonar.sourceEncoding=UTF-8 \ + -Dsonar.test.inclusions="**/**_test.go" \ + -Dsonar.test.exclusions="**/vendor/**" \ + -Dsonar.go.coverage.reportPaths=coverage.out \ + -Dsonar.go.tests.reportPaths=test-report.out \ + -Dsonar.coverage.dtdVerification=false \ + -Dsonar.log.level=INFO + cat -n .scannerwork/report-task.txt + # Artifacts definitions + - store_test_results: # Store sonar scan reports + path: reports # Workflow definition workflows: @@ -310,28 +363,40 @@ workflows: # CassKop operator workflow. ci-casskop-operator: jobs: - - build: + - operator/generate: + name: generate-casskop filters: tags: only: /.*/ - - unit-test: + - operator/unit-test: + name: unit-test-casskop requires: - - build + - generate-casskop filters: tags: only: /.*/ - - docker-dgoss-test: + - operator/build: + name: build-casskop requires: - - build + - unit-test-casskop filters: tags: only: /.*/ - - report: + - operator/docker-dgoss-test: + name: docker-dgoss-test-casskop requires: - - unit-test + - build-casskop + filters: + tags: + only: /.*/ + + - operator/report: + name: report-casskop + requires: + - unit-test-casskop filters: tags: only: /.*/ @@ -343,7 +408,7 @@ workflows: name: e2e-test RollingRestart test_name: RollingRestart requires: - - unit-test + - unit-test-casskop filters: tags: only: /.*/ @@ -355,7 +420,7 @@ workflows: name: e2e-test ScaleUp test_name: ClusterScaleUp requires: - - unit-test + - unit-test-casskop filters: tags: only: /.*/ @@ -367,7 +432,7 @@ workflows: name: e2e-test ScaleDown test_name: ClusterScaleDown requires: - - unit-test + - unit-test-casskop filters: tags: only: /.*/ @@ -379,7 +444,7 @@ workflows: name: e2e-test ScaleDownSimple test_name: ClusterScaleDownSimple requires: - - unit-test + - unit-test-casskop filters: tags: only: /.*/ @@ -391,7 +456,7 @@ workflows: name: e2e-test CreateOneClusterService test_name: CreateOneClusterService requires: - - unit-test + - unit-test-casskop filters: tags: only: /.*/ @@ -403,7 +468,7 @@ workflows: name: e2e-test UpdateConfigMap test_name: UpdateConfigMap requires: - - unit-test + - unit-test-casskop filters: tags: only: /.*/ @@ -415,7 +480,7 @@ workflows: name: e2e-test ExecuteCleanup test_name: ExecuteCleanup requires: - - unit-test + - unit-test-casskop filters: tags: only: /.*/ @@ -426,25 +491,46 @@ workflows: # CassKop operator workflow. ci-multicasskop-operator: jobs: - - build: + - operator/generate: + name: generate-casskop + operatorDir: cassandra-k8s-operator + operatorName: CassKop + filters: + tags: + only: /.*/ + - operator/generate: + name: generate-multicasskop + operatorDir: cassandra-k8s-operator/multi-casskop + operatorName: MultiCassKop + requires: + - generate-casskop + filters: + tags: + only: /.*/ + - operator/unit-test: + name: unit-test-multicasskop operatorDir: cassandra-k8s-operator/multi-casskop operatorName: MultiCassKop + requires: + - generate-multicasskop filters: tags: only: /.*/ - - unit-test: + - operator/build: + name: build-multicasskop operatorDir: cassandra-k8s-operator/multi-casskop operatorName: MultiCassKop requires: - - build + - unit-test-multicasskop filters: tags: only: /.*/ - - report: + - operator/report: + name: report-multicasskop operatorDir: cassandra-k8s-operator/multi-casskop operatorName: MultiCassKop requires: - - unit-test + - unit-test-multicasskop filters: tags: only: /.*/ diff --git a/Makefile b/Makefile index 7914fbc83..7eb40f3d5 100644 --- a/Makefile +++ b/Makefile @@ -161,6 +161,13 @@ helm-package: helm repo index docs/helm/ make -C multi-casskop helm-package +# +.PHONY: generate +generate: + echo "Generate zzz-deepcopy objects" + operator-sdk version + operator-sdk generate k8s + # Build cassandra-k8s-operator executable file in local go env .PHONY: build build: From a52af86546700abdebb7c45d4705df38d0355b52 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 16:13:17 +0100 Subject: [PATCH 14/41] try to restructure pipeline --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 360853c22..7dbaad29b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,7 @@ orbs: at: /tmp/workspace - run: name: Copying artifacts - command: mv /tmp/workspace/* ./ + command: if [ -d "/tmp/workspace" ]; then mv /tmp/workspace/* ./; fi # Check environment set up correctly - run: # Wait until minikube nodes are ready name: Wait for minikube @@ -140,7 +140,7 @@ orbs: at: /tmp/workspace - run: name: Copying artifacts - command: mv /tmp/workspace/* ./ + command: if [ -d "/tmp/workspace" ]; then mv /tmp/workspace/* ./; fi # Build steps - run: name: Download dependencies @@ -186,7 +186,7 @@ orbs: at: /tmp/workspace - run: name: Copying artifacts - command: mv /tmp/workspace/* ./ + command: if [ -d "/tmp/workspace" ]; then mv /tmp/workspace/* ./; fi # Build steps - run: name: Build Operator @@ -240,7 +240,7 @@ orbs: at: /tmp/workspace - run: name: Copying artifacts - command: mv /tmp/workspace/* ./ + command: if [ -d "/tmp/workspace" ]; then mv /tmp/workspace/* ./; fi # Unit tests steps - run: name: Unit Tests @@ -278,7 +278,7 @@ orbs: at: /tmp/workspace - run: name: Copying artifacts - command: mv /tmp/workspace/* ./ + command: if [ -d "/tmp/workspace" ]; then mv /tmp/workspace/* ./; fi # Dgoss tests steps - run: # Run test wich enable to validate bootstrap image. name: Dgoss tests From 3fc7f54088085155d7c0f7684811e8dadce4fab3 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 16:20:51 +0100 Subject: [PATCH 15/41] try to restructure pipeline --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7dbaad29b..7d3cf87ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,7 @@ orbs: at: /tmp/workspace - run: name: Copying artifacts - command: if [ -d "/tmp/workspace" ]; then mv /tmp/workspace/* ./; fi + command: if [[ -d "/tmp/workspace" && "$(ls -A /tmp/workspace)" ]]; then mv /tmp/workspace/* ./; fi # Check environment set up correctly - run: # Wait until minikube nodes are ready name: Wait for minikube @@ -140,7 +140,7 @@ orbs: at: /tmp/workspace - run: name: Copying artifacts - command: if [ -d "/tmp/workspace" ]; then mv /tmp/workspace/* ./; fi + command: if [[ -d "/tmp/workspace" && "$(ls -A /tmp/workspace)" ]]; then mv /tmp/workspace/* ./; fi # Build steps - run: name: Download dependencies @@ -186,7 +186,7 @@ orbs: at: /tmp/workspace - run: name: Copying artifacts - command: if [ -d "/tmp/workspace" ]; then mv /tmp/workspace/* ./; fi + command: if [[ -d "/tmp/workspace" && "$(ls -A /tmp/workspace)" ]]; then mv /tmp/workspace/* ./; fi # Build steps - run: name: Build Operator @@ -240,7 +240,7 @@ orbs: at: /tmp/workspace - run: name: Copying artifacts - command: if [ -d "/tmp/workspace" ]; then mv /tmp/workspace/* ./; fi + command: if [[ -d "/tmp/workspace" && "$(ls -A /tmp/workspace)" ]]; then mv /tmp/workspace/* ./; fi # Unit tests steps - run: name: Unit Tests @@ -278,7 +278,7 @@ orbs: at: /tmp/workspace - run: name: Copying artifacts - command: if [ -d "/tmp/workspace" ]; then mv /tmp/workspace/* ./; fi + command: if [[ -d "/tmp/workspace" && "$(ls -A /tmp/workspace)" ]]; then mv /tmp/workspace/* ./; fi # Dgoss tests steps - run: # Run test wich enable to validate bootstrap image. name: Dgoss tests From 5fd450d77e25c104a97433b5f357a72385733141 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 16:29:35 +0100 Subject: [PATCH 16/41] try to restructure pipeline --- .circleci/config.yml | 49 +++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7d3cf87ae..8d365f512 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,10 +67,7 @@ orbs: - checkout: path: /home/circleci/cassandra-k8s-operator - attach_workspace: # Create artifact from workdir - at: /tmp/workspace - - run: - name: Copying artifacts - command: if [[ -d "/tmp/workspace" && "$(ls -A /tmp/workspace)" ]]; then mv /tmp/workspace/* ./; fi + at: /home/circleci # Check environment set up correctly - run: # Wait until minikube nodes are ready name: Wait for minikube @@ -137,10 +134,7 @@ orbs: keys: # Use checksum of go.sum to version cache. - << parameters.operatorName >>-generate-{{ .Branch }}-{{ checksum "go.sum" }} - attach_workspace: # Create artifact from workdir - at: /tmp/workspace - - run: - name: Copying artifacts - command: if [[ -d "/tmp/workspace" && "$(ls -A /tmp/workspace)" ]]; then mv /tmp/workspace/* ./; fi + at: /home/circleci # Build steps - run: name: Download dependencies @@ -153,11 +147,11 @@ orbs: command: make generate # Create artifacts - persist_to_workspace: - root: /tmp/workspace/<< parameters.operatorDir >> + root: /home/circleci paths: - - build/_output - - pkg/apis/db/v1alpha1/zz_generated.deepcopy.go - - vendor + - << parameters.operatorDir >>/build/_output + - << parameters.operatorDir >>/pkg/apis/db/v1alpha1/zz_generated.deepcopy.go + - << parameters.operatorDir >>/vendor # Build job, which build operator docker image (with operator-sdk build) build: @@ -183,21 +177,18 @@ orbs: keys: # Use checksum of go.sum to version cache. - << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} - attach_workspace: # Create artifact from workdir - at: /tmp/workspace - - run: - name: Copying artifacts - command: if [[ -d "/tmp/workspace" && "$(ls -A /tmp/workspace)" ]]; then mv /tmp/workspace/* ./; fi + at: /home/circleci # Build steps - run: name: Build Operator command: make build # Create artifacts - persist_to_workspace: - root: /tmp/workspace/<< parameters.operatorDir >> + root: /home/circleci paths: - - build/_output - - pkg/apis/db/v1alpha1/zz_generated.deepcopy.go - - vendor + - << parameters.operatorDir >>/build/_output + - << parameters.operatorDir >>/pkg/apis/db/v1alpha1/zz_generated.deepcopy.go + - << parameters.operatorDir >>/vendor # Deploy steps - deploy: name: Push image to Docker Hub @@ -237,10 +228,7 @@ orbs: - checkout: path: /home/circleci/cassandra-k8s-operator - attach_workspace: # Create artifact from workdir - at: /tmp/workspace - - run: - name: Copying artifacts - command: if [[ -d "/tmp/workspace" && "$(ls -A /tmp/workspace)" ]]; then mv /tmp/workspace/* ./; fi + at: /home/circleci # Unit tests steps - run: name: Unit Tests @@ -249,10 +237,10 @@ orbs: - store_artifacts: # Store unit test coverage reports path: coverage.html - persist_to_workspace: - root: /tmp/workspace/<< parameters.operatorDir >> + root: /home/circleci paths: - - coverage.out - - test-report.out + - << parameters.operatorDir >>/coverage.out + - << parameters.operatorDir >>/test-report.out # Dgoss tests job, will execute make 'dgoss-bootstrap' docker-dgoss-test: @@ -275,10 +263,7 @@ orbs: path: /home/circleci/cassandra-k8s-operator - setup_remote_docker - attach_workspace: # Create artifact from workdir - at: /tmp/workspace - - run: - name: Copying artifacts - command: if [[ -d "/tmp/workspace" && "$(ls -A /tmp/workspace)" ]]; then mv /tmp/workspace/* ./; fi + at: /home/circleci # Dgoss tests steps - run: # Run test wich enable to validate bootstrap image. name: Dgoss tests @@ -311,7 +296,7 @@ orbs: - checkout: path: /home/circleci/cassandra-k8s-operator - attach_workspace: - at: ./ + at: /home/circleci - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. name: Restore SonarQube Scanner keys: From aa25dc934aa24ddfe66aefb5dea37de62a526f2d Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 16:36:32 +0100 Subject: [PATCH 17/41] try to restructure pipeline --- multi-casskop/Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/multi-casskop/Makefile b/multi-casskop/Makefile index 22e6b5969..22e91d195 100644 --- a/multi-casskop/Makefile +++ b/multi-casskop/Makefile @@ -144,6 +144,15 @@ helm-package: export CGO_ENABLED:=0 export PURE:="on" + +.PHONY: generate +generate: + @echo "Generate zzz-deepcopy objects" + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o build/_output/bin/health ../tools/health/main.go + echo "Generate zzz-deepcopy objects" + operator-sdk version + operator-sdk generate k8s + .PHONY: build build: @echo "Generate zzz-deepcopy objects" From bc266603b2ed3a3eb639c8722789cd304700777a Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 17:16:38 +0100 Subject: [PATCH 18/41] try to restructure pipeline --- multi-casskop/Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/multi-casskop/Makefile b/multi-casskop/Makefile index 22e91d195..8a5f73fd1 100644 --- a/multi-casskop/Makefile +++ b/multi-casskop/Makefile @@ -231,3 +231,8 @@ endif .PHONY: release release: tag image publish +unit-test-with-vendor: + echo TODO +# $(UNIT_TEST_CMD_WITH_VENDOR) && echo "success!" || { echo "failure!"; cat test-report.out; exit 1; } +# cat test-report.out +# $(UNIT_TEST_COVERAGE) From e3c13c004954d171ccf18e84fec90233f17cce91 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 18:02:10 +0100 Subject: [PATCH 19/41] remove unit test for multi casskop (not implemented yet) --- .circleci/config.yml | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d365f512..c3a01a39d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -143,7 +143,7 @@ orbs: name: Vendor dependencies command: if [ ! -d vendor ]; then go mod vendor; fi - run: - name: Build Operator + name: Generate k8s command: make generate # Create artifacts - persist_to_workspace: @@ -152,6 +152,14 @@ orbs: - << parameters.operatorDir >>/build/_output - << parameters.operatorDir >>/pkg/apis/db/v1alpha1/zz_generated.deepcopy.go - << parameters.operatorDir >>/vendor + # Cache configuration + - save_cache: + name: Save build artefacts in cache + key: << parameters.operatorName >>-generate-{{ .Branch }}-{{ checksum "go.sum" }} + paths: + - build/_output + - pkg/apis/db/v1alpha1/zz_generated.deepcopy.go + - vendor # Build job, which build operator docker image (with operator-sdk build) build: @@ -492,15 +500,15 @@ workflows: filters: tags: only: /.*/ - - operator/unit-test: - name: unit-test-multicasskop - operatorDir: cassandra-k8s-operator/multi-casskop - operatorName: MultiCassKop - requires: - - generate-multicasskop - filters: - tags: - only: /.*/ +# - operator/unit-test: +# name: unit-test-multicasskop +# operatorDir: cassandra-k8s-operator/multi-casskop +# operatorName: MultiCassKop +# requires: +# - generate-multicasskop +# filters: +# tags: +# only: /.*/ - operator/build: name: build-multicasskop operatorDir: cassandra-k8s-operator/multi-casskop From fb6ba9fc9077ec0103f0c5368168571fae60ab56 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 18:02:48 +0100 Subject: [PATCH 20/41] remove unit test for multi casskop (not implemented yet) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3a01a39d..fb0469b93 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -523,7 +523,7 @@ workflows: operatorDir: cassandra-k8s-operator/multi-casskop operatorName: MultiCassKop requires: - - unit-test-multicasskop + - build-multicasskop filters: tags: only: /.*/ From 3b42bb27bf2e7790700c3caede0b45362c5bae89 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 18:03:57 +0100 Subject: [PATCH 21/41] remove unit test for multi casskop (not implemented yet) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fb0469b93..7ff860785 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -514,7 +514,7 @@ workflows: operatorDir: cassandra-k8s-operator/multi-casskop operatorName: MultiCassKop requires: - - unit-test-multicasskop + - build-multicasskop filters: tags: only: /.*/ From 7df8d9514b497597b9d7926a337768ec67d9b20e Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 18:06:26 +0100 Subject: [PATCH 22/41] remove unit test for multi casskop (not implemented yet) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ff860785..40502d05b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -514,7 +514,7 @@ workflows: operatorDir: cassandra-k8s-operator/multi-casskop operatorName: MultiCassKop requires: - - build-multicasskop + - generate-multicasskop filters: tags: only: /.*/ From 818c2763e0329894a81e0fb7d3ccd7b9b74a3221 Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 21:42:48 +0100 Subject: [PATCH 23/41] add anchors to refactor declarations & correct sonar path --- .circleci/config.yml | 153 +++++++++++++++-------------------------- multi-casskop/Makefile | 3 - 2 files changed, 54 insertions(+), 102 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 40502d05b..e998724fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,7 +66,7 @@ orbs: # Checkout steps - checkout: path: /home/circleci/cassandra-k8s-operator - - attach_workspace: # Create artifact from workdir + - attach_workspace: # Attach artifact from workdir at: /home/circleci # Check environment set up correctly - run: # Wait until minikube nodes are ready @@ -133,7 +133,7 @@ orbs: - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. keys: # Use checksum of go.sum to version cache. - << parameters.operatorName >>-generate-{{ .Branch }}-{{ checksum "go.sum" }} - - attach_workspace: # Create artifact from workdir + - attach_workspace: # Attach artifact from workdir at: /home/circleci # Build steps - run: @@ -184,7 +184,7 @@ orbs: - restore_cache: # If exist, restore dependencies libs download cache, from previous pipeline execution. keys: # Use checksum of go.sum to version cache. - << parameters.operatorName >>-build-{{ .Branch }}-{{ checksum "go.sum" }} - - attach_workspace: # Create artifact from workdir + - attach_workspace: # Attach artifact from workdir at: /home/circleci # Build steps - run: @@ -235,7 +235,7 @@ orbs: # Checkout steps - checkout: path: /home/circleci/cassandra-k8s-operator - - attach_workspace: # Create artifact from workdir + - attach_workspace: # Attach artifact from workdir at: /home/circleci # Unit tests steps - run: @@ -270,7 +270,7 @@ orbs: - checkout: path: /home/circleci/cassandra-k8s-operator - setup_remote_docker - - attach_workspace: # Create artifact from workdir + - attach_workspace: # Attach artifact from workdir at: /home/circleci # Dgoss tests steps - run: # Run test wich enable to validate bootstrap image. @@ -313,14 +313,14 @@ orbs: - run: name: Download SonarQube Scanner command: | - chmod +x /home/circleci/cassandra-k8s-operator/install_sonar-scanner.sh + chmod +x /home/circleci/<< parameters.operatorDir >>/install_sonar-scanner.sh ./.circleci/install_sonar-scanner.sh - save_cache: # Cache sonar download name: Cache SonarQube Scanner key: sonar-scanner-3.3.0.1492-linux paths: - sonar-scanner-3.3.0.1492-linux - - /home/circleci/cassandra-k8s-operator/.sonar + - /home/circleci/<< parameters.operatorDir >>/.sonar - run: # Run sonar scan name: Sonarcloud Scan command: | @@ -350,6 +350,34 @@ orbs: - store_test_results: # Store sonar scan reports path: reports + +# Filters anchors +.everytime_filter : &everytime_filter + filters: + tags: + only: /.*/ + +.canonical_repo_filter : &canonical_repo_filter + filters: + tags: + only: /.*/ + branches: + # report sonar only from canonical repository + only: /^(?!pull\/).*$/ # https://stackoverflow.com/a/5334825/358804 + +.ignore_fork_pr_filter: &ignore_fork_pr_filter + filters: + tags: + only: /.*/ + branches: + # Forked pull requests have CIRCLE_BRANCH set to pull/XXX + ignore: /pull\/[0-9]+/ + +# Parameters anchors +.params_multicasskop: ¶ms_multicasskop + operatorDir: cassandra-k8s-operator/multi-casskop + operatorName: MultiCassKop + # Workflow definition workflows: version: 2 @@ -358,175 +386,102 @@ workflows: jobs: - operator/generate: name: generate-casskop - filters: - tags: - only: /.*/ + <<: *everytime_filter - operator/unit-test: name: unit-test-casskop requires: - generate-casskop - filters: - tags: - only: /.*/ + <<: *everytime_filter - operator/build: name: build-casskop requires: - unit-test-casskop - filters: - tags: - only: /.*/ + <<: *everytime_filter - operator/docker-dgoss-test: name: docker-dgoss-test-casskop requires: - build-casskop - filters: - tags: - only: /.*/ + <<: *everytime_filter - operator/report: name: report-casskop requires: - unit-test-casskop - filters: - tags: - only: /.*/ - branches: - # report sonar only from canonical repository - only: /^(?!pull\/).*$/ # https://stackoverflow.com/a/5334825/358804 + <<: *canonical_repo_filter - minikube/e2e-orb: name: e2e-test RollingRestart test_name: RollingRestart requires: - unit-test-casskop - filters: - tags: - only: /.*/ - branches: - # Forked pull requests have CIRCLE_BRANCH set to pull/XXX - ignore: /pull\/[0-9w]+/ + <<: *ignore_fork_pr_filter - minikube/e2e-orb: name: e2e-test ScaleUp test_name: ClusterScaleUp requires: - unit-test-casskop - filters: - tags: - only: /.*/ - branches: - # Forked pull requests have CIRCLE_BRANCH set to pull/XXX - ignore: /pull\/[0-9]+/ + <<: *ignore_fork_pr_filter - minikube/e2e-orb: name: e2e-test ScaleDown test_name: ClusterScaleDown requires: - unit-test-casskop - filters: - tags: - only: /.*/ - branches: - # Forked pull requests have CIRCLE_BRANCH set to pull/XXX - ignore: /pull\/[0-9]+/ + <<: *ignore_fork_pr_filter - minikube/e2e-orb: name: e2e-test ScaleDownSimple test_name: ClusterScaleDownSimple requires: - unit-test-casskop - filters: - tags: - only: /.*/ - branches: - # Forked pull requests have CIRCLE_BRANCH set to pull/XXX - ignore: /pull\/[0-9]+/ + <<: *ignore_fork_pr_filter - minikube/e2e-orb: name: e2e-test CreateOneClusterService test_name: CreateOneClusterService requires: - unit-test-casskop - filters: - tags: - only: /.*/ - branches: - # Forked pull requests have CIRCLE_BRANCH set to pull/XXX - ignore: /pull\/[0-9]+/ + <<: *ignore_fork_pr_filter - minikube/e2e-orb: name: e2e-test UpdateConfigMap test_name: UpdateConfigMap requires: - unit-test-casskop - filters: - tags: - only: /.*/ - branches: - # Forked pull requests have CIRCLE_BRANCH set to pull/XXX - ignore: /pull\/[0-9]+/ + <<: *ignore_fork_pr_filter - minikube/e2e-orb: name: e2e-test ExecuteCleanup test_name: ExecuteCleanup requires: - unit-test-casskop - filters: - tags: - only: /.*/ - branches: - # Forked pull requests have CIRCLE_BRANCH set to pull/XXX - ignore: /pull\/[0-9]+/ + <<: *ignore_fork_pr_filter # CassKop operator workflow. ci-multicasskop-operator: jobs: - operator/generate: name: generate-casskop - operatorDir: cassandra-k8s-operator - operatorName: CassKop - filters: - tags: - only: /.*/ + <<: *everytime_filter - operator/generate: name: generate-multicasskop - operatorDir: cassandra-k8s-operator/multi-casskop - operatorName: MultiCassKop + <<: *params_multicasskop requires: - generate-casskop - filters: - tags: - only: /.*/ -# - operator/unit-test: -# name: unit-test-multicasskop -# operatorDir: cassandra-k8s-operator/multi-casskop -# operatorName: MultiCassKop -# requires: -# - generate-multicasskop -# filters: -# tags: -# only: /.*/ + <<: *everytime_filter - operator/build: name: build-multicasskop - operatorDir: cassandra-k8s-operator/multi-casskop - operatorName: MultiCassKop + <<: *params_multicasskop requires: - generate-multicasskop - filters: - tags: - only: /.*/ + <<: *everytime_filter - operator/report: name: report-multicasskop - operatorDir: cassandra-k8s-operator/multi-casskop - operatorName: MultiCassKop + <<: *params_multicasskop requires: - build-multicasskop - filters: - tags: - only: /.*/ - branches: - # report sonar only from canonical repository - only: /^(?!pull\/).*$/ # https://stackoverflow.com/a/5334825/358804 \ No newline at end of file + <<: *canonical_repo_filter \ No newline at end of file diff --git a/multi-casskop/Makefile b/multi-casskop/Makefile index 8a5f73fd1..dde13304e 100644 --- a/multi-casskop/Makefile +++ b/multi-casskop/Makefile @@ -233,6 +233,3 @@ release: tag image publish unit-test-with-vendor: echo TODO -# $(UNIT_TEST_CMD_WITH_VENDOR) && echo "success!" || { echo "failure!"; cat test-report.out; exit 1; } -# cat test-report.out -# $(UNIT_TEST_COVERAGE) From c26ed4237a8fa50d2ab7dd07da56b81796f1412f Mon Sep 17 00:00:00 2001 From: erdrix Date: Thu, 26 Dec 2019 21:57:54 +0100 Subject: [PATCH 24/41] even more anchor --- .circleci/config.yml | 104 +++++++++++++++---------------------------- 1 file changed, 35 insertions(+), 69 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e998724fb..0b7f9884c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,6 +103,24 @@ orbs: false; } operator: + # Parameters anchor + .params_operator: ¶ms_operator + # PARAMETERS + parameters: + operatorDir: + type: string + description: relative path where the build must be run + default: cassandra-k8s-operator + operatorName: + type: string + description: operator's name + default: CassKop + working_directory: /home/circleci/<< parameters.operatorDir >> + executor: operator-build + # Job anchor + .job_operator: &job_operator + working_directory: /home/circleci/<< parameters.operatorDir >> + executor: operator-build # Setting up docker executor using casskop-build image. executors: operator-build: @@ -112,18 +130,8 @@ orbs: jobs: # Build job, which build operator docker image (with operator-sdk build) generate: - # PARAMETERS - parameters: - operatorDir: - type: string - description: relative path where the build must be run - default: cassandra-k8s-operator - operatorName: - type: string - description: operator's name - default: CassKop - working_directory: /home/circleci/<< parameters.operatorDir >> - executor: operator-build + <<: *params_operator + <<: *job_operator # STEPS steps: # Checkout steps @@ -163,19 +171,8 @@ orbs: # Build job, which build operator docker image (with operator-sdk build) build: - # PARAMETERS - parameters: - operatorDir: - type: string - description: relative path where the build must be run - default: cassandra-k8s-operator - operatorName: - type: string - description: operator's name - default: CassKop - working_directory: /home/circleci/<< parameters.operatorDir >> - executor: operator-build - # STEPS + <<: *params_operator + <<: *job_operator steps: # Checkout steps - checkout: @@ -218,18 +215,8 @@ orbs: # Unit test job, will execute makefile 'unit-test-with-vendor' step unit-test: - # PARAMETERS - parameters: - operatorDir: - type: string - description: relative path where the build must be run - default: cassandra-k8s-operator - operatorName: - type: string - description: operator's name - default: CassKop - working_directory: /home/circleci/<< parameters.operatorDir >> - executor: operator-build + <<: *params_operator + <<: *job_operator # STEPS steps: # Checkout steps @@ -252,18 +239,8 @@ orbs: # Dgoss tests job, will execute make 'dgoss-bootstrap' docker-dgoss-test: - # PARAMETERS - parameters: - operatorDir: - type: string - description: relative path where the build must be run - default: cassandra-k8s-operator - operatorName: - type: string - description: operator's name - default: CassKop - working_directory: /home/circleci/<< parameters.operatorDir >> - executor: operator-build + <<: *params_operator + <<: *job_operator # STEPS steps: # Checkout steps @@ -286,18 +263,8 @@ orbs: # Reporting step, will run sonar report from scan. report: - # PARAMETERS - parameters: - operatorDir: - type: string - description: relative path where the build must be run - default: cassandra-k8s-operator - operatorName: - type: string - description: operator's name - default: CassKop - working_directory: /home/circleci/<< parameters.operatorDir >> - executor: operator-build + <<: *params_operator + <<: *job_operator # STEPS steps: # Checkout steps @@ -313,8 +280,8 @@ orbs: - run: name: Download SonarQube Scanner command: | - chmod +x /home/circleci/<< parameters.operatorDir >>/install_sonar-scanner.sh - ./.circleci/install_sonar-scanner.sh + chmod +x /home/circleci/cassandra-k8s-operator/.circleci/install_sonar-scanner.sh + /home/circleci/cassandra-k8s-operator/.circleci/install_sonar-scanner.sh - save_cache: # Cache sonar download name: Cache SonarQube Scanner key: sonar-scanner-3.3.0.1492-linux @@ -349,9 +316,8 @@ orbs: # Artifacts definitions - store_test_results: # Store sonar scan reports path: reports - - -# Filters anchors +# Anchors +## Filters anchors .everytime_filter : &everytime_filter filters: tags: @@ -373,7 +339,7 @@ orbs: # Forked pull requests have CIRCLE_BRANCH set to pull/XXX ignore: /pull\/[0-9]+/ -# Parameters anchors +## Parameters anchors .params_multicasskop: ¶ms_multicasskop operatorDir: cassandra-k8s-operator/multi-casskop operatorName: MultiCassKop @@ -381,7 +347,7 @@ orbs: # Workflow definition workflows: version: 2 - # CassKop operator workflow. + ## CassKop operator workflow. ci-casskop-operator: jobs: - operator/generate: @@ -461,7 +427,7 @@ workflows: - unit-test-casskop <<: *ignore_fork_pr_filter - # CassKop operator workflow. + ## CassKop operator workflow. ci-multicasskop-operator: jobs: - operator/generate: From 37a46d7de20775fcce39010afb5237ff1362ea02 Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 09:15:14 +0100 Subject: [PATCH 25/41] ignore telepresence env file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8befd4603..f4e8731eb 100644 --- a/.gitignore +++ b/.gitignore @@ -113,6 +113,7 @@ debug .idea/ .vscode cassandra-operator.env +multi-casskop/multi-casskop.env telepresence.log build/_output pkg/apis/db/v1alpha1/zz_generated.deepcopy.go From 27b4bec7b5bf4bafdacaba16bb8c526fa8660e6d Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 11:30:51 +0100 Subject: [PATCH 26/41] add script to publish helm on gcs --- tools/publish_helm_gcs.sh | 108 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tools/publish_helm_gcs.sh diff --git a/tools/publish_helm_gcs.sh b/tools/publish_helm_gcs.sh new file mode 100644 index 000000000..9d152b56f --- /dev/null +++ b/tools/publish_helm_gcs.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +HELM_TARGET_DIR=$(pwd)/incubator +readonly HELM_VERSION=$(cat helm/cassandra-operator/Chart.yaml| grep version | awk -F"version: " '{print $2}') +readonly HELM_URL=https://storage.googleapis.com/kubernetes-helm +readonly HELM_TARBALL=helm-v2.9.1-linux-amd64.tar.gz +#readonly STABLE_REPO_URL=https://orange-kubernetes-charts.storage.googleapis.com/ +readonly INCUBATOR_REPO_URL=https://orange-kubernetes-charts-incubator.storage.googleapis.com/ +#readonly GCS_BUCKET_STABLE=gs://orange-kubernetes-charts +readonly GCS_BUCKET_INCUBATOR=gs://orange-kubernetes-charts-incubator + +main() { + setup_helm_client + authenticate + +# if ! sync_repo stable "$GCS_BUCKET_STABLE" "$STABLE_REPO_URL"; then +# log_error "Not all stable charts could be packaged and synced!" +# fi + if ! sync_repo ${HELM_TARGET_DIR} "$GCS_BUCKET_INCUBATOR" "$INCUBATOR_REPO_URL"; then + log_error "Not all incubator charts could be packaged and synced!" + fi +} + +setup_helm_client() { + echo "Setting up Helm client..." + + curl --user-agent curl-ci-sync -sSL -o "$HELM_TARBALL" "$HELM_URL/$HELM_TARBALL" + tar xzfv "$HELM_TARBALL" + +# PATH="$(pwd)/linux-amd64/:$PATH" + PATH="$(pwd)/darwin-amd64/:$PATH" + + helm init --client-only + helm repo add incubator "$INCUBATOR_REPO_URL" +} + +authenticate() { + echo "Authenticating with Google Cloud..." + gcloud auth activate-service-account --key-file <(base64 --decode <<< "$SYNC_CREDS") +} + +sync_repo() { + local target_dir="${1?Specify repo dir}" + local bucket="${2?Specify repo bucket}" + local repo_url="${3?Specify repo url}" + local index_dir="${target_dir}-index" + + echo "Syncing repo '$target_dir'..." + + mkdir -p "$target_dir" + if ! gsutil cp "$bucket/index.yaml" "$index_dir/index.yaml"; then + log_error "Exiting because unable to copy index locally. Not safe to proceed." + exit 1 + fi + + local exit_code=0 + + echo "Packaging operators ..." + if ! HELM_TARGET_DIR=${target_dir} make helm-package; then + log_error "Problem packaging casskop operator" + exit_code=1 + fi + if ! HELM_TARGET_DIR=${target_dir} make -C multi-casskop helm-package; then + log_error "Problem packaging multi-casskop operator" + exit_code=1 + fi + + if helm repo index --url "$repo_url" --merge "$index_dir/index.yaml" "$target_dir"; then + # Move updated index.yaml to index folder so we don't push the old one again + mv -f "$target_dir/index.yaml" "$index_dir/index.yaml" + + gsutil cp "$target_dir/*" "$bucket" + + # Make sure index.yaml is synced last + gsutil cp "$index_dir/index.yaml" "$bucket" + else + log_error "Exiting because unable to update index. Not safe to push update." + exit 1 + fi + + ls -l "$target_dir" + + return "$exit_code" +} + +log_error() { + printf '\e[31mERROR: %s\n\e[39m' "$1" >&2 +} + +main \ No newline at end of file From 0e30211f240ee3ada4815a64874c7c536cf00106 Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 11:33:47 +0100 Subject: [PATCH 27/41] adapt helm-package step to ci --- Makefile | 8 ++++---- multi-casskop/Makefile | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 7eb40f3d5..e35aaa798 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,8 @@ else VERSION := $(BASEVERSION)-${BRANCH} endif -HELM_VERSION := $(shell cat helm/cassandra-operator/Chart.yaml| grep version | awk -F"version: " '{print $$2}') +HELM_VERSION := $(shell cat helm/cassandra-operator/Chart.yaml| grep version | awk -F"version: " '{print $$2}') +HELM_TARGET_DIR ?= docs/helm #si branche master, on pousse le tag latest ifeq ($(CIRCLE_BRANCH),master) @@ -157,9 +158,8 @@ clean: helm-package: @echo Packaging $(HELM_VERSION) helm package helm/cassandra-operator - mv cassandra-operator-$(HELM_VERSION).tgz docs/helm - helm repo index docs/helm/ - make -C multi-casskop helm-package + mv cassandra-operator-$(HELM_VERSION).tgz $(HELM_TARGET_DIR) + helm repo index $(HELM_TARGET_DIR)/ # .PHONY: generate diff --git a/multi-casskop/Makefile b/multi-casskop/Makefile index dde13304e..a21a1871b 100644 --- a/multi-casskop/Makefile +++ b/multi-casskop/Makefile @@ -66,8 +66,8 @@ else VERSION := $(BASEVERSION)-${BRANCH} endif -HELM_VERSION := $(shell cat helm/multi-casskop/Chart.yaml| grep version | awk -F"version: " '{print $$2}') - +HELM_VERSION := $(shell cat helm/cassandra-operator/Chart.yaml| grep version | awk -F"version: " '{print $$2}') +HELM_TARGET_DIR ?= ../docs/helm #si branche master, on pousse le tag latest ifeq ($(CIRCLE_BRANCH),master) PUSHLATEST := true @@ -136,9 +136,9 @@ clean: helm-package: @echo Packaging $(HELM_VERSION) - helm package helm/multi-casskop - mv multi-casskop-$(HELM_VERSION).tgz ../docs/helm - helm repo index ../docs/helm/ + helm package helm/cassandra-operator + mv cassandra-operator-$(HELM_VERSION).tgz $(HELM_TARGET_DIR) + helm repo index $(HELM_TARGET_DIR)/ # Build cassandra-k8s-operator executable file in local go env From 17a992205f89486a0ad508420e61d4e2f7e123f6 Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 11:48:54 +0100 Subject: [PATCH 28/41] correct makefile --- multi-casskop/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multi-casskop/Makefile b/multi-casskop/Makefile index a21a1871b..b09b8fb99 100644 --- a/multi-casskop/Makefile +++ b/multi-casskop/Makefile @@ -66,7 +66,7 @@ else VERSION := $(BASEVERSION)-${BRANCH} endif -HELM_VERSION := $(shell cat helm/cassandra-operator/Chart.yaml| grep version | awk -F"version: " '{print $$2}') +HELM_VERSION := $(shell cat helm/multi-casskop/Chart.yaml| grep version | awk -F"version: " '{print $$2}') HELM_TARGET_DIR ?= ../docs/helm #si branche master, on pousse le tag latest ifeq ($(CIRCLE_BRANCH),master) @@ -136,8 +136,8 @@ clean: helm-package: @echo Packaging $(HELM_VERSION) - helm package helm/cassandra-operator - mv cassandra-operator-$(HELM_VERSION).tgz $(HELM_TARGET_DIR) + helm package helm/multi-casskop + mv multi-casskop-$(HELM_VERSION).tgz $(HELM_TARGET_DIR) helm repo index $(HELM_TARGET_DIR)/ # Build cassandra-k8s-operator executable file in local go env From ae3d7cce56b28388e8d1f60f777d0863b62cafc2 Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 11:51:02 +0100 Subject: [PATCH 29/41] rewrite for linux --- tools/publish_helm_gcs.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) mode change 100644 => 100755 tools/publish_helm_gcs.sh diff --git a/tools/publish_helm_gcs.sh b/tools/publish_helm_gcs.sh old mode 100644 new mode 100755 index 9d152b56f..943cd3a01 --- a/tools/publish_helm_gcs.sh +++ b/tools/publish_helm_gcs.sh @@ -18,7 +18,7 @@ set -o errexit set -o nounset set -o pipefail -HELM_TARGET_DIR=$(pwd)/incubator +HELM_TARGET_DIR=$(pwd)/tmp/incubator readonly HELM_VERSION=$(cat helm/cassandra-operator/Chart.yaml| grep version | awk -F"version: " '{print $2}') readonly HELM_URL=https://storage.googleapis.com/kubernetes-helm readonly HELM_TARBALL=helm-v2.9.1-linux-amd64.tar.gz @@ -28,6 +28,7 @@ readonly INCUBATOR_REPO_URL=https://orange-kubernetes-charts-incubator.storage.g readonly GCS_BUCKET_INCUBATOR=gs://orange-kubernetes-charts-incubator main() { + mkdir -p tmp setup_helm_client authenticate @@ -35,7 +36,7 @@ main() { # log_error "Not all stable charts could be packaged and synced!" # fi if ! sync_repo ${HELM_TARGET_DIR} "$GCS_BUCKET_INCUBATOR" "$INCUBATOR_REPO_URL"; then - log_error "Not all incubator charts could be packaged and synced!" + log_error "Not all incubator charts could be packaged and pushed!" fi } @@ -43,18 +44,17 @@ setup_helm_client() { echo "Setting up Helm client..." curl --user-agent curl-ci-sync -sSL -o "$HELM_TARBALL" "$HELM_URL/$HELM_TARBALL" - tar xzfv "$HELM_TARBALL" + tar xzfv "$HELM_TARBALL" -C tmp -# PATH="$(pwd)/linux-amd64/:$PATH" - PATH="$(pwd)/darwin-amd64/:$PATH" + PATH="$(pwd)/tmp/linux-amd64/:$PATH" helm init --client-only - helm repo add incubator "$INCUBATOR_REPO_URL" + helm repo add incubator-orange "$INCUBATOR_REPO_URL" } authenticate() { echo "Authenticating with Google Cloud..." - gcloud auth activate-service-account --key-file <(base64 --decode <<< "$SYNC_CREDS") + gcloud auth activate-service-account --key-file <(base64 --decode <<< "$GCP_SA_CREDS") } sync_repo() { From fd8b28e7826f004cf839ff6b3b90c84d3ef65b1b Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 11:54:22 +0100 Subject: [PATCH 30/41] generic script --- tools/publish_helm_gcs.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/publish_helm_gcs.sh b/tools/publish_helm_gcs.sh index 943cd3a01..080069954 100755 --- a/tools/publish_helm_gcs.sh +++ b/tools/publish_helm_gcs.sh @@ -75,11 +75,7 @@ sync_repo() { echo "Packaging operators ..." if ! HELM_TARGET_DIR=${target_dir} make helm-package; then - log_error "Problem packaging casskop operator" - exit_code=1 - fi - if ! HELM_TARGET_DIR=${target_dir} make -C multi-casskop helm-package; then - log_error "Problem packaging multi-casskop operator" + log_error "Problem packaging operator" exit_code=1 fi From 40abeae62b48ab944384267c1cff801cdc878428 Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 12:25:29 +0100 Subject: [PATCH 31/41] add helm publish step for test --- .circleci/config.yml | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b7f9884c..dece95560 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -316,6 +316,18 @@ orbs: # Artifacts definitions - store_test_results: # Store sonar scan reports path: reports + + publish-helm: + working_directory: /home/circleci/<< parameters.operatorDir >> + docker: + - image: google/cloud-sdk + steps: + # Checkout steps + - checkout: + path: /home/circleci/cassandra-k8s-operator + - run: + name: publish-helm + command: /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh # Anchors ## Filters anchors .everytime_filter : &everytime_filter @@ -339,6 +351,14 @@ orbs: # Forked pull requests have CIRCLE_BRANCH set to pull/XXX ignore: /pull\/[0-9]+/ +.only_on_release_tag: &only_on_release_tag + # ignore any commit on any branch by default + branches: + ignore: /.*/ + # only act on version tags + tags: + only: /v[0-9]+(\.[0-9]+)*-release/ + ## Parameters anchors .params_multicasskop: ¶ms_multicasskop operatorDir: cassandra-k8s-operator/multi-casskop @@ -426,6 +446,12 @@ workflows: requires: - unit-test-casskop <<: *ignore_fork_pr_filter + - operator/publish-helm: + name: publish-helm-orange-incubator + requires: + - report-casskop + # <<: *only_on_release_tag + <<: *everytime_filter ## CassKop operator workflow. ci-multicasskop-operator: @@ -450,4 +476,11 @@ workflows: <<: *params_multicasskop requires: - build-multicasskop - <<: *canonical_repo_filter \ No newline at end of file + <<: *canonical_repo_filter + - operator/publish-helm: + name: publish-helm-orange-incubator + <<: *params_multicasskop + requires: + - report-multicasskop +# <<: *only_on_release_tag + <<: *everytime_filter \ No newline at end of file From cc77d54894ad4e82579169e3d19032bb0164f201 Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 12:32:29 +0100 Subject: [PATCH 32/41] add helm publish step for test --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index dece95560..e5dba66a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -318,6 +318,7 @@ orbs: path: reports publish-helm: + <<: *params_operator working_directory: /home/circleci/<< parameters.operatorDir >> docker: - image: google/cloud-sdk From 6a26d8592963271bbf4ef419897f8c3238357da7 Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 12:41:44 +0100 Subject: [PATCH 33/41] add helm publish step for test --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e5dba66a7..35d24aeb6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -449,9 +449,9 @@ workflows: <<: *ignore_fork_pr_filter - operator/publish-helm: name: publish-helm-orange-incubator - requires: - - report-casskop - # <<: *only_on_release_tag +# requires: +# - report-casskop +# <<: *only_on_release_tag <<: *everytime_filter ## CassKop operator workflow. @@ -481,7 +481,7 @@ workflows: - operator/publish-helm: name: publish-helm-orange-incubator <<: *params_multicasskop - requires: - - report-multicasskop +# requires: +# - report-multicasskop # <<: *only_on_release_tag <<: *everytime_filter \ No newline at end of file From b010404cc5dcf5e3856860383da6ce94131b04da Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 12:50:23 +0100 Subject: [PATCH 34/41] add explicit sa key declaration --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 35d24aeb6..652a8b51a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -328,7 +328,7 @@ orbs: path: /home/circleci/cassandra-k8s-operator - run: name: publish-helm - command: /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh + command: GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh # Anchors ## Filters anchors .everytime_filter : &everytime_filter From c3e1d3f3554bb40a8ae22f31bf568d665ea3c09c Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 12:56:12 +0100 Subject: [PATCH 35/41] debyg --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 652a8b51a..e1d274396 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -328,7 +328,7 @@ orbs: path: /home/circleci/cassandra-k8s-operator - run: name: publish-helm - command: GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh + command: echo ${GCP_POC_RTC_SA_KEY} && GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh # Anchors ## Filters anchors .everytime_filter : &everytime_filter From dc40f808fe433c70a01cc9e2bdaec0b4b6eac328 Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 12:59:15 +0100 Subject: [PATCH 36/41] debyg --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e1d274396..fac2598d9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -328,7 +328,7 @@ orbs: path: /home/circleci/cassandra-k8s-operator - run: name: publish-helm - command: echo ${GCP_POC_RTC_SA_KEY} && GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh + command: echo "test > ${GCP_POC_RTC_SA_KEY}" && GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh # Anchors ## Filters anchors .everytime_filter : &everytime_filter From c631b1ea107c27e917028c26856672b473665833 Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 13:24:12 +0100 Subject: [PATCH 37/41] clean up --- .circleci/config.yml | 8 +++++++- tools/publish_helm_gcs.sh | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fac2598d9..6256a8f53 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -328,7 +328,13 @@ orbs: path: /home/circleci/cassandra-k8s-operator - run: name: publish-helm - command: echo "test > ${GCP_POC_RTC_SA_KEY}" && GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh + command: GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh +# | +# if [[ $(echo "$CIRCLE_BRANCH" | grep -c "pull") -gt 0 ]]; then +# echo "This is a PR, we don't push to Hub." +# else +# echo "test > ${GCP_POC_RTC_SA_KEY}" && GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh +# fi # Anchors ## Filters anchors .everytime_filter : &everytime_filter diff --git a/tools/publish_helm_gcs.sh b/tools/publish_helm_gcs.sh index 080069954..10cd56e24 100755 --- a/tools/publish_helm_gcs.sh +++ b/tools/publish_helm_gcs.sh @@ -19,7 +19,6 @@ set -o nounset set -o pipefail HELM_TARGET_DIR=$(pwd)/tmp/incubator -readonly HELM_VERSION=$(cat helm/cassandra-operator/Chart.yaml| grep version | awk -F"version: " '{print $2}') readonly HELM_URL=https://storage.googleapis.com/kubernetes-helm readonly HELM_TARBALL=helm-v2.9.1-linux-amd64.tar.gz #readonly STABLE_REPO_URL=https://orange-kubernetes-charts.storage.googleapis.com/ From ebe621a554975ce763cc663e1a4f506bc389e25f Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 13:52:01 +0100 Subject: [PATCH 38/41] clean and fix setup --- .circleci/config.yml | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6256a8f53..62449008e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,5 @@ version: 2.1 - experimental: notify: branches: @@ -328,13 +327,12 @@ orbs: path: /home/circleci/cassandra-k8s-operator - run: name: publish-helm - command: GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh -# | -# if [[ $(echo "$CIRCLE_BRANCH" | grep -c "pull") -gt 0 ]]; then -# echo "This is a PR, we don't push to Hub." -# else -# echo "test > ${GCP_POC_RTC_SA_KEY}" && GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh -# fi + command: | + if [[ $(echo "$CIRCLE_BRANCH" | grep -c "pull") -gt 0 ]]; then + echo "This is a PR, we don't publish to helm repo." + else + echo GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh + fi # Anchors ## Filters anchors .everytime_filter : &everytime_filter @@ -455,10 +453,9 @@ workflows: <<: *ignore_fork_pr_filter - operator/publish-helm: name: publish-helm-orange-incubator -# requires: -# - report-casskop -# <<: *only_on_release_tag - <<: *everytime_filter + requires: + - report-casskop + <<: *only_on_release_tag ## CassKop operator workflow. ci-multicasskop-operator: @@ -487,7 +484,6 @@ workflows: - operator/publish-helm: name: publish-helm-orange-incubator <<: *params_multicasskop -# requires: -# - report-multicasskop -# <<: *only_on_release_tag - <<: *everytime_filter \ No newline at end of file + requires: + - report-multicasskop + <<: *only_on_release_tag \ No newline at end of file From ecaa5fb94bf0cceb90282b8d75788f026afb6da0 Mon Sep 17 00:00:00 2001 From: erdrix Date: Mon, 6 Jan 2020 22:23:21 +0100 Subject: [PATCH 39/41] correct some configurations --- .circleci/config.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 62449008e..48d8e7515 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -330,8 +330,8 @@ orbs: command: | if [[ $(echo "$CIRCLE_BRANCH" | grep -c "pull") -gt 0 ]]; then echo "This is a PR, we don't publish to helm repo." - else - echo GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh + else + GCP_SA_CREDS=${GCP_POC_RTC_SA_KEY} /home/circleci/cassandra-k8s-operator/tools/publish_helm_gcs.sh fi # Anchors ## Filters anchors @@ -357,12 +357,13 @@ orbs: ignore: /pull\/[0-9]+/ .only_on_release_tag: &only_on_release_tag - # ignore any commit on any branch by default - branches: - ignore: /.*/ - # only act on version tags - tags: - only: /v[0-9]+(\.[0-9]+)*-release/ + filters: + # ignore any commit on any branch by default + branches: + ignore: /.*/ + # only act on version tags + tags: + only: /v[0-9]+(\.[0-9]+)*-release/ ## Parameters anchors .params_multicasskop: ¶ms_multicasskop @@ -452,7 +453,7 @@ workflows: - unit-test-casskop <<: *ignore_fork_pr_filter - operator/publish-helm: - name: publish-helm-orange-incubator + name: publish-helm-orange-incubator-casskop requires: - report-casskop <<: *only_on_release_tag @@ -482,7 +483,7 @@ workflows: - build-multicasskop <<: *canonical_repo_filter - operator/publish-helm: - name: publish-helm-orange-incubator + name: publish-helm-orange-incubator-multicasskop <<: *params_multicasskop requires: - report-multicasskop From 8f8d8b933ef92dfb0c51a2e898131b18078ad4e1 Mon Sep 17 00:00:00 2001 From: erdrix Date: Tue, 7 Jan 2020 08:52:40 +0100 Subject: [PATCH 40/41] adapt documentation with helm repo migration --- Readme.md | 15 ++++----------- docs/slides/Slides-CassKop-demo.md | 6 +++--- multi-casskop/Readme.md | 6 +++--- multi-casskop/docs/tutorials/gke.md | 7 ++----- multi-casskop/helm/multi-casskop/readme.md | 8 ++++---- multi-casskop/samples/gke/terraform/kubernetes.tf | 4 ++-- 6 files changed, 18 insertions(+), 28 deletions(-) diff --git a/Readme.md b/Readme.md index b9dacc8ce..ace36f2b7 100644 --- a/Readme.md +++ b/Readme.md @@ -170,7 +170,7 @@ created Add the Helm incubator repo if you do not already have it: ``` -helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/ +helm repo add orange-incubator https://orange-charts-incubator.storage.googleapis.com ``` Get the latest information about charts from the chart repositories. @@ -179,24 +179,17 @@ Get the latest information about charts from the chart repositories. helm repo update ``` +Helm is available in the orange helm/charts/incubator: -Helm is available in the official helm/charts/incubator: - -``` -helm install --name casskop incubator/cassandra-operator ``` - -you can also add the CassKop repository from Github - -```console -helm repo add casskop https://Orange-OpenSource.github.io/cassandra-k8s-operator/helm +helm install --name casskop orange-incubator/cassandra-operator ``` Deploy CassKop: ```console -$ helm install --name casskop casskop/cassandra-operator +$ helm install --name casskop orange-incubator/cassandra-operator NAME: casskop LAST DEPLOYED: Thu May 23 15:34:27 2019 NAMESPACE: cassandra-demo diff --git a/docs/slides/Slides-CassKop-demo.md b/docs/slides/Slides-CassKop-demo.md index d1957ed02..dbecf1723 100644 --- a/docs/slides/Slides-CassKop-demo.md +++ b/docs/slides/Slides-CassKop-demo.md @@ -378,8 +378,8 @@ $ helm install --name casskop incubator/cassandra-operator Or You can also add the CassKop repository from Github ```yaml -helm repo add casskop https://Orange-OpenSource.github.io/cassandra-k8s-operator/helm -helm install --name casskop casskop/cassandra-operator +helm repo add orange-incubator https://orange-charts-incubator.storage.googleapis.com +helm install --name casskop orange-incubator/cassandra-operator ``` > This deploy the Operator and it's CRD via a helm hook. @@ -393,7 +393,7 @@ Error: customresourcedefinitions.apiextensions.k8s.io "cassandraclusters.db.oran In this case, add the `--no-hooks` helm cli flag to tell helm not to deploy the CRD hook: ```yaml -$ helm install --name casskop casskop/cassandra-operator --no-hooks +$ helm install --name casskop orange-incubator/cassandra-operator --no-hooks ``` Check operator's logs: diff --git a/multi-casskop/Readme.md b/multi-casskop/Readme.md index ff68dfbdc..968487a20 100644 --- a/multi-casskop/Readme.md +++ b/multi-casskop/Readme.md @@ -87,14 +87,14 @@ CassKop must be deployed on each targeted Kubernetes clusters. Add the Helm repository for CassKop ```console -$ helm repo add casskop https://Orange-OpenSource.github.io/cassandra-k8s-operator/helm +$ helm repo add orange-incubator https://orange-charts-incubator.storage.googleapis.com $ helm repo update ``` Connect to each kubernetes you want to deploy your Cassandra clusters to and install CassKop: ```console -$ helm install --name casskop casskop/cassandra-operator +$ helm install --name casskop orange-incubator/cassandra-operator ``` ### Install External-DNS @@ -113,7 +113,7 @@ Proceed with Multi-CassKop installation only when [Pre-requisites](#pre-requisit Deployment with Helm. Multi-CassKop and CassKop shared the same github/helm repo and semantic version. ``` -helm install --name multi-casskop casskop/multi-casskop --set k8s.local=k8s-cluster1 --set k8s.remote={k8s-cluster2} +helm install --name multi-casskop orange-incubator/multi-casskop --set k8s.local=k8s-cluster1 --set k8s.remote={k8s-cluster2} ``` > if you get an error complaining that the CRD already exists, then replay it with `--no-hooks` diff --git a/multi-casskop/docs/tutorials/gke.md b/multi-casskop/docs/tutorials/gke.md index e938a391d..217fb71d3 100644 --- a/multi-casskop/docs/tutorials/gke.md +++ b/multi-casskop/docs/tutorials/gke.md @@ -202,19 +202,16 @@ Switched to context "gke__europe-west1-c_cassandra-europe-west1-c- $ kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/cassandra-k8s-operator/master/multi-casskop/deploy/crds/multicluster_v1alpha1_cassandramulticluster_crd.yaml ``` -@TODO : To correct once the helm release and docker image will be properly deployed Deployment with Helm : ```sh $ kubectx # Switch context on master cluster Switched to context "gke__europe-west1-b_cassandra-europe-west1-b-master". $ helm init --client-only -$ helm repo add casskop https://Orange-OpenSource.github.io/cassandra-k8s-operator/helm +$ helm repo add orange-incubator https://orange-charts-incubator.storage.googleapis.com $ helm repo update -# @TODO $ cd ${CASSKOP_WORKSPACE} -$ helm install --name multi-casskop ./multi-casskop/helm/multi-casskop --set k8s.local=gke-master-west1-b --set k8s.remote={gke-slave-west1-c} --set image.tag=0.5.0-multi-cluster #--no-hooks if crd already install -#$ helm install --name multi-casskop casskop/multi-casskop --set k8s.local=gke-master-west1-b --set k8s.remote={gke-slave-west1-c} --set image.tag=0.5.0-multi-cluster #--no-hooks if crd already install +$ helm install --name multi-casskop orange-incubator/multi-casskop --set k8s.local=gke-master-west1-b --set k8s.remote={gke-slave-west1-c} --set image.tag=0.5.0-multi-cluster #--no-hooks if crd already install ``` ### Create the MultiCasskop CRD diff --git a/multi-casskop/helm/multi-casskop/readme.md b/multi-casskop/helm/multi-casskop/readme.md index 7fdee82d4..e816e9338 100644 --- a/multi-casskop/helm/multi-casskop/readme.md +++ b/multi-casskop/helm/multi-casskop/readme.md @@ -39,8 +39,8 @@ Specify each parameter using the `--set key=value[,key=value]` argument to `helm Install a multi-casskop release : ```console -$ helm repo add casskop https://Orange-OpenSource.github.io/cassandra-k8s-operator/helm -$ helm install --name multi-casskop casskop/multi-casskop +$ helm repo add orange-incubator https://orange-charts-incubator.storage.googleapis.com +$ helm install --name multi-casskop orange-incubator/multi-casskop ``` ### Listing deployed charts @@ -111,12 +111,12 @@ In that case you can get an error like : ``` -$ helm install --name multi-casskop casskop/multi-casskop +$ helm install --name multi-casskop orange-incubator/multi-casskop Error: customresourcedefinitions.apiextensions.k8s.io "multicasskop.db.orange.com" already exists ``` In this case there si a parameter to say to not uses the hook to install the CRD : ``` -$ helm install --name multi-casskop casskop/multi-casskop --no-hooks +$ helm install --name multi-casskop orange-incubator/multi-casskop --no-hooks ``` diff --git a/multi-casskop/samples/gke/terraform/kubernetes.tf b/multi-casskop/samples/gke/terraform/kubernetes.tf index 24c329f24..44051796c 100644 --- a/multi-casskop/samples/gke/terraform/kubernetes.tf +++ b/multi-casskop/samples/gke/terraform/kubernetes.tf @@ -89,8 +89,8 @@ resource "kubernetes_storage_class" "cassandra-standard" { // helm repository data "helm_repository" "casskop" { - name = "casskop" - url = "https://Orange-OpenSource.github.io/cassandra-k8s-operator/helm" + name = "orange-incubator" + url = "https://orange-charts-incubator.storage.googleapis.com" depends_on = [kubernetes_cluster_role_binding.tiller-admin-binding] } From fbfd796cf1e8baf10ef8ed6dac909de775109c0f Mon Sep 17 00:00:00 2001 From: erdrix Date: Fri, 10 Jan 2020 09:34:09 +0100 Subject: [PATCH 41/41] add openapi generation --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 091a18420..dce8b606f 100644 --- a/Makefile +++ b/Makefile @@ -167,6 +167,7 @@ generate: echo "Generate zzz-deepcopy objects" operator-sdk version operator-sdk generate k8s + operator-sdk generate openapi # Build cassandra-k8s-operator executable file in local go env .PHONY: build