From a57232ca003329a412652ade59bb8f089204ff5a Mon Sep 17 00:00:00 2001 From: murphp15 Date: Thu, 25 May 2023 08:31:16 +0100 Subject: [PATCH 1/6] control-service: fix failing tests Signed-off-by: murphp15 --- projects/control-service/cicd/.gitlab-ci.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/projects/control-service/cicd/.gitlab-ci.yml b/projects/control-service/cicd/.gitlab-ci.yml index c7d16c175a..15cd8ef117 100644 --- a/projects/control-service/cicd/.gitlab-ci.yml +++ b/projects/control-service/cicd/.gitlab-ci.yml @@ -255,18 +255,14 @@ control_service_deploy_testing_data_pipelines: script: - apk --no-cache add bash openssl curl git gettext zip py-pip - pip install --upgrade pip && pip install awscli - - export DESIRED_VERSION=v3.11.3 # helm version 3.11.3 + - export DESIRED_VERSION=v3.12.0 # helm version - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash - bash -ex ./projects/control-service/cicd/deploy-testing-pipelines-service.sh retry: !reference [.control_service_retry, retry_options] - rules: - - if: '$CI_COMMIT_BRANCH == "main"' - changes: [ projects/control-service/projects/helm_charts/pipelines-control-service/version.txt ] - when: never - - if: '$CI_COMMIT_BRANCH == "main"' - changes: *control_service_code_change_locations - - if: '$CI_COMMIT_BRANCH == "main"' - changes: *control_service_helm_change_locations + only: + refs: + - external_pull_requests + changes: *control_service_code_change_locations # vdk-heartbeat source: https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-heartbeat control_service_post_deployment_test: From 5556af68cc4370c66b2836b16ea7e512e140a2c5 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Thu, 25 May 2023 08:33:55 +0100 Subject: [PATCH 2/6] control-service: fix failing tests Signed-off-by: murphp15 --- projects/control-service/cicd/.gitlab-ci.yml | 71 -------------------- 1 file changed, 71 deletions(-) diff --git a/projects/control-service/cicd/.gitlab-ci.yml b/projects/control-service/cicd/.gitlab-ci.yml index 15cd8ef117..7685a74cb2 100644 --- a/projects/control-service/cicd/.gitlab-ci.yml +++ b/projects/control-service/cicd/.gitlab-ci.yml @@ -39,77 +39,6 @@ - apk --no-cache add openjdk17-jdk git zip curl zip py-pip - pip install --upgrade pip && pip install awscli -control_service_build_image: - extends: - - .control_service_base_build - - .images:dind - stage: build - script: - - apk --no-cache add git openjdk17-jdk - - export TAG=$(git rev-parse --short HEAD) - - cd projects/control-service/projects - - ./gradlew -p ./model build publishToMavenLocal --info --stacktrace - - ./gradlew build jacocoTestReport -x integrationTest --info --stacktrace -# - ./gradlew :pipelines_control_service:docker --info --stacktrace -Pversion=$TAG - retry: !reference [.control_service_retry, retry_options] - coverage: "/ - Line Coverage: ([0-9.]+)%/" - artifacts: - when: always - paths: - - ./projects/control-service/projects/*/build/reports/** - expire_in: 1 week - reports: - junit: ./projects/control-service/projects/**/build/test-results/test/TEST-*.xml - only: - refs: - - external_pull_requests - changes: *control_service_code_change_locations - -control_service_integration_test: - extends: .control_service_base_build - stage: build - variables: - DEPLOYMENT_K8S_KUBECONFIG: "~/.kube/config" - DEPLOYMENT_K8S_NAMESPACE: $CICD_KUBERNETES_NAMESPACE - CONTROL_K8S_KUBECONFIG: "~/.kube/config" - CONTROL_K8S_NAMESPACE: $CICD_KUBERNETES_NAMESPACE - DOCKER_REGISTRY_URL: $CICD_CONTAINER_REGISTRY_URI - DOCKER_REGISTRY_USERNAME: $CICD_CONTAINER_REGISTRY_USER_NAME - DOCKER_REGISTRY_PASSWORD: $CICD_CONTAINER_REGISTRY_USER_PASSWORD - GIT_URL: $CICD_GIT_URI - GIT_USERNAME: $CICD_GIT_USER - GIT_PASSWORD: $CICD_GIT_PASSWORD - GIT_USERNAME_READ_WRITE: $CICD_GIT_USER - GIT_PASSWORD_READ_WRITE: $CICD_GIT_PASSWORD - DATAJOBS_AWS_SERVICE_ACCOUNT_ACCESS_KEY_ID: $DATAJOBS_AWS_SERVICE_ACCOUNT_ACCESS_KEY_ID - DATAJOBS_AWS_SERVICE_ACCOUNT_SECRET_ACCESS_KEY: $DATAJOBS_AWS_SERVICE_ACCOUNT_SECRET_ACCESS_KEY - script: - - cd projects/control-service/projects - - ./gradlew -p ./model build publishToMavenLocal - - mkdir -p ~/.kube - - cp $KUBECONFIG ~/.kube/config - - ./gradlew -v - - ./gradlew projects --info - - ./gradlew -Djdk.tls.client.protocols=TLSv1.2 :pipelines_control_service:integrationTest --info --stacktrace - retry: !reference [.control_service_retry, retry_options] - artifacts: - when: always - paths: - - ./projects/control-service/projects/*/build/reports/** - expire_in: 1 week - reports: - junit: ./projects/control-service/projects/**/build/test-results/integrationTest/TEST-*.xml - only: - refs: - - external_pull_requests - changes: *control_service_code_change_locations - except: - changes: - - projects/control-service/projects/helm_charts/pipelines-control-service/version.txt - - projects/control-service/projects/model/CONTRIBUTING.MD - - projects/control-service/projects/**/README.md - - projects/control-service/projects/model/apidefs/datajob-api/config-python.json - control_service_test_helm_chart: stage: build script: From 63e2ae7299eed569ec4dc07a6570bf4cf129d75e Mon Sep 17 00:00:00 2001 From: murphp15 Date: Thu, 25 May 2023 09:02:19 +0100 Subject: [PATCH 3/6] control-service: fix failing tests Signed-off-by: murphp15 --- .../cicd/deploy-testing-pipelines-service.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/projects/control-service/cicd/deploy-testing-pipelines-service.sh b/projects/control-service/cicd/deploy-testing-pipelines-service.sh index 41b89def52..c78491dab5 100755 --- a/projects/control-service/cicd/deploy-testing-pipelines-service.sh +++ b/projects/control-service/cicd/deploy-testing-pipelines-service.sh @@ -69,6 +69,21 @@ envsubst < $VDK_OPTIONS > $VDK_OPTIONS_SUBSTITUTED cd $TPCS_CHART || exit helm dependency update --kubeconfig=$KUBECONFIG + +helm_latest_deployment=`helm history cicd-control-service | tail -1` +echo $helm_latest_deployment +if [[ $helm_latest_deployment == *"pending-upgrade"* ]]; then + echo "Pipeline failed because of an existing deployment in the pending-state. + If the problem persists use 'helm history cicd-control-service' to see the last successful deployment. + then use 'helm rollback cicd-control-service ' to rollback to that deployment. then re run this pipeline" + exit 125 +fi + + + +if[] + +# # TODO :change container images with official ones when they are being deployed (I've currently uploaded them once in ghcr.io/tozka) # # image.tag is fixed during release. It is set here to deploy using latest change in source code. From b4c2f9b7a9b8ad4be19de5b298caf84da68469f7 Mon Sep 17 00:00:00 2001 From: murphp15 Date: Thu, 25 May 2023 09:05:42 +0100 Subject: [PATCH 4/6] control-service: fix failing tests Signed-off-by: murphp15 --- .../control-service/cicd/deploy-testing-pipelines-service.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/projects/control-service/cicd/deploy-testing-pipelines-service.sh b/projects/control-service/cicd/deploy-testing-pipelines-service.sh index c78491dab5..799e6b7fcc 100755 --- a/projects/control-service/cicd/deploy-testing-pipelines-service.sh +++ b/projects/control-service/cicd/deploy-testing-pipelines-service.sh @@ -80,9 +80,6 @@ if [[ $helm_latest_deployment == *"pending-upgrade"* ]]; then fi - -if[] - # # TODO :change container images with official ones when they are being deployed (I've currently uploaded them once in ghcr.io/tozka) # From bd0c0598351b4ea7f4dd6f8ef9957d429297ad0c Mon Sep 17 00:00:00 2001 From: murphp15 Date: Thu, 25 May 2023 09:20:24 +0100 Subject: [PATCH 5/6] control-service: fix failing tests Signed-off-by: murphp15 --- projects/control-service/cicd/.gitlab-ci.yml | 83 +++++++++++++++++++- 1 file changed, 79 insertions(+), 4 deletions(-) diff --git a/projects/control-service/cicd/.gitlab-ci.yml b/projects/control-service/cicd/.gitlab-ci.yml index 7685a74cb2..b8dc3f3530 100644 --- a/projects/control-service/cicd/.gitlab-ci.yml +++ b/projects/control-service/cicd/.gitlab-ci.yml @@ -39,6 +39,77 @@ - apk --no-cache add openjdk17-jdk git zip curl zip py-pip - pip install --upgrade pip && pip install awscli +control_service_build_image: + extends: + - .control_service_base_build + - .images:dind + stage: build + script: + - apk --no-cache add git openjdk17-jdk + - export TAG=$(git rev-parse --short HEAD) + - cd projects/control-service/projects + - ./gradlew -p ./model build publishToMavenLocal --info --stacktrace + - ./gradlew build jacocoTestReport -x integrationTest --info --stacktrace +# - ./gradlew :pipelines_control_service:docker --info --stacktrace -Pversion=$TAG + retry: !reference [.control_service_retry, retry_options] + coverage: "/ - Line Coverage: ([0-9.]+)%/" + artifacts: + when: always + paths: + - ./projects/control-service/projects/*/build/reports/** + expire_in: 1 week + reports: + junit: ./projects/control-service/projects/**/build/test-results/test/TEST-*.xml + only: + refs: + - external_pull_requests + changes: *control_service_code_change_locations + +control_service_integration_test: + extends: .control_service_base_build + stage: build + variables: + DEPLOYMENT_K8S_KUBECONFIG: "~/.kube/config" + DEPLOYMENT_K8S_NAMESPACE: $CICD_KUBERNETES_NAMESPACE + CONTROL_K8S_KUBECONFIG: "~/.kube/config" + CONTROL_K8S_NAMESPACE: $CICD_KUBERNETES_NAMESPACE + DOCKER_REGISTRY_URL: $CICD_CONTAINER_REGISTRY_URI + DOCKER_REGISTRY_USERNAME: $CICD_CONTAINER_REGISTRY_USER_NAME + DOCKER_REGISTRY_PASSWORD: $CICD_CONTAINER_REGISTRY_USER_PASSWORD + GIT_URL: $CICD_GIT_URI + GIT_USERNAME: $CICD_GIT_USER + GIT_PASSWORD: $CICD_GIT_PASSWORD + GIT_USERNAME_READ_WRITE: $CICD_GIT_USER + GIT_PASSWORD_READ_WRITE: $CICD_GIT_PASSWORD + DATAJOBS_AWS_SERVICE_ACCOUNT_ACCESS_KEY_ID: $DATAJOBS_AWS_SERVICE_ACCOUNT_ACCESS_KEY_ID + DATAJOBS_AWS_SERVICE_ACCOUNT_SECRET_ACCESS_KEY: $DATAJOBS_AWS_SERVICE_ACCOUNT_SECRET_ACCESS_KEY + script: + - cd projects/control-service/projects + - ./gradlew -p ./model build publishToMavenLocal + - mkdir -p ~/.kube + - cp $KUBECONFIG ~/.kube/config + - ./gradlew -v + - ./gradlew projects --info + - ./gradlew -Djdk.tls.client.protocols=TLSv1.2 :pipelines_control_service:integrationTest --info --stacktrace + retry: !reference [.control_service_retry, retry_options] + artifacts: + when: always + paths: + - ./projects/control-service/projects/*/build/reports/** + expire_in: 1 week + reports: + junit: ./projects/control-service/projects/**/build/test-results/integrationTest/TEST-*.xml + only: + refs: + - external_pull_requests + changes: *control_service_code_change_locations + except: + changes: + - projects/control-service/projects/helm_charts/pipelines-control-service/version.txt + - projects/control-service/projects/model/CONTRIBUTING.MD + - projects/control-service/projects/**/README.md + - projects/control-service/projects/model/apidefs/datajob-api/config-python.json + control_service_test_helm_chart: stage: build script: @@ -188,10 +259,14 @@ control_service_deploy_testing_data_pipelines: - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash - bash -ex ./projects/control-service/cicd/deploy-testing-pipelines-service.sh retry: !reference [.control_service_retry, retry_options] - only: - refs: - - external_pull_requests - changes: *control_service_code_change_locations + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + changes: [ projects/control-service/projects/helm_charts/pipelines-control-service/version.txt ] + when: never + - if: '$CI_COMMIT_BRANCH == "main"' + changes: *control_service_code_change_locations + - if: '$CI_COMMIT_BRANCH == "main"' + changes: *control_service_helm_change_locations # vdk-heartbeat source: https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-heartbeat control_service_post_deployment_test: From d3e94b93f4aa5da4ee510edaae92b0f52e11688a Mon Sep 17 00:00:00 2001 From: murphp15 Date: Thu, 25 May 2023 09:21:16 +0100 Subject: [PATCH 6/6] control-service: fix failing tests Signed-off-by: murphp15 --- .../control-service/cicd/deploy-testing-pipelines-service.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/control-service/cicd/deploy-testing-pipelines-service.sh b/projects/control-service/cicd/deploy-testing-pipelines-service.sh index 799e6b7fcc..b9151becde 100755 --- a/projects/control-service/cicd/deploy-testing-pipelines-service.sh +++ b/projects/control-service/cicd/deploy-testing-pipelines-service.sh @@ -75,7 +75,7 @@ echo $helm_latest_deployment if [[ $helm_latest_deployment == *"pending-upgrade"* ]]; then echo "Pipeline failed because of an existing deployment in the pending-state. If the problem persists use 'helm history cicd-control-service' to see the last successful deployment. - then use 'helm rollback cicd-control-service ' to rollback to that deployment. then re run this pipeline" + then use 'helm rollback cicd-control-service ' to rollback to that deployment. then re run this pipeline" exit 125 fi