Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch test environments to cfd bosh lites #3371

Open
wants to merge 2 commits into
base: v7
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/ops-files/diego-cell-instances.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,7 @@
- type: replace
path: /instance_groups/name=diego-cell/instances
value: 4

- type: replace
path: /instance_groups/name=isolated-diego-cell/jobs/name=rep/properties?/set_kernel_parameters
value: false
45 changes: 28 additions & 17 deletions .github/workflows/tests-integration-reusable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# secrets.CLIENT_SECRET
# secrets.GITHUB_TOKEN
# secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN
# vars.SHEPHERD_POOL_NAME
# vars.TEST_FLAKE_ATTEMPTS

name: "pvt: run integration tests"

Expand All @@ -22,16 +22,29 @@ on:
lease-id:
required: true
type: string
lease-namespace:
required: false
type: string
default: 'tas-devex'
gitRef:
type: string
default: ${{github.event.workflow_run.head_sha}}
nodes:
type: string
default: "16"

env:
NODES: ${{ inputs.nodes }}
FLAKE_ATTEMPTS: ${{ vars.TEST_FLAKE_ATTEMPTS || '4' }}

jobs:
run-integration-tests:
defaults:
run:
shell: bash
runs-on: ${{ inputs.os }}
runs-on: ${{ inputs.os }}
container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest
timeout-minutes: 120
steps:
- name: Checkout cli
uses: actions/checkout@v4
Expand All @@ -53,7 +66,9 @@ jobs:

- name: Install Tools
env:
account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
lease_namespace: ${{ inputs.lease-namespace }}
lease_id: ${{ inputs.lease-id }}
run: |
go version

Expand Down Expand Up @@ -87,23 +102,21 @@ jobs:
apt-get install -y build-essential unzip

shepherd login service-account ${account_token}
shepherd get lease ${{ inputs.lease-id }} --namespace tas-devex --json | jq .output > metadata.json
shepherd get lease ${lease_id} --namespace ${lease_namespace} --json | jq .output > metadata.json

- name: Run Integration Tests
if: ${{ !inputs.run-with-client-creds }}
run: |
ENV=$(cat metadata.json | jq -r '.name')
jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${ENV}.priv
env_name=$(cat metadata.json | jq -r '.name')
jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv
eval "$(bbl print-env --metadata-file ./metadata.json)"
export CF_INT_PASSWORD="$(credhub get -n /bosh-$ENV/cf/cf_admin_password | bosh interpolate --path /value -)"
export CF_INT_PASSWORD="$(credhub get -n /bosh-lite/cf/cf_admin_password | bosh interpolate --path /value -)"
export CF_INT_OIDC_USERNAME="admin-oidc"
export CF_INT_OIDC_PASSWORD=$(credhub get -n /bosh-$ENV/cf/uaa_oidc_admin_password | bosh interpolate --path /value -)
export CF_INT_OIDC_PASSWORD=$(credhub get -n /bosh-lite/cf/uaa_oidc_admin_password | bosh interpolate --path /value -)
API_URL="$(jq -r .cf.api_url metadata.json)"
export CF_INT_API="https://$API_URL"
export CF_DIAL_TIMEOUT=15
export CF_USERNAME=admin
export FLAKE_ATTEMPTS=2
export NODES=16
export GOPATH=$PWD/go
export PATH="$GOPATH/bin:$PATH"
export PATH="$PWD/out:$PATH"
Expand All @@ -123,18 +136,16 @@ jobs:
CF_INT_CLIENT_ID: 'potato-face'
CF_INT_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
run: |
ENV=$(cat metadata.json | jq -r '.name')
jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${ENV}.priv
env_name=$(cat metadata.json | jq -r '.name')
jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv
eval "$(bbl print-env --metadata-file ./metadata.json)"
export CF_INT_PASSWORD="$(credhub get -n /bosh-$ENV/cf/cf_admin_password | bosh interpolate --path /value -)"
export CF_INT_PASSWORD="$(credhub get -n /bosh-lite/cf/cf_admin_password | bosh interpolate --path /value -)"
export CF_INT_OIDC_USERNAME="admin-oidc"
export CF_INT_OIDC_PASSWORD=$(credhub get -n /bosh-$ENV/cf/uaa_oidc_admin_password | bosh interpolate --path /value -)
export CF_INT_OIDC_PASSWORD=$(credhub get -n /bosh-lite/cf/uaa_oidc_admin_password | bosh interpolate --path /value -)
API_URL="$(jq -r .cf.api_url metadata.json)"
export CF_INT_API="https://$API_URL"
export CF_DIAL_TIMEOUT=15
export CF_USERNAME=admin
export FLAKE_ATTEMPTS=2
export NODES=16
export GOPATH=$PWD/go
export PATH="$GOPATH/bin:$PATH"
export PATH="$PWD/out:$PATH"
Expand All @@ -146,4 +157,4 @@ jobs:
cf api ${CF_INT_API} --skip-ssl-validation
cf auth

make integration-tests-full-ci
make integration-tests-ci-client-creds
128 changes: 113 additions & 15 deletions .github/workflows/tests-integration.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,14 @@
# GitHub repo level Secrets and Variables

# secrets.CLIENT_SECRET
# secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN
# vars.CAPI_RELEASE_VERSION
# vars.SHEPHERD_LEASE_DURATION
# vars.SHEPHERD_LEASE_NAMESPACE
# vars.SHEPHERD_TEMPLATE_ARGUMENT
# vars.SHEPHERD_TEMPLATE_NAME
# vars.SHEPHERD_TEMPLATE_NAMESPACE

name: "Tests: Integration"

run-name: "Integration [${{ github.event_name }}: ${{ github.event.pull_request.head.sha || github.event.push.after || github.event.workflow_run.head_sha}}]: ${{ github.event.workflow_run.head_commit.message }}"
Expand All @@ -13,6 +24,34 @@ on:
- all
- run-integration-tests-cf-env
- run-integration-tests-cf-env-with-client-creds
nodes:
description: Number of test nodes
required: false
type: string
default: "12"
lease_id:
description: Pre-provisioned environment lease-id to use in tests
required: false
type: string
lease_namespace:
description: Pre-provisioned environment lease namespace to use in tests
required: false
type: string
cfd_version:
description: Use specific version of CFD. Leave empty to use latest.
default: ""
type: string
run_unit_tests:
description: Run unit tests
required: false
type: boolean
default: true
reinstall_cfd:
description: Force re-installation of CFD
required: false
type: boolean
default: true

push:
tags:
- "v8.*"
Expand All @@ -26,7 +65,16 @@ on:
- "doc/**"
- ".gitpod.yml"
- "README.md"
- ".github/**"
- ".grype.yaml"
- ".git*"
- ".golangci.json"

env:
SHEPHERD_LEASE_ID: ${{ inputs.lease_id }}

jobs:

get-sha:
runs-on: ubuntu-latest
outputs:
Expand All @@ -52,15 +100,18 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v4
if: ${{ inputs.run_unit_tests == 'true' }}
with:
ref: ${{needs.get-sha.outputs.gitRef}}
fetch-depth: 0
- name: Set Up Go
uses: actions/setup-go@v5
if: ${{ inputs.run_unit_tests == 'true' }}
with:
go-version-file: go.mod
check-latest: true
- name: Run Units
if: ${{ inputs.run_unit_tests == 'true' }}
run: make units

claim-env:
Expand All @@ -84,23 +135,63 @@ jobs:
- name: claim
id: claim
env:
account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
pool_name: ${{ vars.SHEPHERD_POOL_NAME }}
pool_namespace: official
account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
template_argument: ${{ vars.SHEPHERD_TEMPLATE_ARGUMENT }}
template_name: ${{ vars.SHEPHERD_TEMPLATE_NAME || 'cfd-bosh-lite@1.0' }}
template_namespace: ${{ vars.SHEPHERD_TEMPLATE_NAMESPACE || 'official' }}
lease_duration: ${{ vars.SHEPHERD_LEASE_DURATION || '8h' }}
lease_namespace: ${{ inputs.lease_namespace || vars.SHEPHERD_LEASE_NAMESPACE || 'tas-devex' }}
cfd_version: ${{ inputs.cfd_version || vars.CFD_VERSION || '' }}
run: |
shepherd login service-account ${account_token}

echo "shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --description 'CLI GHA'"
lease_id=$(shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --json | jq -r .id)

if [[ -z $SHEPHERD_LEASE_ID ]]; then

if [ -z "$template_argument" ]; then
export template_argument=$(cat <<EOF
{
"gcp_region": "us-west2",
"vm_type": "n1-standard-8",
"root_disk_gb": 32,
"disk_pool_gb": 150,
"cfd_version": "${cfd_version}",
"additional_opsfiles_b64": ""
}
EOF
)
fi

lease_id=$( shepherd create lease \
--template-argument "$template_argument" \
--template-namespace "${template_namespace}" \
--template "${template_name}" \
--namespace "${lease_namespace}" \
--duration "${lease_duration}" \
--description "Claimed by CF CLI workflow ${{ github.workflow_run.url }}" \
--json \
| jq -r .id
)
else
lease_id=$SHEPHERD_LEASE_ID
fi

echo "Shepherd lease ID: ${lease_id}"

# Give sometime for the lease to complete. Shepherd may take upto an 3 hours to create an env
# if the pool is empty.
count=0
while [ $count -lt 360 ] ; do
sleep 30
status=$(shepherd get lease ${lease_id} --namespace tas-devex --json | jq -r .status)
status=$( shepherd get lease ${lease_id} \
--namespace ${lease_namespace} \
--json \
| jq -r .status
)
if [ $status == "LEASED" ] ; then
shepherd get lease ${lease_id} --namespace tas-devex --json | jq .output > metadata.json
shepherd get lease ${lease_id} \
--namespace ${lease_namespace} \
--json \
| jq .output > metadata.json
break
elif [ $status == "FAILED" -o $status == "EXPIRED" ] ; then
echo "There was an error obtaining the lease. Lease status is ${status}."
Expand All @@ -115,7 +206,7 @@ jobs:
echo "env name is ${env_name}"
echo "leaseid=${lease_id}" >> "${GITHUB_OUTPUT}"

cf_deployment_version=$(jq -r '."cf-deployment_version"' metadata.json)
cf_deployment_version=$(jq -r '."cf_deployment_version"' metadata.json)
echo "cf_deployment_version is ${cf_deployment_version}"
echo "cf_deployment_version=${cf_deployment_version}" >> "${GITHUB_OUTPUT}"

Expand All @@ -126,6 +217,7 @@ jobs:
check-latest: true

- name: Install Tools
if: ${{ (inputs.lease_id == '') || (inputs.reinstall_cfd == true) }}
run: |
go version

Expand All @@ -145,11 +237,11 @@ jobs:
apt-get install -y build-essential unzip

- name: Upload latest CAPI release
if: ${{ (inputs.lease_id == '') || (inputs.reinstall_cfd == true) }}
env:
capi_release_version: ${{ vars.CAPI_RELEASE_VERSION }}
run: |
if [ -z "$capi_release_version" ]
then
if [ -z "$capi_release_version" ]; then
capi_release_version=$(curl -s https://api.github.com/repos/cloudfoundry/capi-release/releases/latest | jq -r .tag_name)
fi

Expand All @@ -162,19 +254,20 @@ jobs:
bosh upload-release "https://bosh.io/d/github.com/cloudfoundry/capi-release?v=$capi_release_version"

- name: Checkout cf-deployment
if: ${{ (inputs.lease_id == '') || (inputs.reinstall_cfd == true) }}
uses: actions/checkout@v4
with:
repository: cloudfoundry/cf-deployment
path: cf-deployment
ref: ${{steps.claim.outputs.cf_deployment_version}}

- name: Deploy Isolation Segment and OIDC Provider
if: ${{ (inputs.lease_id == '') || (inputs.reinstall_cfd == true) }}
run: |
env_name=$(jq -r .name metadata.json)
jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv
eval "$(bbl print-env --metadata-file metadata.json)"

# deploy
bosh -d cf manifest > /tmp/manifest.yml
bosh interpolate /tmp/manifest.yml \
-o cf-deployment/operations/enable-v2-api.yml \
Expand Down Expand Up @@ -205,6 +298,8 @@ jobs:
name: Integration
gitRef: ${{needs.get-sha.outputs.gitRef}}
lease-id: ${{ needs.claim-env.outputs.leaseid }}
lease-namespace: ${{ inputs.lease_namespace || vars.SHEPHERD_LEASE_NAMESPACE || 'tas-devex' }}
nodes: ${{ inputs.nodes }}
secrets: inherit

run-integration-tests-cf-env-with-client-creds:
Expand All @@ -221,22 +316,25 @@ jobs:
name: Integration client creds
gitRef: ${{needs.get-sha.outputs.gitRef}}
lease-id: ${{ needs.claim-env.outputs.leaseid }}
lease-namespace: ${{ inputs.lease_namespace || vars.SHEPHERD_LEASE_NAMESPACE || 'tas-devex' }}
secrets: inherit

unclaim-env:
name: Unclaim environment
if: ${{ inputs.lease_id == '' }}
runs-on: ubuntu-latest
container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest
needs:
- claim-env
- run-integration-tests-cf-env
- run-integration-tests-cf-env-with-client-creds
if: always()
steps:
- name: unclaim
env:
account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
lease_namespace: ${{ inputs.lease_namespace || vars.SHEPHERD_LEASE_NAMESPACE || 'tas-devex' }}
run: |
shepherd login service-account ${account_token}
set -x
shepherd delete lease ${{ needs.claim-env.outputs.leaseid }} --namespace tas-devex
shepherd delete lease ${{ needs.claim-env.outputs.leaseid }} \
--namespace ${lease_namespace}
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,5 @@ integration/assets/test_plugin/test_plugin

### VisualStudioCode ###
.vscode
.secrets
.vars
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
CF_DIAL_TIMEOUT ?= 15
NODES ?= 10
FLAKE_ATTEMPTS ?=5
PACKAGES ?= api actor command types util version integration/helpers
LC_ALL = "en_US.UTF-8"

Expand Down Expand Up @@ -131,6 +132,7 @@ integration-selfcontained: build install-test-deps

integration-tests: build integration-cleanup integration-isolated integration-push integration-global integration-selfcontained ## Run all isolated, push, selfcontained, and global integration tests

integration-tests-ci-client-creds: build integration-cleanup integration-push integration-global integration-selfcontained

i: integration-tests-full
integration-full-tests: integration-tests-full
Expand Down
Loading
Loading