From 36a5ec1db1b6711898beb06fca43590745b1bea4 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 6 Feb 2023 11:43:54 -0800 Subject: [PATCH] Improvements in Pipeline parameters (#2748) * Improvements in Pipeline parameters * remove extra space * fix jq --------- Co-authored-by: narrieta --- tests_e2e/orchestrator/docker/Dockerfile | 1 + tests_e2e/orchestrator/runbook.yml | 49 ++++++------ tests_e2e/orchestrator/scripts/run-scenarios | 83 -------------------- tests_e2e/pipeline/pipeline-cleanup.yml | 56 +++++++++++++ tests_e2e/pipeline/pipeline.yml | 16 ++-- tests_e2e/pipeline/scripts/execute_tests.sh | 60 ++++++++++---- 6 files changed, 134 insertions(+), 131 deletions(-) delete mode 100755 tests_e2e/orchestrator/scripts/run-scenarios create mode 100644 tests_e2e/pipeline/pipeline-cleanup.yml diff --git a/tests_e2e/orchestrator/docker/Dockerfile b/tests_e2e/orchestrator/docker/Dockerfile index 6699ff4bba..08fd4ca312 100644 --- a/tests_e2e/orchestrator/docker/Dockerfile +++ b/tests_e2e/orchestrator/docker/Dockerfile @@ -73,5 +73,6 @@ RUN \ # \ echo 'export PYTHONPATH="$HOME/WALinuxAgent"' >> $HOME/.bash_profile && \ echo 'export PATH="$HOME/.local/bin:$PATH"' >> $HOME/.bash_profile && \ + echo 'cd $HOME' >> $HOME/.bash_profile && \ : diff --git a/tests_e2e/orchestrator/runbook.yml b/tests_e2e/orchestrator/runbook.yml index 0d91b376f3..74bbec46e4 100644 --- a/tests_e2e/orchestrator/runbook.yml +++ b/tests_e2e/orchestrator/runbook.yml @@ -8,12 +8,11 @@ extension: - "./lib" variable: + # + # These variables define runbook parameters; they are handled by LISA. + # - name: subscription_id value: "" - - name: keep_environment - value: "no" - - name: wait_delete - value: false - name: user value: "waagent" - name: identity_file @@ -22,9 +21,28 @@ variable: - name: admin_password value: "" is_secret: true + - name: keep_environment + value: "no" + # + # These variables define parameters for the AgentTestSuite; see the test wiki for details. + # + # The test suites to execute + - name: test_suites + value: "agent_bvt" + is_case_visible: true + + # Whether to collect logs from the test VM + - name: collect_logs + value: "failed" + is_case_visible: true + + # Whether to skip setup of the test VM + - name: skip_setup + value: false + is_case_visible: true # - # Set these to use an SSH proxy + # Set these to use an SSH proxy when executing the runbook # - name: proxy value: False @@ -48,25 +66,6 @@ variable: - name: default_location value: "westus2" - # - # These variables define parameters for the AgentTestSuite; see the test wiki for details - # - # The test suites to execute - - name: test_suites - value: "agent_bvt" - is_case_visible: true - - # Whether to collect logs from the test VM - - name: collect_logs - value: "failed" - is_case_visible: true - - # Whether to skip setup of the test VM - - name: skip_setup - value: false - is_case_visible: true - - platform: - type: azure admin_username: $(user) @@ -76,7 +75,7 @@ platform: azure: deploy: True subscription_id: $(subscription_id) - wait_delete: $(wait_delete) + wait_delete: false requirement: core_count: min: 2 diff --git a/tests_e2e/orchestrator/scripts/run-scenarios b/tests_e2e/orchestrator/scripts/run-scenarios deleted file mode 100755 index 09306ecc95..0000000000 --- a/tests_e2e/orchestrator/scripts/run-scenarios +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -# Microsoft Azure Linux Agent -# -# Copyright 2018 Microsoft Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# This script runs on the container executing the tests. It creates the SSH keys (private and public) used -# to manage the test VMs taking the initial key value from the file shared by the container host, then it -# executes the daily test runbook. -# -set -euxo pipefail - -usage() ( - echo "Usage: run-scenarios [-t|--test-suites ] [-l|--collect-logs ] [-k|--skip-setup ]" - exit 1 -) - -test_suite_parameters="" - -while [[ $# -gt 0 ]] -do - case $1 in - -t|--test-suites) - shift - if [ "$#" -lt 1 ]; then - usage - fi - test_suite_parameters="$test_suite_parameters -v test_suites:$1" - ;; - -l|--collect-logs) - shift - if [ "$#" -lt 1 ]; then - usage - fi - test_suite_parameters="$test_suite_parameters -v collect_logs:$1" - ;; - -k|--skip-setup) - shift - if [ "$#" -lt 1 ]; then - usage - fi - test_suite_parameters="$test_suite_parameters -v skip_setup:$1" - ;; - *) - usage - esac - shift -done - -# The private ssh key is shared from the container host as $HOME/id_rsa; copy it to -# $HOME/.ssh, set the correct mode and generate the public key. -cd "$HOME" -mkdir .ssh -cp id_rsa .ssh -chmod 700 .ssh/id_rsa -ssh-keygen -y -f .ssh/id_rsa > .ssh/id_rsa.pub - -# -# Now start the runbook -# -lisa_logs="$HOME/logs/lisa" - -lisa \ - --runbook "$HOME/WALinuxAgent/tests_e2e/orchestrator/runbook.yml" \ - --log_path "$lisa_logs" \ - --working_path "$lisa_logs" \ - -v subscription_id:"$SUBSCRIPTION_ID" \ - -v identity_file:"$HOME/.ssh/id_rsa" \ - $test_suite_parameters diff --git a/tests_e2e/pipeline/pipeline-cleanup.yml b/tests_e2e/pipeline/pipeline-cleanup.yml new file mode 100644 index 0000000000..fd01212131 --- /dev/null +++ b/tests_e2e/pipeline/pipeline-cleanup.yml @@ -0,0 +1,56 @@ +# +# Pipeline for cleaning up any remaining Resource Groups generated by the Azure.WALinuxAgent pipeline. +# +# Runs every 3 hours and deletes any resource groups that are more than a day old and contain string "lisa-WALinuxAgent-" +# +schedules: + - cron: "0 */3 * * *" # Run every 3 hours + displayName: cleanup build + branches: + include: + - develop + always: true + +# no PR triggers +pr: none + +pool: + vmImage: ubuntu-latest + +variables: + - name: azureConnection + value: 'azuremanagement' + - name: rgPrefix + value: 'lisa-WALinuxAgent-' + +steps: + + - task: AzureKeyVault@2 + displayName: "Fetch secrets from KV" + inputs: + azureSubscription: '$(azureConnection)' + KeyVaultName: 'dcrV2SPs' + SecretsFilter: '*' + RunAsPreJob: true + + - task: AzureCLI@2 + inputs: + azureSubscription: '$(azureConnection)' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -euxo pipefail + date=`date --utc +%Y-%m-%d'T'%H:%M:%S.%N'Z' -d "1 day ago"` + + # Using the Azure REST GET resourceGroups API call as we can add the createdTime to the results. + # This feature is not available via the az-cli commands directly so we have to use the Azure REST APIs + + az rest --method GET \ + --url "https://management.azure.com/subscriptions/$(SUBSCRIPTION-ID)/resourcegroups" \ + --url-parameters api-version=2021-04-01 \$expand=createdTime \ + --output json \ + --query value \ + | jq --arg date "$date" '.[] | select (.createdTime < $date).name' \ + | grep "$(rgPrefix)" \ + | xargs -l -t -r az group delete --no-wait -y -n \ + || echo "No resource groups found to delete" diff --git a/tests_e2e/pipeline/pipeline.yml b/tests_e2e/pipeline/pipeline.yml index d0c570e662..69154d08db 100644 --- a/tests_e2e/pipeline/pipeline.yml +++ b/tests_e2e/pipeline/pipeline.yml @@ -14,10 +14,14 @@ parameters: - failed - no - - name: skip_setup - displayName: Skip setup of the test VMs - type: boolean - default: false + - name: keep_environment + displayName: Keep the test VMs (do not delete them) + type: string + default: no + values: + - always + - failed + - no variables: - name: azureConnection @@ -26,8 +30,8 @@ variables: value: ${{ parameters.test_suites }} - name: collect_logs value: ${{ parameters.collect_logs }} - - name: skip_setup - value: ${{ parameters.skip_setup }} + - name: keep_environment + value: ${{ parameters.keep_environment }} trigger: - develop diff --git a/tests_e2e/pipeline/scripts/execute_tests.sh b/tests_e2e/pipeline/scripts/execute_tests.sh index 06ed8c1ef6..b02df16bcc 100755 --- a/tests_e2e/pipeline/scripts/execute_tests.sh +++ b/tests_e2e/pipeline/scripts/execute_tests.sh @@ -2,36 +2,62 @@ set -euxo pipefail +# +# Set the correct mode for the private SSH key and generate the public key. +# +cd "$HOME" +mkdir ssh +cp "$DOWNLOADSSHKEY_SECUREFILEPATH" ssh +chmod 700 ssh/id_rsa +ssh-keygen -y -f ssh/id_rsa > ssh/id_rsa.pub + +# +# Change the ownership of the "ssh" directory we just created, as well as the sources and staging directories. +# Make waagent (UID 1000 in the container) the owner of both locations, so that it can write to them. +# This is needed because building the agent package writes the egg info to the source code directory, and +# tests write their logs to the staging directory. +# +sudo find ssh -exec chown 1000 {} \; +sudo chown 1000 "$BUILD_SOURCESDIRECTORY" +sudo chown 1000 "$BUILD_ARTIFACTSTAGINGDIRECTORY" + +# # Pull the container image used to execute the tests +# az login --service-principal --username "$AZURE_CLIENT_ID" --password "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID" > /dev/null az acr login --name waagenttests docker pull waagenttests.azurecr.io/waagenttests:latest -# Building the agent package writes the egg info to the source code directory, and test write their logs to the staging directory. -# Make waagent (UID 1000 in the container) the owner of both locations, so that it can write to them. -sudo chown 1000 "$BUILD_SOURCESDIRECTORY" -sudo chown 1000 "$BUILD_ARTIFACTSTAGINGDIRECTORY" - # A test failure will cause automation to exit with an error code and we don't want this script to stop so we force the command # to succeed and capture the exit code to return it at the end of the script. echo "exit 0" > /tmp/exit.sh docker run --rm \ - --volume "$BUILD_SOURCESDIRECTORY:/home/waagent/WALinuxAgent" \ - --volume "$DOWNLOADSSHKEY_SECUREFILEPATH:/home/waagent/id_rsa" \ - --volume "$BUILD_ARTIFACTSTAGINGDIRECTORY:/home/waagent/logs" \ - --env SUBSCRIPTION_ID \ - --env AZURE_CLIENT_ID \ - --env AZURE_CLIENT_SECRET \ - --env AZURE_TENANT_ID \ - waagenttests.azurecr.io/waagenttests \ - bash --login -c \ - "\$HOME/WALinuxAgent/tests_e2e/orchestrator/scripts/run-scenarios -t $TEST_SUITES -l $COLLECT_LOGS -k $SKIP_SETUP" \ + --volume "$BUILD_SOURCESDIRECTORY:/home/waagent/WALinuxAgent" \ + --volume "$HOME"/ssh:/home/waagent/.ssh \ + --volume "$BUILD_ARTIFACTSTAGINGDIRECTORY":/home/waagent/logs \ + --env AZURE_CLIENT_ID \ + --env AZURE_CLIENT_SECRET \ + --env AZURE_TENANT_ID \ + waagenttests.azurecr.io/waagenttests \ + bash --login -c \ + "lisa \ + --runbook \$HOME/WALinuxAgent/tests_e2e/orchestrator/runbook.yml \ + --log_path \$HOME/logs/lisa \ + --working_path \$HOME/logs/lisa \ + -v subscription_id:$SUBSCRIPTION_ID \ + -v identity_file:\$HOME/.ssh/id_rsa \ + -v test_suites:\"$TEST_SUITES\" \ + -v collect_logs:\"$COLLECT_LOGS\" \ + -v keep_environment:\"$KEEP_ENVIRONMENT\"" \ || echo "exit $?" > /tmp/exit.sh -# Retake ownership of the source and staging directory (note that the former does not need to be done recursively) +# +# Retake ownership of the source and staging directories (note that the former does not need to be done recursively; also, we don't need to +# retake ownership of the ssh directory) +# sudo chown "$USER" "$BUILD_SOURCESDIRECTORY" sudo find "$BUILD_ARTIFACTSTAGINGDIRECTORY" -exec chown "$USER" {} \; @@ -51,4 +77,4 @@ mv "$BUILD_ARTIFACTSTAGINGDIRECTORY"/lisa/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0- rm -r "$BUILD_ARTIFACTSTAGINGDIRECTORY"/lisa/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9] cat /tmp/exit.sh -bash /tmp/exit.sh \ No newline at end of file +bash /tmp/exit.sh