From 270bd6c92a50741aefb2315eaf66d52ace2b6141 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Mon, 12 Aug 2024 11:00:41 +0100 Subject: [PATCH 1/5] See if we can use an existing scale set --- .azure-pipelines/ultimate-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml index e914b60a9554..fb2726ab7af3 100644 --- a/.azure-pipelines/ultimate-pipeline.yml +++ b/.azure-pipelines/ultimate-pipeline.yml @@ -5991,7 +5991,7 @@ stages: variables: smokeTestAppDir: "$(System.DefaultWorkingDirectory)/tracer/test/test-applications/regression/AspNetCoreSmokeTest" pool: - vmImage: windows-2022 + name: azure-windows-scale-set steps: - template: steps/clone-repo.yml From 69bfa204360e13b5686a8170cdc189a4468ed1e3 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Mon, 12 Aug 2024 13:42:19 +0100 Subject: [PATCH 2/5] Try something else --- .../steps/install-docker-compose-v1.yml | 24 ++++++++++++++++ .azure-pipelines/steps/run-snapshot-test.yml | 28 +++++++++++-------- .azure-pipelines/ultimate-pipeline.yml | 8 +++++- 3 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 .azure-pipelines/steps/install-docker-compose-v1.yml diff --git a/.azure-pipelines/steps/install-docker-compose-v1.yml b/.azure-pipelines/steps/install-docker-compose-v1.yml new file mode 100644 index 000000000000..7a985149a7e6 --- /dev/null +++ b/.azure-pipelines/steps/install-docker-compose-v1.yml @@ -0,0 +1,24 @@ +parameters: + - name: isLinux + type: boolean + default: false + + - name: 'dockerComposePath' + type: string + default: 'C:\docker-compose\docker-compose.exe' + +steps: +- ${{ if eq(parameters.isLinux, true) }}: + - bash: | + sudo mkdir -p "$(dirname "${{ parameters.dockerComposePath }}")" + sudo curl -SL https://github.com/docker/compose/releases/download/1.29.2/docker-compose-linux-x86_64 -o ${{ parameters.dockerComposePath }} + sudo chmod 755 ${{ parameters.dockerComposePath }} + displayName: Download docker-compose +- ${{ else }}: + - powershell: | + $dir= (Split-Path -parent "${{ parameters.dockerComposePath }}") + mkdir -f -p $dir + # GitHub now requires TLS1.2. In PowerShell, run the following + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest "https://github.com/docker/compose/releases/download/1.29.1/docker-compose-windows-x86_64.exe" -OutFile "${{ parameters.dockerComposePath }}" + displayName: Download docker-compose diff --git a/.azure-pipelines/steps/run-snapshot-test.yml b/.azure-pipelines/steps/run-snapshot-test.yml index 781a4cebeb0a..1c37bd42c821 100644 --- a/.azure-pipelines/steps/run-snapshot-test.yml +++ b/.azure-pipelines/steps/run-snapshot-test.yml @@ -16,6 +16,10 @@ parameters: - name: 'apiKey' type: string default: '' + + - name: 'dockerComposePath' + type: string + default: 'docker-compose' steps: - template: ./clean-docker-containers.yml @@ -60,7 +64,7 @@ steps: displayName: Set env-specific variables - bash: | - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET) + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET) env: dockerTag: $(dockerTag) DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} @@ -69,27 +73,27 @@ steps: - script: | echo "Starting snapshot session" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)" displayName: start snapshot session env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} - bash: | - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e PROFILER_IS_NOT_REQUIRED=${{ parameters.isNoop }} ${{ parameters.target }} + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e PROFILER_IS_NOT_REQUIRED=${{ parameters.isNoop }} ${{ parameters.target }} env: dockerTag: $(dockerTag) DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} - displayName: docker-compose run ${{ parameters.target }} + displayName: ${{ parameters.dockerComposePath }} run ${{ parameters.target }} - script: | echo "Dumping traces" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)" echo "Dumping stats" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)" echo "Dumping all requests" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)" displayName: dump snapshots env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} @@ -97,7 +101,7 @@ steps: - ${{ if and(eq(parameters.isLinux, true), eq(parameters.isNoop, false)) }}: - bash: | echo "Verifying snapshot session (fail on mis-match)" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)" displayName: check snapshots env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} @@ -120,19 +124,19 @@ steps: - ${{ else }}: - bash: | echo "Verifying snapshot session (fail on mis-match)" - docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)" + ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)" displayName: check snapshots env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} -- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET) +- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET) displayName: dump docker-compose logs for $(TEST_AGENT_TARGET) env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} condition: succeededOrFailed() continueOnError: true -- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down +- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down displayName: docker-compose stop services env: DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }} @@ -142,7 +146,7 @@ steps: # Run crash tests - ${{ if and(eq(parameters.isLinux, true), eq(parameters.isNoop, false)) }}: - bash: | - LOGS=$(docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }}) + LOGS=$(${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }}) echo $LOGS # check logs for evidence of crash detection in the output diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml index fb2726ab7af3..5faa66d1e0e1 100644 --- a/.azure-pipelines/ultimate-pipeline.yml +++ b/.azure-pipelines/ultimate-pipeline.yml @@ -5979,6 +5979,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:\docker-compose\docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5994,6 +5995,10 @@ stages: name: azure-windows-scale-set steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -6012,7 +6017,7 @@ stages: path: $(smokeTestAppDir)/artifacts - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -6029,6 +6034,7 @@ stages: target: 'tracer-home-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) From 710e4dbdf720a08ddfd1e8d0f3bc04ddc922b3b3 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 13 Aug 2024 09:54:13 +0100 Subject: [PATCH 3/5] grmbl grmbl --- .azure-pipelines/steps/install-docker-compose-v1.yml | 2 +- .azure-pipelines/ultimate-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/steps/install-docker-compose-v1.yml b/.azure-pipelines/steps/install-docker-compose-v1.yml index 7a985149a7e6..3c61eea2e67c 100644 --- a/.azure-pipelines/steps/install-docker-compose-v1.yml +++ b/.azure-pipelines/steps/install-docker-compose-v1.yml @@ -5,7 +5,7 @@ parameters: - name: 'dockerComposePath' type: string - default: 'C:\docker-compose\docker-compose.exe' + default: 'C:/docker-compose/docker-compose.exe' steps: - ${{ if eq(parameters.isLinux, true) }}: diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml index 5faa66d1e0e1..375128d322d1 100644 --- a/.azure-pipelines/ultimate-pipeline.yml +++ b/.azure-pipelines/ultimate-pipeline.yml @@ -5979,7 +5979,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] - dockerComposePath: 'C:\docker-compose\docker-compose.exe' + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: From bd037b38d76750addf0d833270cf1dbfb65fe894 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 13 Aug 2024 11:51:20 +0100 Subject: [PATCH 4/5] Try fix remaining snapshot tests --- .azure-pipelines/ultimate-pipeline.yml | 35 ++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml index 375128d322d1..41a55df7e548 100644 --- a/.azure-pipelines/ultimate-pipeline.yml +++ b/.azure-pipelines/ultimate-pipeline.yml @@ -5687,6 +5687,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5702,6 +5703,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5725,7 +5730,7 @@ stages: displayName: create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5742,9 +5747,10 @@ stages: target: 'nuget-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5761,6 +5767,7 @@ stages: target: 'nuget-dddotnet-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) @@ -5777,6 +5784,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5792,6 +5800,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5812,7 +5824,7 @@ stages: displayName: Create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5828,6 +5840,7 @@ stages: target: 'dotnet-tool-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) @@ -5844,6 +5857,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5859,6 +5873,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5879,7 +5897,7 @@ stages: displayName: Create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5895,6 +5913,7 @@ stages: target: 'smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) @@ -5911,6 +5930,7 @@ stages: variables: targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + dockerComposePath: 'C:/docker-compose/docker-compose.exe' jobs: - template: steps/update-github-status-jobs.yml parameters: @@ -5927,6 +5947,10 @@ stages: vmImage: windows-2022 steps: + - template: steps/install-docker-compose-v1.yml + parameters: + dockerComposePath: $(dockerComposePath) + - template: steps/clone-repo.yml parameters: targetShaId: $(targetShaId) @@ -5947,7 +5971,7 @@ stages: displayName: Create test data directories - bash: | - docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ + $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \ --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \ --build-arg RUNTIME_IMAGE=$(runtimeImage) \ --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \ @@ -5963,6 +5987,7 @@ stages: target: 'dd-dotnet-smoke-tests.windows' snapshotPrefix: "smoke_test" isLinux: false + dockerComposePath: $(dockerComposePath) - publish: tracer/build_data artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) From 860007719c93abcbd4e53445b801bd718c5d7c9a Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 13 Aug 2024 12:31:39 +0100 Subject: [PATCH 5/5] Clean up unused networks in snapshot tests --- .azure-pipelines/steps/run-snapshot-test.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run-snapshot-test.yml b/.azure-pipelines/steps/run-snapshot-test.yml index 1c37bd42c821..bf87591f011b 100644 --- a/.azure-pipelines/steps/run-snapshot-test.yml +++ b/.azure-pipelines/steps/run-snapshot-test.yml @@ -168,4 +168,12 @@ steps: - script: | sudo chmod -R 644 tracer/build_data/dumps/* || true displayName: Make dumps uploadable to AzDo - condition: succeededOrFailed() \ No newline at end of file + condition: succeededOrFailed() + +- script: | + docker network prune -f + displayName: Clean up docker networks + condition: succeededOrFailed() + continueOnError: true + + \ No newline at end of file