From 8e9ccdc89c083143af609107ee459a1d1deb8259 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 17 Dec 2024 10:26:10 +0100 Subject: [PATCH 1/8] fix linux ci --- .github/workflows/LocalTesting.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 5bb46fe..264cb51 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -11,7 +11,6 @@ jobs: azurite-tests-linux: name: Azurite tests (Linux) runs-on: ubuntu-latest - container: 'quay.io/pypa/manylinux2014_x86_64' env: VCPKG_TARGET_TRIPLET: 'x64-linux' GEN: Ninja @@ -22,17 +21,18 @@ jobs: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 submodules: 'true' + - uses: actions/setup-node@v4 + - name: install Azure test service run: | - yum install -y nodejs npm squid + sudo apt-get install -y azure-cli squid npm install -g azurite echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" | tee /etc/yum.repos.d/azure-cli.repo - yum install -y azure-cli - name: Launch & populate Azure test service run: | @@ -42,10 +42,6 @@ jobs: sleep 10 ./scripts/upload_test_files_to_azurite.sh - - name: Setup ManyLinux2014 - run: | - ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh openssl python_alias - - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: From 504d46e2ab6a5b5d72ba78993b3464625fca816f Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 17 Dec 2024 11:05:02 +0100 Subject: [PATCH 2/8] remove yum related line --- .github/workflows/LocalTesting.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 264cb51..e2c525e 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -32,7 +32,6 @@ jobs: run: | sudo apt-get install -y azure-cli squid npm install -g azurite - echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" | tee /etc/yum.repos.d/azure-cli.repo - name: Launch & populate Azure test service run: | From d538263e2b19d0724feed440610b926764f84256 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 7 Jan 2025 11:50:34 +0100 Subject: [PATCH 3/8] bumping versions --- .github/workflows/CloudTesting.yml | 5 ++--- .github/workflows/LocalTesting.yml | 12 ++++++------ duckdb | 2 +- extension-ci-tools | 2 +- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/CloudTesting.yml b/.github/workflows/CloudTesting.yml index 092a837..54c9c66 100644 --- a/.github/workflows/CloudTesting.yml +++ b/.github/workflows/CloudTesting.yml @@ -14,9 +14,8 @@ jobs: env: VCPKG_TARGET_TRIPLET: x64-linux VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - GEN: Ninja + GEN: ninja DUCKDB_PLATFORM: linux_amd64 - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - name: Install required ubuntu packages @@ -40,7 +39,7 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: - vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 + vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574 - name: Build extension env: diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index e2c525e..038029d 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest env: VCPKG_TARGET_TRIPLET: 'x64-linux' - GEN: Ninja + GEN: ninja VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' AZURE_STORAGE_ACCOUNT: devstoreaccount1 @@ -44,7 +44,7 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: - vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 + vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574 # Build extension - name: Build extension @@ -81,7 +81,7 @@ jobs: env: VCPKG_TARGET_TRIPLET: 'x64-osx' OSX_BUILD_ARCH: 'x86_64' - GEN: Ninja + GEN: ninja VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' AZURE_STORAGE_ACCOUNT: devstoreaccount1 @@ -108,7 +108,7 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11 with: - vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 + vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574 - name: Build extension shell: bash @@ -145,7 +145,7 @@ jobs: runs-on: windows-latest env: VCPKG_TARGET_TRIPLET: 'x64-windows' - GEN: Ninja + GEN: ninja VCPKG_ROOT: ${{ github.workspace }}\vcpkg VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' @@ -160,7 +160,7 @@ jobs: - name: Setup vcpkg uses: lukka/run-vcpkg@v11 with: - vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 + vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574 - uses: actions/setup-python@v2 with: diff --git a/duckdb b/duckdb index 78ebe44..f99785b 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 78ebe44ef9f31b43cfc41b3bf739ab9069e16ae8 +Subproject commit f99785b78ae4724b31d9b41435ad8c17e57ee8f4 diff --git a/extension-ci-tools b/extension-ci-tools index 46a8dd6..0cd9d80 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit 46a8dd6850b218b2c1a0ede5daf5053bdc225102 +Subproject commit 0cd9d804a40f256e90170c1c3c9922734e198e3f From 9dc6b7e8a2551254749b3b9a2291fc2f58d04183 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 7 Jan 2025 12:08:22 +0100 Subject: [PATCH 4/8] install ninja on linux, disable on windows --- .github/workflows/LocalTesting.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 038029d..c850b61 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -18,7 +18,6 @@ jobs: AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' AZURE_STORAGE_ACCOUNT: devstoreaccount1 HTTP_PROXY_RUNNING: '1' - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - uses: actions/checkout@v4 @@ -30,7 +29,7 @@ jobs: - name: install Azure test service run: | - sudo apt-get install -y azure-cli squid + sudo apt-get install -y azure-cli squid ninja-build npm install -g azurite - name: Launch & populate Azure test service @@ -145,7 +144,6 @@ jobs: runs-on: windows-latest env: VCPKG_TARGET_TRIPLET: 'x64-windows' - GEN: ninja VCPKG_ROOT: ${{ github.workspace }}\vcpkg VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' From ba1d3ef98e7e00213ea3117bb8aaec0838b4a4a7 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 7 Jan 2025 13:24:50 +0100 Subject: [PATCH 5/8] create missing directories --- .github/workflows/LocalTesting.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index c850b61..119c8a0 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -35,6 +35,8 @@ jobs: - name: Launch & populate Azure test service run: | azurite > azurite_log.txt 2>&1 & + mkdir squid_logs + mkdir squid_auth_logs ./scripts/run_squid.sh --port 3128 --log_dir squid_logs & ./scripts/run_squid.sh --port 3129 --log_dir squid_auth_logs --auth & sleep 10 From cb8cea3650267d10c41e22bf2b6660f09a4002df Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 7 Jan 2025 13:25:05 +0100 Subject: [PATCH 6/8] run squid with sudo --- .github/workflows/LocalTesting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 119c8a0..e11b0e8 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -37,8 +37,8 @@ jobs: azurite > azurite_log.txt 2>&1 & mkdir squid_logs mkdir squid_auth_logs - ./scripts/run_squid.sh --port 3128 --log_dir squid_logs & - ./scripts/run_squid.sh --port 3129 --log_dir squid_auth_logs --auth & + sudo ./scripts/run_squid.sh --port 3128 --log_dir squid_logs & + sudo ./scripts/run_squid.sh --port 3129 --log_dir squid_auth_logs --auth & sleep 10 ./scripts/upload_test_files_to_azurite.sh From e99121c71fda0409c46bb52d7a2469faf8c3e459 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 7 Jan 2025 14:02:25 +0100 Subject: [PATCH 7/8] make squid use sudo --- .github/workflows/LocalTesting.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index e11b0e8..9df1496 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -35,8 +35,6 @@ jobs: - name: Launch & populate Azure test service run: | azurite > azurite_log.txt 2>&1 & - mkdir squid_logs - mkdir squid_auth_logs sudo ./scripts/run_squid.sh --port 3128 --log_dir squid_logs & sudo ./scripts/run_squid.sh --port 3129 --log_dir squid_auth_logs --auth & sleep 10 @@ -71,10 +69,10 @@ jobs: cat azurite_log.txt echo "## squid" - cat squid_logs/* + sudo cat squid_logs/* echo "## squid auth" - cat squid_auth_logs/* + sudo cat squid_auth_logs/* azurite-tests-macos: name: Azurite tests (MacOS) From 021b2468fe1e7c7637c309e0eeae9dfeae11ef53 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 8 Jan 2025 10:32:14 +0100 Subject: [PATCH 8/8] disable mingw --- .github/workflows/MainDistributionPipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 90a39a6..1c33eb0 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,7 +19,7 @@ jobs: extension_name: azure duckdb_version: main ci_tools_version: main - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools' + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw' duckdb-stable-deploy: name: Deploy extension binaries @@ -30,5 +30,5 @@ jobs: extension_name: azure duckdb_version: main ci_tools_version: main - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools' + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw' deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}