Move over httpfs tests from main repo to duckdb-httpfs + add CI run

.github/workflows/MinioTests.yml → .github/workflows/IntegrationTests.yml

@@ -1,4 +1,4 @@
-name: Minio Tests
+name: Integration Tests
 on: [push, pull_request,repository_dispatch]
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
@@ -8,17 +8,19 @@ defaults:
     shell: bash
 
 jobs:
-  minio-tests:
-    name: Minio Tests
-    runs-on: ubuntu-24.04
+  linux-tests-httpfs:
+    name: MinIO Tests
+    runs-on: ubuntu-latest
     env:
       S3_TEST_SERVER_AVAILABLE: 1
       AWS_DEFAULT_REGION: eu-west-1
       AWS_ACCESS_KEY_ID: minio_duckdb_user
       AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
       DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000
       DUCKDB_S3_USE_SSL: false
-      CORE_EXTENSIONS: 'parquet;json'
+      HTTP_PROXY_PUBLIC: localhost:3128
+      TEST_PERSISTENT_SECRETS_AVAILABLE: true
+      CORE_EXTENSIONS: "parquet;json;tpch"
       GEN: ninja
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
       VCPKG_TARGET_TRIPLET: x64-linux
@@ -43,37 +45,44 @@ jobs:
         uses: hendrikmuhs/ccache-action@main
         with:
           key: ${{ github.job }}
-          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
+          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-httpfs' }}
 
       - name: Setup vcpkg
         uses: lukka/run-vcpkg@v11.1
         with:
           vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574
 
-      - name: Build
+      - name: Fix permissions of test secrets
         shell: bash
-        run: |
-          echo -e "\nduckdb_extension_load(tpch)\n" >> extension_config.cmake
-          make
+        run: chmod -R 700 data/secrets
 
-      - name: Start S3/HTTP test server
+      # TODO: fix the authenticated proxy here
+      - name: Install and run http proxy squid
         shell: bash
         run: |
-          cd duckdb
-          mkdir data/attach_test
-          touch data/attach_test/attach.db
-          sudo ./scripts/install_s3_test_server.sh
-          source ./scripts/run_s3_test_server.sh
-          sleep 30
+          sudo apt-get install squid
+          ./scripts/run_squid.sh --port 3128 --log_dir squid_logs &
 
-      - name: Run & Populate test server
+      - name: Run & Populate Python test server
         shell: bash
         run: |
           mkdir -p $PYTHON_HTTP_SERVER_DIR
           cd $PYTHON_HTTP_SERVER_DIR
           python3 -m http.server 8008 &
 
-      - name: Test
+      - name: Build
+        shell: bash
+        run: make
+
+      - name: Install test server
+        shell: bash
+        run: |
+          sudo ./scripts/install_s3_test_server.sh
+          ./scripts/generate_presigned_url.sh
+
+      - name: Start test server & run tests
         shell: bash
         run: |
-          make test
+          source ./scripts/run_s3_test_server.sh
+          source ./scripts/set_s3_test_server_variables.sh
+          make test

.gitignore

@@ -355,3 +355,4 @@ extension_external
 
 test/sql/pragma/output.json
 tools/pythonpkg/duckdb_build/
+test/test_data

Makefile

@@ -4,21 +4,5 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 EXT_NAME=httpfs
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 
-CORE_EXTENSIONS=''
-
 # Include the Makefile from extension-ci-tools
 include extension-ci-tools/makefiles/duckdb_extension.Makefile
-
-
-## Add some more extra tests
-test_release_internal:
-	./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*"
-	./build/release/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json
-
-test_debug_internal:
-	./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*"
-	./build/debug/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json
-
-test_reldebug_internal:
-	./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*"
-	./build/reldebug/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json

data/secrets/README.md

@@ -0,0 +1,10 @@
+# Test secrets
+DuckDB only allows persistent secrets with the x00 permission (e.g. 600 or 700). Therefore to use these 
+secrets, the permissions need to be set before running any tests that uses them.
+
+The recommended way to add tests that touch these persistent secret files is to put them behind a
+```shell
+require-env TEST_PERSISTENT_SECRETS_AVAILABLE
+```
+statement, which ensures the tests only run in CI jobs where the permissions are set correctly.
+

extension_config.cmake

@@ -1,18 +1,10 @@
 # This file is included by DuckDB's build system. It specifies which extension to load
 
 ################# HTTPFS
-# Windows MinGW tests for httpfs currently not working
-if (NOT MINGW)
-    set(LOAD_HTTPFS_TESTS "LOAD_TESTS")
-else ()
-    set(LOAD_HTTPFS_TESTS "")
-endif()
-
 duckdb_extension_load(json)
 duckdb_extension_load(parquet)
 
 duckdb_extension_load(httpfs
 	SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
 	INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/extension/httpfs/include
-	${LOAD_HTTPFS_TESTS}
 )

scripts/generate_presigned_url.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+#Note: DONT run as root
+
+DUCKDB_PATH=duckdb
+if command -v duckdb; then
+  DUCKDB_PATH=duckdb
+elif test -f build/release/duckdb; then
+  DUCKDB_PATH=build/release/duckdb
+elif test -f build/reldebug/duckdb; then
+  DUCKDB_PATH=build/reldebug/duckdb
+elif test -f build/debug/duckdb; then
+  DUCKDB_PATH=build/debug/duckdb
+fi
+
+rm -rf test/test_data
+mkdir -p test/test_data
+
+generate_large_parquet_query=$(cat <<EOF
+
+CALL DBGEN(sf=1);
+COPY lineitem TO 'test/test_data/presigned-url-lineitem.parquet' (FORMAT 'parquet');
+
+EOF
+)
+$DUCKDB_PATH -c "$generate_large_parquet_query"
+
+# Generate Storage Version
+$DUCKDB_PATH  test/test_data/attach.db < duckdb/test/sql/storage_version/generate_storage_version.sql
+$DUCKDB_PATH  test/test_data/lineitem_sf1.db -c "CALL dbgen(sf=1)"

scripts/install_s3_test_server.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Note: needs sudo
+
+unamestr=$(uname)
+if [[ "$unamestr" == 'Linux' ]]; then
+	apt-get install -y docker.io
+fi
+
+docker --version
+echo '127.0.0.1 duckdb-minio.com' >> /etc/hosts
+echo '127.0.0.1 test-bucket.duckdb-minio.com' >> /etc/hosts
+echo '127.0.0.1 test-bucket-2.duckdb-minio.com' >> /etc/hosts
+echo '127.0.0.1 test-bucket-public.duckdb-minio.com' >> /etc/hosts

scripts/minio_s3.yml

@@ -0,0 +1,82 @@
+services:
+  minio:
+    image: minio/minio:RELEASE.2021-11-03T03-36-36Z
+    hostname: duckdb-minio.com
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - /tmp/minio_test_data:/data
+      - /tmp/minio_root_data:/root/.minio
+    environment:
+      - MINIO_ROOT_USER=duckdb_minio_admin
+      - MINIO_ROOT_PASSWORD=duckdb_minio_admin_password
+      - MINIO_REGION_NAME=eu-west-1
+      - MINIO_DOMAIN=duckdb-minio.com
+      - MINIO_ACCESS_KEY=duckdb_minio_admin
+      - MINIO_SECRET_KEY=duckdb_minio_admin_password
+    command: server /data --console-address ":9001"
+
+  minio_setup:
+    image: minio/mc:RELEASE.2021-11-05T10-05-06Z
+    depends_on:
+      - minio
+    links:
+      - minio
+    volumes:
+      - ${PWD}/duckdb/data:/duckdb/data
+      - ${PWD}/test/test_data:/duckdb/test_data
+
+    entrypoint: >
+      /bin/sh -c "
+        until (
+          /usr/bin/mc config host add myminio http://duckdb-minio.com:9000 duckdb_minio_admin duckdb_minio_admin_password
+        ) do
+          echo '...waiting...' && sleep 1;
+        done;
+
+        /usr/bin/mc admin user add myminio minio_duckdb_user minio_duckdb_user_password
+        /usr/bin/mc admin user list myminio
+        /usr/bin/mc admin user info myminio minio_duckdb_user
+        /usr/bin/mc admin policy set myminio readwrite user=minio_duckdb_user
+
+        /usr/bin/mc admin user add myminio minio_duckdb_user_2 minio_duckdb_user_2_password
+        /usr/bin/mc admin user list myminio
+        /usr/bin/mc admin user info myminio minio_duckdb_user_2
+        /usr/bin/mc admin policy set myminio readwrite user=minio_duckdb_user_2
+
+        /usr/bin/mc rb --force myminio/test-bucket
+        /usr/bin/mc mb myminio/test-bucket
+        /usr/bin/mc policy get myminio/test-bucket
+
+        /usr/bin/mc rb --force myminio/test-bucket-2
+        /usr/bin/mc mb myminio/test-bucket-2
+        /usr/bin/mc policy get myminio/test-bucket-2
+
+        /usr/bin/mc rb --force myminio/test-bucket-public
+        /usr/bin/mc mb myminio/test-bucket-public
+        /usr/bin/mc policy set download myminio/test-bucket-public
+        /usr/bin/mc policy get myminio/test-bucket-public
+
+        # This is for the test of presigned URLs
+        # !!! When missing, be sure that you have ran 'scripts/generate_presigned_url.sh' !!!
+
+        # small file upload
+        /usr/bin/mc cp /duckdb/data/csv/phonenumbers.csv myminio/test-bucket/presigned/phonenumbers.csv
+        /usr/bin/mc cp /duckdb/data/parquet-testing/glob/t1.parquet myminio/test-bucket/presigned/t1.parquet
+
+        # large file upload
+        /usr/bin/mc cp /duckdb/test_data/presigned-url-lineitem.parquet myminio/test-bucket/presigned/lineitem_large.parquet
+
+        # Upload the db for the attach 
+        /usr/bin/mc cp /duckdb/test_data/attach.db myminio/test-bucket/presigned/attach.db
+        /usr/bin/mc cp /duckdb/test_data/lineitem_sf1.db myminio/test-bucket/presigned/lineitem_sf1.db
+
+        /usr/bin/mc share download myminio/test-bucket/presigned/phonenumbers.csv
+        /usr/bin/mc share download myminio/test-bucket/presigned/t1.parquet
+        /usr/bin/mc share download myminio/test-bucket/presigned/lineitem_large.parquet
+        /usr/bin/mc share download myminio/test-bucket/presigned/attach.db
+
+        echo 'FINISHED SETTING UP MINIO'
+        exit 0;
+      "

scripts/run_s3_test_server.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#Note: DONT run as root
+
+if [ ! -f test/test_data/attach.db ]; then
+    echo "File test/test_data/attach.db not found, run ./scripts/generate_presigned_url.sh to generate"
+else
+  rm -rf /tmp/minio_test_data
+  rm -rf /tmp/minio_root_data
+  mkdir -p /tmp/minio_test_data
+  mkdir -p /tmp/minio_root_data
+  docker compose -f scripts/minio_s3.yml -p duckdb-minio up -d
+
+  # for testing presigned url
+  container_name=$(docker ps -a --format '{{.Names}}' | grep -m 1 "duckdb-minio")
+  echo $container_name
+
+  for i in $(seq 1 360);
+  do
+    docker_finish_logs=$(docker logs $container_name 2>/dev/null | grep -m 1 'FINISHED SETTING UP MINIO' || echo '')
+    if [ ! -z "${docker_finish_logs}" ]; then
+      break
+    fi
+    sleep 1
+  done
+
+
+  export S3_SMALL_CSV_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*phonenumbers\.csv' | grep -o 'http[s]\?://[^ ]\+')
+  echo $S3_SMALL_CSV_PRESIGNED_URL
+
+  export S3_SMALL_PARQUET_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*t1\.parquet' | grep -o 'http[s]\?://[^ ]\+')
+  echo $S3_SMALL_PARQUET_PRESIGNED_URL
+
+  export S3_LARGE_PARQUET_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*lineitem_large\.parquet' | grep -o 'http[s]\?://[^ ]\+')
+  echo $S3_LARGE_PARQUET_PRESIGNED_URL
+
+  export S3_ATTACH_DB_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*attach\.db' | grep -o 'http[s]\?://[^ ]\+')
+  echo $S3_ATTACH_DB_PRESIGNED_URL
+
+  export S3_ATTACH_DB="s3://test-bucket/presigned/attach.db"
+fi