Add OPFS (Origin Private File System) Support #5914
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: 'Main' | |
on: | |
push: | |
pull_request: | |
create: | |
tags: | |
- v* | |
workflow_call: | |
inputs: | |
override_git_describe: | |
type: string | |
git_ref: | |
type: string | |
skip_tests: | |
type: string | |
workflow_dispatch: | |
inputs: | |
override_git_describe: | |
type: string | |
git_ref: | |
type: string | |
skip_tests: | |
type: string | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} | |
cancel-in-progress: true | |
env: | |
EMSCRIPTEN_VERSION: 'latest' | |
jobs: | |
clang_format: | |
name: Clang-Format | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
- name: Lint ./lib | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
python3 ./scripts/run_clang_format.py \ | |
--exclude ./lib/build \ | |
--exclude ./lib/third_party \ | |
-r ./lib/ | |
eslint: | |
name: ESLint | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
- name: Cache node_modules | |
uses: actions/cache@v4 | |
with: | |
path: | | |
./node_modules | |
./packages/benchmarks/node_modules | |
./packages/duckdb-wasm/node_modules | |
./packages/duckdb-wasm-shell/node_modules | |
key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }} | |
restore-keys: | | |
${{ runner.os }}-yarn- | |
- name: Prepare repository | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
yarn install --frozen-lockfile --prefer-offline | |
- name: Lint @duckdb/duckdb-wasm | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
yarn workspace @duckdb/duckdb-wasm run lint | |
- name: Lint @duckdb/duckdb-wasm-shell | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
yarn workspace @duckdb/duckdb-wasm-shell run lint | |
- name: Lint @duckdb/benchmarks | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
yarn workspace @duckdb/benchmarks run lint | |
tpchgen: | |
name: TPCH Generator | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
- name: Cache generator | |
uses: actions/cache@v4 | |
id: cache-generator | |
with: | |
path: ./submodules/tpch-dbgen/dbgen/dbgen | |
key: ${{ runner.os }}-tpch-dbgen | |
- name: Build generator | |
if: steps.cache-generator.outputs.cache-hit != 'true' | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
make -C ./submodules/tpch-dbgen/dbgen/ dbgen | |
- name: Upload artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: tpch-dbgen | |
path: | | |
./submodules/tpch-dbgen/dbgen/dbgen | |
retention-days: 1 | |
dataprep: | |
name: Dataprep | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
- name: Cache rust build | |
uses: actions/cache@v4 | |
with: | |
path: | | |
./.cargo/bin/ | |
./.cargo/registry/index/ | |
./.cargo/registry/cache/ | |
./.cargo/git/db/ | |
./target | |
key: ${{ runner.os }}-dataprep-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./tools/dataprep/src/*.rs') }} | |
restore-keys: | | |
${{ runner.os }}-dataprep- | |
- name: Build generator | |
shell: bash | |
run: | | |
cargo build --manifest-path=./Cargo.toml --release -p dataprep | |
- name: Upload artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: dataprep | |
path: | | |
./target/release/dataprep | |
retention-days: 1 | |
native_debug: | |
name: Native / Debug | |
runs-on: ubuntu-latest | |
if: false | |
needs: | |
- dataprep | |
- tpchgen | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
fetch-depth: 0 | |
- uses: actions/download-artifact@v4 | |
with: | |
name: dataprep | |
path: ./target/release/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: tpch-dbgen | |
path: ./submodules/tpch-dbgen/dbgen/ | |
- name: Git submodule status | |
run: | | |
git submodule status > git_submodule_status.txt | |
- name: Cache native build | |
uses: actions/cache@v4 | |
with: | |
path: | | |
./.cargo/bin/ | |
./.cargo/registry/index/ | |
./.cargo/registry/cache/ | |
./.cargo/git/db/ | |
./.ccache | |
./.emscripten_cache | |
key: ${{ runner.os }}-native2-debug-${{ hashFiles('git_submodule_status.txt') }}-${{ hashFiles('lib/src/**') }}-${{ hashFiles('lib/include/**') }} | |
restore-keys: | | |
${{ runner.os }}-native2-debug-${{ hashFiles('git_submodule_status.txt') }} | |
${{ runner.os }}-native2-debug- | |
${{ runner.os }}-native2- | |
- name: Prepare repository | |
run: | | |
(cd ./submodules/duckdb && git fetch --all --tags) | |
make apply_patches | |
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }} | |
- name: Prepare environment | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
mkdir -p ./lib/build/debug ./reports | |
./scripts/generate_tpch_tbl.sh 0.01 | |
./scripts/generate_tpch_arrow.sh 0.01 | |
./scripts/generate_uni.sh | |
ccache --max-size 200M | |
ccache -s | |
- name: Build project | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
cmake \ | |
-S./lib/ \ | |
-B./lib/build/debug \ | |
-DCODE_COVERAGE=1 \ | |
-DCMAKE_C_COMPILER_LAUNCHER=ccache \ | |
-DCMAKE_C_COMPILER=clang \ | |
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
-DCMAKE_CXX_COMPILER=clang++ \ | |
-DCMAKE_BUILD_TYPE=Debug | |
ccache -z | |
make -C./lib/build/debug -j`nproc` || exit 1 | |
ccache -s | |
- name: Test project | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
export LLVM_PROFILE_FILE=./reports/lib.profraw | |
./lib/build/debug/tester \ | |
--source_dir=./lib/ \ | |
--gtest_output=xml:./reports/tests_lib_debug.xml | |
- name: Code Coverage | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
llvm-profdata merge \ | |
-sparse ./reports/lib.profraw \ | |
-o ./reports/lib.profdata | |
llvm-cov show ./lib/build/debug/tester \ | |
--instr-profile ./reports/lib.profdata \ | |
-ignore-filename-regex submodules \ | |
> ./reports/coverage_lib.txt | |
native_release: | |
name: Native / Release | |
runs-on: ubuntu-latest | |
if: false | |
needs: | |
- dataprep | |
- tpchgen | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
fetch-depth: 0 | |
- uses: actions/download-artifact@v4 | |
with: | |
name: dataprep | |
path: ./target/release/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: tpch-dbgen | |
path: ./submodules/tpch-dbgen/dbgen/ | |
- name: Git submodule status | |
run: | | |
git submodule status > git_submodule_status.txt | |
- name: Cache native build | |
uses: actions/cache@v4 | |
with: | |
path: | | |
./.cargo/bin/ | |
./.cargo/registry/index/ | |
./.cargo/registry/cache/ | |
./.cargo/git/db/ | |
./.ccache | |
./.emscripten_cache | |
key: ${{ runner.os }}-native2-release-${{ hashFiles('git_submodule_status.txt') }}-${{ hashFiles('lib/src/**') }}-${{ hashFiles('lib/include/**') }} | |
restore-keys: | | |
${{ runner.os }}-native2-release-${{ hashFiles('git_submodule_status.txt') }} | |
${{ runner.os }}-native2-release- | |
${{ runner.os }}-native2- | |
- name: Prepare repository | |
run: | | |
(cd ./submodules/duckdb && git fetch --all --tags) | |
make apply_patches | |
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }} | |
- name: Prepare environment | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
mkdir -p ./lib/build/release ./reports | |
./scripts/generate_tpch_tbl.sh 0.01 | |
./scripts/generate_tpch_arrow.sh 0.01 | |
./scripts/generate_uni.sh | |
ccache --max-size 200M | |
ccache -s | |
- name: Build project | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
cmake \ | |
-S./lib/ \ | |
-B./lib/build/release \ | |
-DCODE_COVERAGE=1 \ | |
-DCMAKE_C_COMPILER_LAUNCHER=ccache \ | |
-DCMAKE_C_COMPILER=clang \ | |
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
-DCMAKE_CXX_COMPILER=clang++ \ | |
-DCMAKE_BUILD_TYPE=Release | |
ccache -z | |
make -C./lib/build/release -j`nproc` || exit 1 | |
ccache -s | |
- name: Test project | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
export LLVM_PROFILE_FILE=./reports/lib.profraw | |
./lib/build/release/tester \ | |
--source_dir=./lib/ \ | |
--gtest_output=xml:./reports/tests_lib_release.xml | |
- name: Code Coverage | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
llvm-profdata merge \ | |
-sparse ./reports/lib.profraw \ | |
-o ./reports/lib.profdata | |
llvm-cov show ./lib/build/release/tester \ | |
--instr-profile ./reports/lib.profdata \ | |
-ignore-filename-regex submodules \ | |
> ./reports/coverage_lib.txt | |
wasm_mvp: | |
name: Wasm / mvp | |
runs-on: ubuntu-latest | |
needs: | |
- dataprep | |
- tpchgen | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
fetch-depth: 0 | |
- name: Prepare repository | |
run: | | |
make apply_patches | |
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }} | |
- uses: mymindstorm/setup-emsdk@v14 | |
with: | |
version: ${{ env.EMSCRIPTEN_VERSION }} | |
- name: Setup Ccache | |
uses: hendrikmuhs/ccache-action@main | |
with: | |
key: ${{ github.job }} | |
- name: Git submodule status | |
run: | | |
git submodule status > git_submodule_status.txt | |
- name: Build Wasm module | |
run: | | |
DUCKDB_PLATFORM="wasm_mvp" ./scripts/wasm_build_lib.sh relperf mvp | |
- name: Upload artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: wasm-mvp | |
path: | | |
./packages/duckdb-wasm/src/bindings/duckdb-mvp.js | |
./packages/duckdb-wasm/src/bindings/duckdb-mvp.wasm | |
retention-days: 1 | |
wasm_eh: | |
name: Wasm / eh | |
runs-on: ubuntu-latest | |
needs: | |
- dataprep | |
- tpchgen | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
fetch-depth: 0 | |
- name: Prepare repository | |
run: | | |
make apply_patches | |
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }} | |
- uses: mymindstorm/setup-emsdk@v14 | |
with: | |
version: ${{ env.EMSCRIPTEN_VERSION }} | |
- name: Setup Ccache | |
uses: hendrikmuhs/ccache-action@main | |
with: | |
key: ${{ github.job }} | |
- name: Git submodule status | |
run: | | |
git submodule status > git_submodule_status.txt | |
- name: Build Wasm module | |
run: | | |
DUCKDB_PLATFORM="wasm_eh" ./scripts/wasm_build_lib.sh relperf eh | |
- name: Upload artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: wasm-eh | |
path: | | |
./packages/duckdb-wasm/src/bindings/duckdb-eh.js | |
./packages/duckdb-wasm/src/bindings/duckdb-eh.wasm | |
retention-days: 1 | |
wasm_coi: | |
name: Wasm / coi | |
runs-on: ubuntu-latest | |
needs: | |
- dataprep | |
- tpchgen | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
fetch-depth: 0 | |
- name: Prepare repository | |
run: | | |
make apply_patches | |
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }} | |
- uses: mymindstorm/setup-emsdk@v14 | |
with: | |
version: ${{ env.EMSCRIPTEN_VERSION }} | |
- name: Setup Ccache | |
uses: hendrikmuhs/ccache-action@main | |
with: | |
key: ${{ github.job }} | |
- name: Git submodule status | |
run: | | |
git submodule status > git_submodule_status.txt | |
- name: Build Wasm module | |
run: | | |
DUCKDB_PLATFORM="wasm_coi" ./scripts/wasm_build_lib.sh relperf coi | |
- name: Upload artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: wasm-coi | |
path: | | |
./packages/duckdb-wasm/src/bindings/duckdb-coi.js | |
./packages/duckdb-wasm/src/bindings/duckdb-coi.pthread.js | |
./packages/duckdb-wasm/src/bindings/duckdb-coi.wasm | |
retention-days: 1 | |
wasm_mvp_loadable: | |
name: Wasm / mvp (loadable version) | |
runs-on: ubuntu-latest | |
needs: | |
- dataprep | |
- tpchgen | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
fetch-depth: 0 | |
- name: Prepare repository | |
run: | | |
make apply_patches | |
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }} | |
- uses: mymindstorm/setup-emsdk@v14 | |
with: | |
version: ${{ env.EMSCRIPTEN_VERSION }} | |
- name: Setup Ccache | |
uses: hendrikmuhs/ccache-action@main | |
with: | |
key: ${{ github.job }} | |
- name: Git submodule status | |
run: | | |
git submodule status > git_submodule_status.txt | |
- name: Build Wasm module | |
run: | | |
DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp | |
- name: Upload artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: wasm-mvp-loadable | |
path: | | |
./packages/duckdb-wasm/src/bindings/duckdb-mvp.js | |
./packages/duckdb-wasm/src/bindings/duckdb-mvp.wasm | |
retention-days: 1 | |
wasm_eh_loadable: | |
name: Wasm / eh (loadable version) | |
runs-on: ubuntu-latest | |
needs: | |
- dataprep | |
- tpchgen | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
fetch-depth: 0 | |
- name: Prepare repository | |
run: | | |
make apply_patches | |
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }} | |
- uses: mymindstorm/setup-emsdk@v14 | |
with: | |
version: ${{ env.EMSCRIPTEN_VERSION }} | |
- name: Setup Ccache | |
uses: hendrikmuhs/ccache-action@main | |
with: | |
key: ${{ github.job }} | |
- name: Git submodule status | |
run: | | |
git submodule status > git_submodule_status.txt | |
- name: Build Wasm module | |
run: | | |
DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh | |
- name: Upload artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: wasm-eh-loadable | |
path: | | |
./packages/duckdb-wasm/src/bindings/duckdb-eh.js | |
./packages/duckdb-wasm/src/bindings/duckdb-eh.wasm | |
retention-days: 1 | |
wasm_coi_loadable: | |
name: Wasm / coi (loadable version) | |
runs-on: ubuntu-latest | |
needs: | |
- dataprep | |
- tpchgen | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
fetch-depth: 0 | |
- name: Prepare repository | |
run: | | |
make apply_patches | |
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }} | |
- uses: mymindstorm/setup-emsdk@v14 | |
with: | |
version: ${{ env.EMSCRIPTEN_VERSION }} | |
- name: Setup Ccache | |
uses: hendrikmuhs/ccache-action@main | |
with: | |
key: ${{ github.job }} | |
- name: Git submodule status | |
run: | | |
git submodule status > git_submodule_status.txt | |
- name: Build Wasm module | |
run: | | |
DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi | |
- name: Upload artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: wasm-coi-loadable | |
path: | | |
./packages/duckdb-wasm/src/bindings/duckdb-coi.js | |
./packages/duckdb-wasm/src/bindings/duckdb-coi.pthread.js | |
./packages/duckdb-wasm/src/bindings/duckdb-coi.wasm | |
retention-days: 1 | |
js_libs: | |
name: Js / Libraries | |
runs-on: ubuntu-latest | |
needs: | |
- wasm_mvp | |
- wasm_eh | |
- wasm_coi | |
- clang_format | |
- eslint | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
fetch-depth: 0 | |
- uses: actions/setup-node@v4 | |
with: | |
node-version: '18.x' | |
registry-url: 'https://registry.npmjs.org' | |
- name: Git submodule status | |
run: | | |
git submodule status > git_submodule_status.txt | |
- name: Cache rust build | |
uses: actions/cache@v4 | |
with: | |
path: | | |
./.cargo/bin/ | |
./.cargo/registry/index/ | |
./.cargo/registry/cache/ | |
./.cargo/git/db/ | |
./target | |
key: ${{ runner.os }}-shell-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./packages/duckdb-wasm-shell/crate/src/**') }} | |
restore-keys: | | |
${{ runner.os }}-shell- | |
- name: Cache node_modules | |
uses: actions/cache@v4 | |
with: | |
path: | | |
./node_modules | |
./packages/benchmarks/node_modules | |
./packages/duckdb-wasm/node_modules | |
./packages/duckdb-wasm-shell/node_modules | |
key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }} | |
restore-keys: | | |
${{ runner.os }}-yarn- | |
- uses: actions/download-artifact@v4 | |
with: | |
name: dataprep | |
path: ./target/release/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: tpch-dbgen | |
path: ./submodules/tpch-dbgen/dbgen/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: wasm-mvp | |
path: ./packages/duckdb-wasm/src/bindings/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: wasm-eh | |
path: ./packages/duckdb-wasm/src/bindings/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: wasm-coi | |
path: ./packages/duckdb-wasm/src/bindings/ | |
- name: Prepare repository | |
run: | | |
git fetch --tags --no-recurse-submodules -f | |
(cd ./submodules/duckdb && git fetch --all --tags) | |
- name: Prepare environment | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
git config --global --add safe.directory '*' | |
mkdir -p ./lib/build/wasm/release ./reports | |
yarn install --frozen-lockfile | |
./scripts/generate_tpch_tbl.sh 0.01 | |
./scripts/generate_tpch_arrow.sh 0.01 | |
./scripts/generate_tpch_duckdb.sh 0.01 | |
./scripts/generate_tpch_sqlite.sh 0.01 | |
./scripts/generate_uni.sh | |
./scripts/npm_version.sh | |
- name: Build @duckdb/duckdb-wasm | |
shell: bash | |
run: | | |
rm -rf ./packages/duckdb-wasm/dist/ | |
yarn workspace @duckdb/duckdb-wasm build:release | |
yarn workspace @duckdb/duckdb-wasm docs | |
- name: Build @duckdb/duckdb-wasm-shell | |
shell: bash | |
run: | | |
rm -rf ./packages/duckdb-wasm-shell/dist/ | |
yarn workspace @duckdb/duckdb-wasm-shell install:wasmpack | |
yarn workspace @duckdb/duckdb-wasm-shell build:release | |
- name: Build @duckdb/duckdb-wasm-app | |
shell: bash | |
run: | | |
rm -rf ./packages/duckdb-wasm-app/build/ | |
yarn workspace @duckdb/duckdb-wasm-app build:release | |
- name: Test @duckdb/duckdb-wasm on Chrome | |
shell: bash | |
run: | | |
CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome | |
# - name: Test @duckdb/duckdb-wasm on Firefox | |
# uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
# with: | |
# script: |- | |
# yarn workspace @duckdb/duckdb-wasm test:firefox | |
- name: Test @duckdb/duckdb-wasm on Node.js | |
shell: bash | |
run: | | |
yarn workspace @duckdb/duckdb-wasm test:node | |
- name: Coverage @duckdb/duckdb-wasm | |
shell: bash | |
run: | | |
CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome:coverage | |
- name: Build examples | |
shell: bash | |
run: | | |
yarn install | |
yarn workspace @duckdb/duckdb-wasm-examples-bare-node test | |
yarn workspace @duckdb/duckdb-wasm-examples-bare-browser build | |
yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node build | |
yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node test | |
yarn workspace @duckdb/duckdb-wasm-examples-esbuild-browser build | |
- name: Package | |
shell: bash | |
run: zip -r duckdb-wasm-packages.zip ./packages | |
- uses: actions/upload-artifact@v4 | |
with: | |
name: duckdb-wasm-packages | |
path: duckdb-wasm-packages.zip | |
- name: Build @duckdb/benchmarks | |
shell: bash | |
run: | | |
rm -r packages/duckdb-wasm/node_modules/* | |
yarn workspace @duckdb/benchmarks build | |
yarn workspace @duckdb/benchmarks bench:system:sort:int | |
yarn workspace @duckdb/benchmarks bench:system:join:2 | |
yarn workspace @duckdb/benchmarks bench:system:join:3 | |
yarn workspace @duckdb/benchmarks bench:system:regex | |
yarn workspace @duckdb/benchmarks bench:system:sum:int | |
yarn workspace @duckdb/benchmarks bench:system:sum:csv | |
- name: Upload reports | |
uses: actions/upload-artifact@v4 | |
with: | |
name: reports_micro | |
path: | | |
./reports/benchmark_system_sort_int.json | |
./reports/benchmark_system_join_2.json | |
./reports/benchmark_system_join_3.json | |
./reports/benchmark_system_regex.json | |
./reports/benchmark_system_sum_csv.json | |
./reports/benchmark_system_sum_int.json | |
retention-days: 1 | |
- name: Preparation TPCH 0.01 | |
if: github.ref == 'refs/heads/main' | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
git config --global --add safe.directory '*' | |
mkdir -p ./lib/build/wasm/release ./reports | |
yarn install --frozen-lockfile | |
./scripts/generate_tpch_tbl.sh 0.01 | |
./scripts/generate_tpch_arrow.sh 0.01 | |
./scripts/generate_tpch_duckdb.sh 0.01 | |
./scripts/generate_tpch_sqlite.sh 0.01 | |
- name: Benchmarks TPCH 0.01 | |
if: github.ref == 'refs/heads/main' | |
shell: bash | |
run: | | |
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.01 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.01 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.01 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.01 | |
- name: Upload reports TPCH 0.01 | |
if: github.ref == 'refs/heads/main' | |
uses: actions/upload-artifact@v4 | |
with: | |
name: reports_tpch_0_01 | |
path: | | |
./reports/benchmark_system_tpch_001_duckdb.json | |
./reports/benchmark_system_tpch_001_sqljs.json | |
./reports/benchmark_system_tpch_001_arquero.json | |
./reports/benchmark_system_tpch_001_lovefield.json | |
retention-days: 1 | |
- name: Preparation TPCH 0.1 | |
if: github.ref == 'refs/heads/main' | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
git config --global --add safe.directory '*' | |
mkdir -p ./lib/build/wasm/release ./reports | |
yarn install --frozen-lockfile | |
./scripts/generate_tpch_tbl.sh 0.1 | |
./scripts/generate_tpch_arrow.sh 0.1 | |
./scripts/generate_tpch_duckdb.sh 0.1 | |
./scripts/generate_tpch_sqlite.sh 0.1 | |
- name: Benchmarks TPCH 0.1 | |
if: github.ref == 'refs/heads/main' | |
shell: bash | |
run: | | |
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.1 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.1 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.1 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.1 | |
- name: Upload reports TPCH 0.1 | |
if: github.ref == 'refs/heads/main' | |
uses: actions/upload-artifact@v4 | |
with: | |
name: reports_tpch_0_1 | |
path: | | |
./reports/benchmark_system_tpch_01_duckdb.json | |
./reports/benchmark_system_tpch_01_sqljs.json | |
./reports/benchmark_system_tpch_01_arquero.json | |
./reports/benchmark_system_tpch_01_lovefield.json | |
retention-days: 1 | |
- name: Preparation TPCH 0.25 | |
if: github.ref == 'refs/heads/main' | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
git config --global --add safe.directory '*' | |
mkdir -p ./lib/build/wasm/release ./reports | |
yarn install --frozen-lockfile | |
./scripts/generate_tpch_tbl.sh 0.25 | |
./scripts/generate_tpch_arrow.sh 0.25 | |
./scripts/generate_tpch_duckdb.sh 0.25 | |
./scripts/generate_tpch_sqlite.sh 0.25 | |
- name: Benchmarks TPCH 0.25 | |
if: github.ref == 'refs/heads/main' | |
shell: bash | |
run: | | |
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.25 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.25 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.25 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.25 | |
- name: Upload reports TPCH 0.25 | |
if: github.ref == 'refs/heads/main' | |
uses: actions/upload-artifact@v4 | |
with: | |
name: reports_tpch_0_25 | |
path: | | |
./reports/benchmark_system_tpch_025_duckdb.json | |
./reports/benchmark_system_tpch_025_sqljs.json | |
./reports/benchmark_system_tpch_025_arquero.json | |
./reports/benchmark_system_tpch_025_lovefield.json | |
retention-days: 1 | |
- name: Preparation TPCH 0.5 | |
if: github.ref == 'refs/heads/main' | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
git config --global --add safe.directory '*' | |
mkdir -p ./lib/build/wasm/release ./reports | |
yarn install --frozen-lockfile | |
./scripts/generate_tpch_tbl.sh 0.5 | |
./scripts/generate_tpch_arrow.sh 0.5 | |
./scripts/generate_tpch_duckdb.sh 0.5 | |
./scripts/generate_tpch_sqlite.sh 0.5 | |
- name: Benchmarks TPCH 0.5 | |
if: github.ref == 'refs/heads/main' | |
shell: bash | |
run: | | |
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.5 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.5 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.5 | |
yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.5 | |
- name: Upload reports TPCH 0.5 | |
if: github.ref == 'refs/heads/main' | |
uses: actions/upload-artifact@v4 | |
with: | |
name: reports_tpch_0_5 | |
path: | | |
./reports/benchmark_system_tpch_05_duckdb.json | |
./reports/benchmark_system_tpch_05_sqljs.json | |
./reports/benchmark_system_tpch_05_arquero.json | |
./reports/benchmark_system_tpch_05_lovefield.json | |
retention-days: 1 | |
js_libs_loadable: | |
name: Js / Libraries (loadable version) | |
runs-on: ubuntu-latest | |
needs: | |
- wasm_mvp_loadable | |
- wasm_eh_loadable | |
- wasm_coi_loadable | |
- clang_format | |
- eslint | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
fetch-depth: 0 | |
- uses: actions/setup-node@v4 | |
with: | |
node-version: '18.x' | |
registry-url: 'https://registry.npmjs.org' | |
- name: Git submodule status | |
run: | | |
git submodule status > git_submodule_status.txt | |
- name: Cache rust build | |
uses: actions/cache@v4 | |
with: | |
path: | | |
./.cargo/bin/ | |
./.cargo/registry/index/ | |
./.cargo/registry/cache/ | |
./.cargo/git/db/ | |
./target | |
key: ${{ runner.os }}-shell-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./packages/duckdb-wasm-shell/crate/src/**') }} | |
restore-keys: | | |
${{ runner.os }}-shell- | |
- name: Cache node_modules | |
uses: actions/cache@v4 | |
with: | |
path: | | |
./node_modules | |
./packages/benchmarks/node_modules | |
./packages/duckdb-wasm/node_modules | |
./packages/duckdb-wasm-shell/node_modules | |
key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }} | |
restore-keys: | | |
${{ runner.os }}-yarn- | |
- uses: actions/download-artifact@v4 | |
with: | |
name: dataprep | |
path: ./target/release/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: tpch-dbgen | |
path: ./submodules/tpch-dbgen/dbgen/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: wasm-mvp-loadable | |
path: ./packages/duckdb-wasm/src/bindings/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: wasm-eh-loadable | |
path: ./packages/duckdb-wasm/src/bindings/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: wasm-coi-loadable | |
path: ./packages/duckdb-wasm/src/bindings/ | |
- name: Prepare repository | |
run: | | |
git fetch --tags --no-recurse-submodules -f | |
(cd ./submodules/duckdb && git fetch --all --tags) | |
- name: Prepare environment | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
git config --global --add safe.directory '*' | |
mkdir -p ./lib/build/wasm/release ./reports | |
yarn install --frozen-lockfile | |
./scripts/generate_tpch_tbl.sh 0.01 | |
./scripts/generate_tpch_arrow.sh 0.01 | |
./scripts/generate_tpch_duckdb.sh 0.01 | |
./scripts/generate_tpch_sqlite.sh 0.01 | |
./scripts/generate_uni.sh | |
./scripts/npm_version.sh | |
- name: Build @duckdb/duckdb-wasm | |
shell: bash | |
run: | | |
rm -rf ./packages/duckdb-wasm/dist/ | |
yarn workspace @duckdb/duckdb-wasm build:release | |
yarn workspace @duckdb/duckdb-wasm docs | |
- name: Build @duckdb/duckdb-wasm-shell | |
shell: bash | |
run: | | |
rm -rf ./packages/duckdb-wasm-shell/dist/ | |
yarn workspace @duckdb/duckdb-wasm-shell install:wasmpack | |
yarn workspace @duckdb/duckdb-wasm-shell build:release | |
- name: Build @duckdb/duckdb-wasm-app | |
shell: bash | |
run: | | |
rm -rf ./packages/duckdb-wasm-app/build/ | |
yarn workspace @duckdb/duckdb-wasm-app build:release | |
- name: Test @duckdb/duckdb-wasm on Chrome | |
shell: bash | |
run: | | |
CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome | |
# - name: Test @duckdb/duckdb-wasm on Firefox | |
# uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
# with: | |
# script: |- | |
# yarn workspace @duckdb/duckdb-wasm test:firefox | |
- name: Test @duckdb/duckdb-wasm on Node.js | |
shell: bash | |
run: | | |
yarn workspace @duckdb/duckdb-wasm test:node | |
#- name: Coverage @duckdb/duckdb-wasm | |
# shell: bash | |
# run: | | |
# CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome:coverage | |
- name: Package | |
shell: bash | |
run: zip -r duckdb-wasm-packages.zip ./packages | |
- uses: actions/upload-artifact@v4 | |
with: | |
name: duckdb-wasm-packages-loadable | |
path: duckdb-wasm-packages.zip | |
- name: Measure NPM stats | |
run: | | |
./scripts/npm_measure_lib.sh || true | |
- name: Deploy GitHub Page | |
if: github.ref == 'refs/heads/main' | |
run: | | |
git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY | |
git fetch origin gh-pages | |
./scripts/deploy_pages.sh | |
- name: Publish to NPM | |
env: | |
NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} | |
TAG: ${{ startsWith(github.head_ref, 'v') && 'latest' || 'next' }} | |
if: env.NODE_AUTH_TOKEN != null | |
run: | | |
echo "${TAG}" | |
./scripts/npm_publish_lib.sh | |
./scripts/npm_publish_shell.sh | |
./scripts/npm_publish_react.sh | |
merge_reports: | |
name: Merge benchmark reports | |
runs-on: ubuntu-latest | |
if: github.ref == 'refs/heads/main' | |
needs: | |
- js_libs | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
submodules: 'recursive' | |
- uses: actions/download-artifact@v4 | |
with: | |
name: dataprep | |
path: ./target/release/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: reports_micro | |
path: ./reports/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: reports_tpch_0_01 | |
path: ./reports/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: reports_tpch_0_1 | |
path: ./reports/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: reports_tpch_0_25 | |
path: ./reports/ | |
- uses: actions/download-artifact@v4 | |
with: | |
name: reports_tpch_0_5 | |
path: ./reports/ | |
- name: Merge reports | |
shell: bash | |
run: | | |
mkdir duckdb | |
cd duckdb | |
wget https://github.com/duckdb/duckdb/releases/download/v0.8.1/duckdb_cli-linux-amd64.zip | |
unzip duckdb_cli-linux-amd64.zip | |
./duckdb -c "COPY (FROM read_json_auto('../reports/*.json', union_by_name=1)) TO 'benchmarks.json'" | |
./duckdb -c "COPY (FROM read_json_auto('../reports/*.json', union_by_name=1)) TO 'benchmarks.csv'" | |
- name: Upload artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: reports_various | |
path: | | |
./reports/benchmarks.arrow | |
./duckdb/benchmarks.json | |
./duckdb/benchmarks.csv | |
retention-days: 1 | |
- name: Merge benchmark reports | |
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd | |
with: | |
script: |- | |
chmod +x ./target/release/dataprep | |
./target/release/dataprep merge-benchmarks -r ./reports/ | |
- name: Upload artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: reports_all | |
path: | | |
./reports/benchmarks.arrow | |
retention-days: 1 | |
- name: Deploy benchmark reports | |
if: github.ref == 'refs/heads/main' | |
run: | | |
git config --global user.name 'github-actions[bot]' | |
git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com' | |
git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY | |
git fetch origin gh-pages | |
./scripts/deploy_benchmarks.sh |