Skip to content
Main

Add OPFS (Origin Private File System) Support #5905

Sign in to view logs

Add OPFS (Origin Private File System) Support

Add OPFS (Origin Private File System) Support #5905

Workflow file for this run

.github/workflows/main.yml at 426a1a4
name: 'Main'
on:
push:
pull_request:
create:
tags:
- v*
workflow_call:
inputs:
override_git_describe:
type: string
git_ref:
type: string
skip_tests:
type: string
workflow_dispatch:
inputs:
override_git_describe:
type: string
git_ref:
type: string
skip_tests:
type: string
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
cancel-in-progress: true
env:
EMSCRIPTEN_VERSION: 'latest'
jobs:
clang_format:
name: Clang-Format
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
- name: Lint ./lib
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
python3 ./scripts/run_clang_format.py \
--exclude ./lib/build \
--exclude ./lib/third_party \
-r ./lib/
eslint:
name: ESLint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
- name: Cache node_modules
uses: actions/cache@v4
with:
path: |
./node_modules
./packages/benchmarks/node_modules
./packages/duckdb-wasm/node_modules
./packages/duckdb-wasm-shell/node_modules
key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Prepare repository
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
yarn install --frozen-lockfile --prefer-offline
- name: Lint @duckdb/duckdb-wasm
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
yarn workspace @duckdb/duckdb-wasm run lint
- name: Lint @duckdb/duckdb-wasm-shell
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
yarn workspace @duckdb/duckdb-wasm-shell run lint
- name: Lint @duckdb/benchmarks
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
yarn workspace @duckdb/benchmarks run lint
tpchgen:
name: TPCH Generator
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
- name: Cache generator
uses: actions/cache@v4
id: cache-generator
with:
path: ./submodules/tpch-dbgen/dbgen/dbgen
key: ${{ runner.os }}-tpch-dbgen
- name: Build generator
if: steps.cache-generator.outputs.cache-hit != 'true'
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
make -C ./submodules/tpch-dbgen/dbgen/ dbgen
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: tpch-dbgen
path: |
./submodules/tpch-dbgen/dbgen/dbgen
retention-days: 1
dataprep:
name: Dataprep
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
- name: Cache rust build
uses: actions/cache@v4
with:
path: |
./.cargo/bin/
./.cargo/registry/index/
./.cargo/registry/cache/
./.cargo/git/db/
./target
key: ${{ runner.os }}-dataprep-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./tools/dataprep/src/*.rs') }}
restore-keys: |
${{ runner.os }}-dataprep-
- name: Build generator
shell: bash
run: |
cargo build --manifest-path=./Cargo.toml --release -p dataprep
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: dataprep
path: |
./target/release/dataprep
retention-days: 1
native_debug:
name: Native / Debug
runs-on: ubuntu-latest
if: false
needs:
- dataprep
- tpchgen
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0
- uses: actions/download-artifact@v4
with:
name: dataprep
path: ./target/release/
- uses: actions/download-artifact@v4
with:
name: tpch-dbgen
path: ./submodules/tpch-dbgen/dbgen/
- name: Git submodule status
run: |
git submodule status > git_submodule_status.txt
- name: Cache native build
uses: actions/cache@v4
with:
path: |
./.cargo/bin/
./.cargo/registry/index/
./.cargo/registry/cache/
./.cargo/git/db/
./.ccache
./.emscripten_cache
key: ${{ runner.os }}-native2-debug-${{ hashFiles('git_submodule_status.txt') }}-${{ hashFiles('lib/src/**') }}-${{ hashFiles('lib/include/**') }}
restore-keys: |
${{ runner.os }}-native2-debug-${{ hashFiles('git_submodule_status.txt') }}
${{ runner.os }}-native2-debug-
${{ runner.os }}-native2-
- name: Prepare repository
run: |
(cd ./submodules/duckdb && git fetch --all --tags)
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- name: Prepare environment
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
mkdir -p ./lib/build/debug ./reports
./scripts/generate_tpch_tbl.sh 0.01
./scripts/generate_tpch_arrow.sh 0.01
./scripts/generate_uni.sh
ccache --max-size 200M
ccache -s
- name: Build project
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
cmake \
-S./lib/ \
-B./lib/build/debug \
-DCODE_COVERAGE=1 \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER=clang++ \
-DCMAKE_BUILD_TYPE=Debug
ccache -z
make -C./lib/build/debug -j`nproc` || exit 1
ccache -s
- name: Test project
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
export LLVM_PROFILE_FILE=./reports/lib.profraw
./lib/build/debug/tester \
--source_dir=./lib/ \
--gtest_output=xml:./reports/tests_lib_debug.xml
- name: Code Coverage
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
llvm-profdata merge \
-sparse ./reports/lib.profraw \
-o ./reports/lib.profdata
llvm-cov show ./lib/build/debug/tester \
--instr-profile ./reports/lib.profdata \
-ignore-filename-regex submodules \
> ./reports/coverage_lib.txt
native_release:
name: Native / Release
runs-on: ubuntu-latest
if: false
needs:
- dataprep
- tpchgen
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0
- uses: actions/download-artifact@v4
with:
name: dataprep
path: ./target/release/
- uses: actions/download-artifact@v4
with:
name: tpch-dbgen
path: ./submodules/tpch-dbgen/dbgen/
- name: Git submodule status
run: |
git submodule status > git_submodule_status.txt
- name: Cache native build
uses: actions/cache@v4
with:
path: |
./.cargo/bin/
./.cargo/registry/index/
./.cargo/registry/cache/
./.cargo/git/db/
./.ccache
./.emscripten_cache
key: ${{ runner.os }}-native2-release-${{ hashFiles('git_submodule_status.txt') }}-${{ hashFiles('lib/src/**') }}-${{ hashFiles('lib/include/**') }}
restore-keys: |
${{ runner.os }}-native2-release-${{ hashFiles('git_submodule_status.txt') }}
${{ runner.os }}-native2-release-
${{ runner.os }}-native2-
- name: Prepare repository
run: |
(cd ./submodules/duckdb && git fetch --all --tags)
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- name: Prepare environment
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
mkdir -p ./lib/build/release ./reports
./scripts/generate_tpch_tbl.sh 0.01
./scripts/generate_tpch_arrow.sh 0.01
./scripts/generate_uni.sh
ccache --max-size 200M
ccache -s
- name: Build project
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
cmake \
-S./lib/ \
-B./lib/build/release \
-DCODE_COVERAGE=1 \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER=clang++ \
-DCMAKE_BUILD_TYPE=Release
ccache -z
make -C./lib/build/release -j`nproc` || exit 1
ccache -s
- name: Test project
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
export LLVM_PROFILE_FILE=./reports/lib.profraw
./lib/build/release/tester \
--source_dir=./lib/ \
--gtest_output=xml:./reports/tests_lib_release.xml
- name: Code Coverage
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
llvm-profdata merge \
-sparse ./reports/lib.profraw \
-o ./reports/lib.profdata
llvm-cov show ./lib/build/release/tester \
--instr-profile ./reports/lib.profdata \
-ignore-filename-regex submodules \
> ./reports/coverage_lib.txt
wasm_mvp:
name: Wasm / mvp
runs-on: ubuntu-latest
needs:
- dataprep
- tpchgen
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0
- name: Prepare repository
run: |
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
with:
version: ${{ env.EMSCRIPTEN_VERSION }}
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
- name: Git submodule status
run: |
git submodule status > git_submodule_status.txt
- name: Build Wasm module
run: |
DUCKDB_PLATFORM="wasm_mvp" ./scripts/wasm_build_lib.sh relperf mvp
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: wasm-mvp
path: |
./packages/duckdb-wasm/src/bindings/duckdb-mvp.js
./packages/duckdb-wasm/src/bindings/duckdb-mvp.wasm
retention-days: 1
wasm_eh:
name: Wasm / eh
runs-on: ubuntu-latest
needs:
- dataprep
- tpchgen
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0
- name: Prepare repository
run: |
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
with:
version: ${{ env.EMSCRIPTEN_VERSION }}
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
- name: Git submodule status
run: |
git submodule status > git_submodule_status.txt
- name: Build Wasm module
run: |
DUCKDB_PLATFORM="wasm_eh" ./scripts/wasm_build_lib.sh relperf eh
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: wasm-eh
path: |
./packages/duckdb-wasm/src/bindings/duckdb-eh.js
./packages/duckdb-wasm/src/bindings/duckdb-eh.wasm
retention-days: 1
wasm_coi:
name: Wasm / coi
runs-on: ubuntu-latest
needs:
- dataprep
- tpchgen
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0
- name: Prepare repository
run: |
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
with:
version: ${{ env.EMSCRIPTEN_VERSION }}
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
- name: Git submodule status
run: |
git submodule status > git_submodule_status.txt
- name: Build Wasm module
run: |
DUCKDB_PLATFORM="wasm_coi" ./scripts/wasm_build_lib.sh relperf coi
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: wasm-coi
path: |
./packages/duckdb-wasm/src/bindings/duckdb-coi.js
./packages/duckdb-wasm/src/bindings/duckdb-coi.pthread.js
./packages/duckdb-wasm/src/bindings/duckdb-coi.wasm
retention-days: 1
wasm_mvp_loadable:
name: Wasm / mvp (loadable version)
runs-on: ubuntu-latest
needs:
- dataprep
- tpchgen
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0
- name: Prepare repository
run: |
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
with:
version: ${{ env.EMSCRIPTEN_VERSION }}
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
- name: Git submodule status
run: |
git submodule status > git_submodule_status.txt
- name: Build Wasm module
run: |
DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: wasm-mvp-loadable
path: |
./packages/duckdb-wasm/src/bindings/duckdb-mvp.js
./packages/duckdb-wasm/src/bindings/duckdb-mvp.wasm
retention-days: 1
wasm_eh_loadable:
name: Wasm / eh (loadable version)
runs-on: ubuntu-latest
needs:
- dataprep
- tpchgen
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0
- name: Prepare repository
run: |
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
with:
version: ${{ env.EMSCRIPTEN_VERSION }}
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
- name: Git submodule status
run: |
git submodule status > git_submodule_status.txt
- name: Build Wasm module
run: |
DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: wasm-eh-loadable
path: |
./packages/duckdb-wasm/src/bindings/duckdb-eh.js
./packages/duckdb-wasm/src/bindings/duckdb-eh.wasm
retention-days: 1
wasm_coi_loadable:
name: Wasm / coi (loadable version)
runs-on: ubuntu-latest
needs:
- dataprep
- tpchgen
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0
- name: Prepare repository
run: |
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
with:
version: ${{ env.EMSCRIPTEN_VERSION }}
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
- name: Git submodule status
run: |
git submodule status > git_submodule_status.txt
- name: Build Wasm module
run: |
DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: wasm-coi-loadable
path: |
./packages/duckdb-wasm/src/bindings/duckdb-coi.js
./packages/duckdb-wasm/src/bindings/duckdb-coi.pthread.js
./packages/duckdb-wasm/src/bindings/duckdb-coi.wasm
retention-days: 1
js_libs:
name: Js / Libraries
runs-on: ubuntu-latest
needs:
- wasm_mvp
- wasm_eh
- wasm_coi
- clang_format
- eslint
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0
- uses: actions/setup-node@v4
with:
node-version: '18.x'
registry-url: 'https://registry.npmjs.org'
- name: Git submodule status
run: |
git submodule status > git_submodule_status.txt
- name: Cache rust build
uses: actions/cache@v4
with:
path: |
./.cargo/bin/
./.cargo/registry/index/
./.cargo/registry/cache/
./.cargo/git/db/
./target
key: ${{ runner.os }}-shell-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./packages/duckdb-wasm-shell/crate/src/**') }}
restore-keys: |
${{ runner.os }}-shell-
- name: Cache node_modules
uses: actions/cache@v4
with:
path: |
./node_modules
./packages/benchmarks/node_modules
./packages/duckdb-wasm/node_modules
./packages/duckdb-wasm-shell/node_modules
key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- uses: actions/download-artifact@v4
with:
name: dataprep
path: ./target/release/
- uses: actions/download-artifact@v4
with:
name: tpch-dbgen
path: ./submodules/tpch-dbgen/dbgen/
- uses: actions/download-artifact@v4
with:
name: wasm-mvp
path: ./packages/duckdb-wasm/src/bindings/
- uses: actions/download-artifact@v4
with:
name: wasm-eh
path: ./packages/duckdb-wasm/src/bindings/
- uses: actions/download-artifact@v4
with:
name: wasm-coi
path: ./packages/duckdb-wasm/src/bindings/
- name: Prepare repository
run: |
git fetch --tags --no-recurse-submodules -f
(cd ./submodules/duckdb && git fetch --all --tags)
- name: Prepare environment
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
git config --global --add safe.directory '*'
mkdir -p ./lib/build/wasm/release ./reports
yarn install --frozen-lockfile
./scripts/generate_tpch_tbl.sh 0.01
./scripts/generate_tpch_arrow.sh 0.01
./scripts/generate_tpch_duckdb.sh 0.01
./scripts/generate_tpch_sqlite.sh 0.01
./scripts/generate_uni.sh
./scripts/npm_version.sh
- name: Build @duckdb/duckdb-wasm
shell: bash
run: |
rm -rf ./packages/duckdb-wasm/dist/
yarn workspace @duckdb/duckdb-wasm build:release
yarn workspace @duckdb/duckdb-wasm docs
- name: Build @duckdb/duckdb-wasm-shell
shell: bash
run: |
rm -rf ./packages/duckdb-wasm-shell/dist/
yarn workspace @duckdb/duckdb-wasm-shell install:wasmpack
yarn workspace @duckdb/duckdb-wasm-shell build:release
- name: Build @duckdb/duckdb-wasm-app
shell: bash
run: |
rm -rf ./packages/duckdb-wasm-app/build/
yarn workspace @duckdb/duckdb-wasm-app build:release
- name: Test @duckdb/duckdb-wasm on Chrome
shell: bash
run: |
CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome
# - name: Test @duckdb/duckdb-wasm on Firefox
# uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
# with:
# script: |-
# yarn workspace @duckdb/duckdb-wasm test:firefox
- name: Test @duckdb/duckdb-wasm on Node.js
shell: bash
run: |
yarn workspace @duckdb/duckdb-wasm test:node
- name: Coverage @duckdb/duckdb-wasm
shell: bash
run: |
CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome:coverage
- name: Build examples
shell: bash
run: |
yarn install
yarn workspace @duckdb/duckdb-wasm-examples-bare-node test
yarn workspace @duckdb/duckdb-wasm-examples-bare-browser build
yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node build
yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node test
yarn workspace @duckdb/duckdb-wasm-examples-esbuild-browser build
- name: Package
shell: bash
run: zip -r duckdb-wasm-packages.zip ./packages
- uses: actions/upload-artifact@v4
with:
name: duckdb-wasm-packages
path: duckdb-wasm-packages.zip
- name: Build @duckdb/benchmarks
shell: bash
run: |
rm -r packages/duckdb-wasm/node_modules/*
yarn workspace @duckdb/benchmarks build
yarn workspace @duckdb/benchmarks bench:system:sort:int
yarn workspace @duckdb/benchmarks bench:system:join:2
yarn workspace @duckdb/benchmarks bench:system:join:3
yarn workspace @duckdb/benchmarks bench:system:regex
yarn workspace @duckdb/benchmarks bench:system:sum:int
yarn workspace @duckdb/benchmarks bench:system:sum:csv
- name: Upload reports
uses: actions/upload-artifact@v4
with:
name: reports_micro
path: |
./reports/benchmark_system_sort_int.json
./reports/benchmark_system_join_2.json
./reports/benchmark_system_join_3.json
./reports/benchmark_system_regex.json
./reports/benchmark_system_sum_csv.json
./reports/benchmark_system_sum_int.json
retention-days: 1
- name: Preparation TPCH 0.01
if: github.ref == 'refs/heads/main'
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
git config --global --add safe.directory '*'
mkdir -p ./lib/build/wasm/release ./reports
yarn install --frozen-lockfile
./scripts/generate_tpch_tbl.sh 0.01
./scripts/generate_tpch_arrow.sh 0.01
./scripts/generate_tpch_duckdb.sh 0.01
./scripts/generate_tpch_sqlite.sh 0.01
- name: Benchmarks TPCH 0.01
if: github.ref == 'refs/heads/main'
shell: bash
run: |
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.01
yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.01
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.01
yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.01
- name: Upload reports TPCH 0.01
if: github.ref == 'refs/heads/main'
uses: actions/upload-artifact@v4
with:
name: reports_tpch_0_01
path: |
./reports/benchmark_system_tpch_001_duckdb.json
./reports/benchmark_system_tpch_001_sqljs.json
./reports/benchmark_system_tpch_001_arquero.json
./reports/benchmark_system_tpch_001_lovefield.json
retention-days: 1
- name: Preparation TPCH 0.1
if: github.ref == 'refs/heads/main'
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
git config --global --add safe.directory '*'
mkdir -p ./lib/build/wasm/release ./reports
yarn install --frozen-lockfile
./scripts/generate_tpch_tbl.sh 0.1
./scripts/generate_tpch_arrow.sh 0.1
./scripts/generate_tpch_duckdb.sh 0.1
./scripts/generate_tpch_sqlite.sh 0.1
- name: Benchmarks TPCH 0.1
if: github.ref == 'refs/heads/main'
shell: bash
run: |
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.1
yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.1
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.1
yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.1
- name: Upload reports TPCH 0.1
if: github.ref == 'refs/heads/main'
uses: actions/upload-artifact@v4
with:
name: reports_tpch_0_1
path: |
./reports/benchmark_system_tpch_01_duckdb.json
./reports/benchmark_system_tpch_01_sqljs.json
./reports/benchmark_system_tpch_01_arquero.json
./reports/benchmark_system_tpch_01_lovefield.json
retention-days: 1
- name: Preparation TPCH 0.25
if: github.ref == 'refs/heads/main'
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
git config --global --add safe.directory '*'
mkdir -p ./lib/build/wasm/release ./reports
yarn install --frozen-lockfile
./scripts/generate_tpch_tbl.sh 0.25
./scripts/generate_tpch_arrow.sh 0.25
./scripts/generate_tpch_duckdb.sh 0.25
./scripts/generate_tpch_sqlite.sh 0.25
- name: Benchmarks TPCH 0.25
if: github.ref == 'refs/heads/main'
shell: bash
run: |
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.25
yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.25
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.25
yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.25
- name: Upload reports TPCH 0.25
if: github.ref == 'refs/heads/main'
uses: actions/upload-artifact@v4
with:
name: reports_tpch_0_25
path: |
./reports/benchmark_system_tpch_025_duckdb.json
./reports/benchmark_system_tpch_025_sqljs.json
./reports/benchmark_system_tpch_025_arquero.json
./reports/benchmark_system_tpch_025_lovefield.json
retention-days: 1
- name: Preparation TPCH 0.5
if: github.ref == 'refs/heads/main'
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
git config --global --add safe.directory '*'
mkdir -p ./lib/build/wasm/release ./reports
yarn install --frozen-lockfile
./scripts/generate_tpch_tbl.sh 0.5
./scripts/generate_tpch_arrow.sh 0.5
./scripts/generate_tpch_duckdb.sh 0.5
./scripts/generate_tpch_sqlite.sh 0.5
- name: Benchmarks TPCH 0.5
if: github.ref == 'refs/heads/main'
shell: bash
run: |
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.5
yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.5
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.5
yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.5
- name: Upload reports TPCH 0.5
if: github.ref == 'refs/heads/main'
uses: actions/upload-artifact@v4
with:
name: reports_tpch_0_5
path: |
./reports/benchmark_system_tpch_05_duckdb.json
./reports/benchmark_system_tpch_05_sqljs.json
./reports/benchmark_system_tpch_05_arquero.json
./reports/benchmark_system_tpch_05_lovefield.json
retention-days: 1
js_libs_loadable:
name: Js / Libraries (loadable version)
runs-on: ubuntu-latest
needs:
- wasm_mvp_loadable
- wasm_eh_loadable
- wasm_coi_loadable
- clang_format
- eslint
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0
- uses: actions/setup-node@v4
with:
node-version: '18.x'
registry-url: 'https://registry.npmjs.org'
- name: Git submodule status
run: |
git submodule status > git_submodule_status.txt
- name: Cache rust build
uses: actions/cache@v4
with:
path: |
./.cargo/bin/
./.cargo/registry/index/
./.cargo/registry/cache/
./.cargo/git/db/
./target
key: ${{ runner.os }}-shell-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./packages/duckdb-wasm-shell/crate/src/**') }}
restore-keys: |
${{ runner.os }}-shell-
- name: Cache node_modules
uses: actions/cache@v4
with:
path: |
./node_modules
./packages/benchmarks/node_modules
./packages/duckdb-wasm/node_modules
./packages/duckdb-wasm-shell/node_modules
key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- uses: actions/download-artifact@v4
with:
name: dataprep
path: ./target/release/
- uses: actions/download-artifact@v4
with:
name: tpch-dbgen
path: ./submodules/tpch-dbgen/dbgen/
- uses: actions/download-artifact@v4
with:
name: wasm-mvp-loadable
path: ./packages/duckdb-wasm/src/bindings/
- uses: actions/download-artifact@v4
with:
name: wasm-eh-loadable
path: ./packages/duckdb-wasm/src/bindings/
- uses: actions/download-artifact@v4
with:
name: wasm-coi-loadable
path: ./packages/duckdb-wasm/src/bindings/
- name: Prepare repository
run: |
git fetch --tags --no-recurse-submodules -f
(cd ./submodules/duckdb && git fetch --all --tags)
- name: Prepare environment
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
git config --global --add safe.directory '*'
mkdir -p ./lib/build/wasm/release ./reports
yarn install --frozen-lockfile
./scripts/generate_tpch_tbl.sh 0.01
./scripts/generate_tpch_arrow.sh 0.01
./scripts/generate_tpch_duckdb.sh 0.01
./scripts/generate_tpch_sqlite.sh 0.01
./scripts/generate_uni.sh
./scripts/npm_version.sh
- name: Build @duckdb/duckdb-wasm
shell: bash
run: |
rm -rf ./packages/duckdb-wasm/dist/
yarn workspace @duckdb/duckdb-wasm build:release
yarn workspace @duckdb/duckdb-wasm docs
- name: Build @duckdb/duckdb-wasm-shell
shell: bash
run: |
rm -rf ./packages/duckdb-wasm-shell/dist/
yarn workspace @duckdb/duckdb-wasm-shell install:wasmpack
yarn workspace @duckdb/duckdb-wasm-shell build:release
- name: Build @duckdb/duckdb-wasm-app
shell: bash
run: |
rm -rf ./packages/duckdb-wasm-app/build/
yarn workspace @duckdb/duckdb-wasm-app build:release
- name: Test @duckdb/duckdb-wasm on Chrome
shell: bash
run: |
CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome
# - name: Test @duckdb/duckdb-wasm on Firefox
# uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
# with:
# script: |-
# yarn workspace @duckdb/duckdb-wasm test:firefox
- name: Test @duckdb/duckdb-wasm on Node.js
shell: bash
run: |
yarn workspace @duckdb/duckdb-wasm test:node
#- name: Coverage @duckdb/duckdb-wasm
# shell: bash
# run: |
# CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome:coverage
- name: Package
shell: bash
run: zip -r duckdb-wasm-packages.zip ./packages
- uses: actions/upload-artifact@v4
with:
name: duckdb-wasm-packages-loadable
path: duckdb-wasm-packages.zip
- name: Measure NPM stats
run: |
./scripts/npm_measure_lib.sh || true
- name: Publish to NPM
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
TAG: ${{ startsWith(github.head_ref, 'v') && 'latest' || 'next' }}
if: env.NODE_AUTH_TOKEN != null
run: |
echo "${TAG}"
./scripts/npm_publish_lib.sh
./scripts/npm_publish_shell.sh
./scripts/npm_publish_react.sh
- name: Deploy GitHub Page
if: github.ref == 'refs/heads/main'
run: |
git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
git fetch origin gh-pages
./scripts/deploy_pages.sh
merge_reports:
name: Merge benchmark reports
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/main'
needs:
- js_libs
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
- uses: actions/download-artifact@v4
with:
name: dataprep
path: ./target/release/
- uses: actions/download-artifact@v4
with:
name: reports_micro
path: ./reports/
- uses: actions/download-artifact@v4
with:
name: reports_tpch_0_01
path: ./reports/
- uses: actions/download-artifact@v4
with:
name: reports_tpch_0_1
path: ./reports/
- uses: actions/download-artifact@v4
with:
name: reports_tpch_0_25
path: ./reports/
- uses: actions/download-artifact@v4
with:
name: reports_tpch_0_5
path: ./reports/
- name: Merge reports
shell: bash
run: |
mkdir duckdb
cd duckdb
wget https://github.com/duckdb/duckdb/releases/download/v0.8.1/duckdb_cli-linux-amd64.zip
unzip duckdb_cli-linux-amd64.zip
./duckdb -c "COPY (FROM read_json_auto('../reports/*.json', union_by_name=1)) TO 'benchmarks.json'"
./duckdb -c "COPY (FROM read_json_auto('../reports/*.json', union_by_name=1)) TO 'benchmarks.csv'"
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: reports_various
path: |
./reports/benchmarks.arrow
./duckdb/benchmarks.json
./duckdb/benchmarks.csv
retention-days: 1
- name: Merge benchmark reports
uses: duckdb/duckdb-wasm-ci-env@f10a0954b7b2b98e8526e0f7bbcb10e614defcbd
with:
script: |-
chmod +x ./target/release/dataprep
./target/release/dataprep merge-benchmarks -r ./reports/
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: reports_all
path: |
./reports/benchmarks.arrow
retention-days: 1
- name: Deploy benchmark reports
if: github.ref == 'refs/heads/main'
run: |
git config --global user.name 'github-actions[bot]'
git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com'
git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
git fetch origin gh-pages
./scripts/deploy_benchmarks.sh