main.yml

name: 'Main'
on:
    push:
      branches:
        - '**'
        - '!master'
      tags:
        - '**'
    pull_request:
    create:
        tags:
            - v*
    workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/master' || github.sha }}
  cancel-in-progress: true

jobs:
    clang_format:
        name: Clang-Format
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'

            - name: Lint ./lib
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      python3 ./scripts/run_clang_format.py \
                        --exclude ./lib/build \
                        --exclude ./lib/third_party \
                        -r ./lib/

    eslint:
        name: ESLint
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'

            - name: Cache node_modules
              uses: actions/cache@v3
              with:
                  path: |
                      ./node_modules
                      ./packages/benchmarks/node_modules
                      ./packages/duckdb-wasm/node_modules
                      ./packages/duckdb-wasm-shell/node_modules
                  key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }}
                  restore-keys: |
                      ${{ runner.os }}-yarn-

            - name: Prepare repository
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      yarn install --frozen-lockfile --prefer-offline

            - name: Lint @duckdb/duckdb-wasm
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      yarn workspace @duckdb/duckdb-wasm run lint

            - name: Lint @duckdb/duckdb-wasm-shell
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      yarn workspace @duckdb/duckdb-wasm-shell run lint

            - name: Lint @duckdb/benchmarks
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      yarn workspace @duckdb/benchmarks run lint

    tpchgen:
        name: TPCH Generator
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'

            - name: Cache generator
              uses: actions/cache@v3
              id: cache-generator
              with:
                  path: ./submodules/tpch-dbgen/dbgen/dbgen
                  key: ${{ runner.os }}-tpch-dbgen

            - name: Build generator
              if: steps.cache-generator.outputs.cache-hit != 'true'
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      make -C ./submodules/tpch-dbgen/dbgen/ dbgen

            - name: Upload artifact
              uses: actions/upload-artifact@v3
              with:
                  name: tpch-dbgen
                  path: |
                      ./submodules/tpch-dbgen/dbgen/dbgen
                  retention-days: 1

    dataprep:
        name: Dataprep
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'

            - name: Cache rust build
              uses: actions/cache@v3
              with:
                  path: |
                      ./.cargo/bin/
                      ./.cargo/registry/index/
                      ./.cargo/registry/cache/
                      ./.cargo/git/db/
                      ./target
                  key: ${{ runner.os }}-dataprep-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./tools/dataprep/src/*.rs') }}
                  restore-keys: |
                      ${{ runner.os }}-dataprep-

            - name: Build generator
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      cargo build --manifest-path=./Cargo.toml --release -p dataprep

            - name: Upload artifact
              uses: actions/upload-artifact@v3
              with:
                  name: dataprep
                  path: |
                      ./target/release/dataprep
                  retention-days: 1

    duckdb_shell:
        name: DuckDB Shell
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'

            - name: Git submodule status
              run: |
                  git submodule status ./submodules/duckdb > git_submodule_status.txt

            - name: Cache ccache
              uses: actions/cache@v3
              with:
                  path: |
                      ./.ccache
                  key: ${{ runner.os }}-duckdb-${{ hashFiles('git_submodule_status.txt') }}
                  restore-keys: |
                      ${{ runner.os }}-duckdb-

            - name: Build DuckDB shell
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      ccache -z
                      ./scripts/build_duckdb_shell.sh
                      ccache -s

            - name: Upload artifact
              uses: actions/upload-artifact@v3
              with:
                  name: duckdb-shell
                  path: ./submodules/duckdb/build/Release/duckdb
                  retention-days: 1

    native_debug:
        name: Native / Debug
        runs-on: ubuntu-latest
        needs:
            - dataprep
            - tpchgen
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'
                  fetch-depth: 0

            - uses: actions/download-artifact@v3
              with:
                  name: dataprep
                  path: ./target/release/

            - uses: actions/download-artifact@v3
              with:
                  name: tpch-dbgen
                  path: ./submodules/tpch-dbgen/dbgen/

            - name: Git submodule status
              run: |
                  git submodule status > git_submodule_status.txt

            - name: Cache native build
              uses: actions/cache@v3
              with:
                  path: |
                      ./.cargo/bin/
                      ./.cargo/registry/index/
                      ./.cargo/registry/cache/
                      ./.cargo/git/db/
                      ./.ccache
                      ./.emscripten_cache
                  key: ${{ runner.os }}-native2-debug-${{ hashFiles('git_submodule_status.txt') }}-${{ hashFiles('lib/src/**') }}-${{ hashFiles('lib/include/**') }}
                  restore-keys: |
                      ${{ runner.os }}-native2-debug-${{ hashFiles('git_submodule_status.txt') }}
                      ${{ runner.os }}-native2-debug-
                      ${{ runner.os }}-native2-

            - name: Prepare repository
              run: |
                  (cd ./submodules/duckdb && git fetch --all --tags)
                  [ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch || echo "No patching needed"

            - name: Prepare environment
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      mkdir -p ./lib/build/debug ./reports
                      ./scripts/generate_tpch_tbl.sh 0.01
                      ./scripts/generate_tpch_arrow.sh 0.01
                      ./scripts/generate_uni.sh
                      ccache --max-size 200M
                      ccache -s

            - name: Build project
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      cmake \
                          -S./lib/ \
                          -B./lib/build/debug \
                          -DCODE_COVERAGE=1 \
                          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
                          -DCMAKE_C_COMPILER=clang \
                          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                          -DCMAKE_CXX_COMPILER=clang++ \
                          -DCMAKE_BUILD_TYPE=Debug
                      ccache -z
                      make -C./lib/build/debug -j`nproc` || exit 1
                      ccache -s

            - name: Test project
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      export LLVM_PROFILE_FILE=./reports/lib.profraw
                      ./lib/build/debug/tester \
                          --source_dir=./lib/ \
                          --gtest_output=xml:./reports/tests_lib_debug.xml

            - name: Code Coverage
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      llvm-profdata merge \
                          -sparse ./reports/lib.profraw \
                          -o ./reports/lib.profdata
                      llvm-cov show ./lib/build/debug/tester \
                          --instr-profile ./reports/lib.profdata \
                          -ignore-filename-regex submodules \
                          > ./reports/coverage_lib.txt

    native_release:
        name: Native / Release
        runs-on: ubuntu-latest
        needs:
            - dataprep
            - tpchgen
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'
                  fetch-depth: 0

            - uses: actions/download-artifact@v3
              with:
                  name: dataprep
                  path: ./target/release/

            - uses: actions/download-artifact@v3
              with:
                  name: tpch-dbgen
                  path: ./submodules/tpch-dbgen/dbgen/

            - name: Git submodule status
              run: |
                  git submodule status > git_submodule_status.txt

            - name: Cache native build
              uses: actions/cache@v3
              with:
                  path: |
                      ./.cargo/bin/
                      ./.cargo/registry/index/
                      ./.cargo/registry/cache/
                      ./.cargo/git/db/
                      ./.ccache
                      ./.emscripten_cache
                  key: ${{ runner.os }}-native2-release-${{ hashFiles('git_submodule_status.txt') }}-${{ hashFiles('lib/src/**') }}-${{ hashFiles('lib/include/**') }}
                  restore-keys: |
                      ${{ runner.os }}-native2-release-${{ hashFiles('git_submodule_status.txt') }}
                      ${{ runner.os }}-native2-release-
                      ${{ runner.os }}-native2-

            - name: Prepare repository
              run: |
                  (cd ./submodules/duckdb && git fetch --all --tags)
                  [ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch || echo "No patching needed"

            - name: Prepare environment
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      mkdir -p ./lib/build/release ./reports
                      ./scripts/generate_tpch_tbl.sh 0.01
                      ./scripts/generate_tpch_arrow.sh 0.01
                      ./scripts/generate_uni.sh
                      ccache --max-size 200M
                      ccache -s

            - name: Build project
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      cmake \
                          -S./lib/ \
                          -B./lib/build/release \
                          -DCODE_COVERAGE=1 \
                          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
                          -DCMAKE_C_COMPILER=clang \
                          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                          -DCMAKE_CXX_COMPILER=clang++ \
                          -DCMAKE_BUILD_TYPE=Release
                      ccache -z
                      make -C./lib/build/release -j`nproc` || exit 1
                      ccache -s

            - name: Test project
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      export LLVM_PROFILE_FILE=./reports/lib.profraw
                      ./lib/build/release/tester \
                          --source_dir=./lib/ \
                          --gtest_output=xml:./reports/tests_lib_release.xml

            - name: Code Coverage
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      llvm-profdata merge \
                          -sparse ./reports/lib.profraw \
                          -o ./reports/lib.profdata
                      llvm-cov show ./lib/build/release/tester \
                          --instr-profile ./reports/lib.profdata \
                          -ignore-filename-regex submodules \
                          > ./reports/coverage_lib.txt

    wasm_mvp:
        name: Wasm / mvp
        runs-on: ubuntu-latest
        needs:
            - dataprep
            - tpchgen
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'
                  fetch-depth: 0

            - name: Prepare repository
              run: |
                  [ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch || echo "No patching needed"

            - uses: mymindstorm/setup-emsdk@v12
              with:
                  version: 'latest'

            - name: Setup Ccache
              uses: hendrikmuhs/ccache-action@main
              with:
                  key: ${{ github.job }}

            - name: Git submodule status
              run: |
                  git submodule status > git_submodule_status.txt

            - name: Build Wasm module
              run: |
                  ./scripts/wasm_build_lib.sh relperf mvp

            - name: Upload artifact
              uses: actions/upload-artifact@v3
              with:
                  name: wasm-mvp
                  path: |
                      ./packages/duckdb-wasm/src/bindings/duckdb-mvp.js
                      ./packages/duckdb-wasm/src/bindings/duckdb-mvp.wasm
                  retention-days: 1

    wasm_eh:
        name: Wasm / eh
        runs-on: ubuntu-latest
        needs:
            - dataprep
            - tpchgen
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'
                  fetch-depth: 0

            - name: Prepare repository
              run: |
                  [ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch || echo "No patching needed"

            - uses: mymindstorm/setup-emsdk@v12
              with:
                  version: 'latest'

            - name: Setup Ccache
              uses: hendrikmuhs/ccache-action@main
              with:
                  key: ${{ github.job }}

            - name: Git submodule status
              run: |
                  git submodule status > git_submodule_status.txt

            - name: Build Wasm module
              run: |
                  ./scripts/wasm_build_lib.sh relperf eh

            - name: Upload artifact
              uses: actions/upload-artifact@v3
              with:
                  name: wasm-eh
                  path: |
                      ./packages/duckdb-wasm/src/bindings/duckdb-eh.js
                      ./packages/duckdb-wasm/src/bindings/duckdb-eh.wasm
                  retention-days: 1

    wasm_coi:
        name: Wasm / coi
        runs-on: ubuntu-latest
        needs:
            - dataprep
            - tpchgen
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'
                  fetch-depth: 0

            - name: Prepare repository
              run: |
                  [ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch || echo "No patching needed"

            - uses: mymindstorm/setup-emsdk@v12
              with:
                  version: 'latest'

            - name: Setup Ccache
              uses: hendrikmuhs/ccache-action@main
              with:
                  key: ${{ github.job }}

            - name: Git submodule status
              run: |
                  git submodule status > git_submodule_status.txt

            - name: Build Wasm module
              run: |
                  ./scripts/wasm_build_lib.sh relperf coi

            - name: Upload artifact
              uses: actions/upload-artifact@v3
              with:
                  name: wasm-coi
                  path: |
                      ./packages/duckdb-wasm/src/bindings/duckdb-coi.js
                      ./packages/duckdb-wasm/src/bindings/duckdb-coi.pthread.js
                      ./packages/duckdb-wasm/src/bindings/duckdb-coi.wasm
                  retention-days: 1

    js_libs:
        name: Js / Libraries
        runs-on: ubuntu-latest
        needs:
            - wasm_mvp
            - wasm_eh
            - wasm_coi
            - duckdb_shell
            - native_debug
            - native_release
            - clang_format
            - eslint
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'
                  fetch-depth: 0

            - uses: actions/setup-node@v3
              with:
                  node-version: '18.x'
                  registry-url: 'https://registry.npmjs.org'

            - name: Git submodule status
              run: |
                  git submodule status > git_submodule_status.txt

            - name: Cache rust build
              uses: actions/cache@v3
              with:
                  path: |
                      ./.cargo/bin/
                      ./.cargo/registry/index/
                      ./.cargo/registry/cache/
                      ./.cargo/git/db/
                      ./target
                  key: ${{ runner.os }}-shell-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./packages/duckdb-wasm-shell/crate/src/**') }}
                  restore-keys: |
                      ${{ runner.os }}-shell-

            - name: Cache node_modules
              uses: actions/cache@v3
              with:
                  path: |
                      ./node_modules
                      ./packages/benchmarks/node_modules
                      ./packages/duckdb-wasm/node_modules
                      ./packages/duckdb-wasm-shell/node_modules
                  key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }}
                  restore-keys: |
                      ${{ runner.os }}-yarn-

            - uses: actions/download-artifact@v3
              with:
                  name: dataprep
                  path: ./target/release/

            - uses: actions/download-artifact@v3
              with:
                  name: tpch-dbgen
                  path: ./submodules/tpch-dbgen/dbgen/

            - uses: actions/download-artifact@v3
              with:
                  name: duckdb-shell
                  path: ./submodules/duckdb/build/Release/

            - uses: actions/download-artifact@v3
              with:
                  name: wasm-mvp
                  path: ./packages/duckdb-wasm/src/bindings/

            - uses: actions/download-artifact@v3
              with:
                  name: wasm-eh
                  path: ./packages/duckdb-wasm/src/bindings/

            - uses: actions/download-artifact@v3
              with:
                  name: wasm-coi
                  path: ./packages/duckdb-wasm/src/bindings/

            - name: Prepare repository
              run: |
                  git fetch --tags --no-recurse-submodules -f
                  (cd ./submodules/duckdb && git fetch --all --tags)

            - name: Prepare environment
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      git config --global --add safe.directory '*'
                      mkdir -p ./lib/build/wasm/release ./reports
                      yarn install --frozen-lockfile
                      ./scripts/generate_tpch_tbl.sh 0.01
                      ./scripts/generate_tpch_arrow.sh 0.01
                      ./scripts/generate_tpch_duckdb.sh 0.01
                      ./scripts/generate_tpch_sqlite.sh 0.01
                      ./scripts/generate_uni.sh
                      ./scripts/npm_version.sh

            - name: Build @duckdb/duckdb-wasm
              shell: bash
              run: |
                      rm -rf ./packages/duckdb-wasm/dist/
                      yarn workspace @duckdb/duckdb-wasm build:release
                      yarn workspace @duckdb/duckdb-wasm docs

            - name: Build @duckdb/duckdb-wasm-shell
              shell: bash
              run: |
                      rm -rf ./packages/duckdb-wasm-shell/dist/
                      yarn workspace @duckdb/duckdb-wasm-shell install:wasmpack
                      yarn workspace @duckdb/duckdb-wasm-shell build:release

            - name: Build @duckdb/duckdb-wasm-app
              shell: bash
              run: |
                      rm -rf ./packages/duckdb-wasm-app/build/
                      yarn workspace @duckdb/duckdb-wasm-app build:release

            - name: Test @duckdb/duckdb-wasm on Chrome
              shell: bash
              run: |
                      CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome
            
            #            - name: Test @duckdb/duckdb-wasm on Firefox
            #              uses: duckdb/duckdb-wasm-ci-env@v0.11
            #              with:
            #                  script: |-
            #                      yarn workspace @duckdb/duckdb-wasm test:firefox

            - name: Test @duckdb/duckdb-wasm on Node.js
              shell: bash
              run: |
                      yarn workspace @duckdb/duckdb-wasm test:node

            - name: Coverage @duckdb/duckdb-wasm
              shell: bash
              run: |
                       CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome:coverage

            - name: Build examples
              shell: bash
              run: |
                      yarn install
                      yarn workspace @duckdb/duckdb-wasm-examples-bare-node test
                      yarn workspace @duckdb/duckdb-wasm-examples-bare-browser build
                      yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node build
                      yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node test
                      yarn workspace @duckdb/duckdb-wasm-examples-esbuild-browser build

            - name: Package
              shell: bash
              run: zip -r duckdb-wasm-packages.zip ./packages

            - uses: actions/upload-artifact@v3
              with:
                  name: duckdb-wasm-packages
                  path: duckdb-wasm-packages.zip

            - name: Publish to NPM
              env:
                  NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
              if: env.NODE_AUTH_TOKEN != null && (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch')
              run: |
                  ./scripts/npm_publish_lib.sh
                  ./scripts/npm_publish_shell.sh
                  ./scripts/npm_publish_react.sh

            - name: Deploy GitHub Page
              if: github.ref == 'refs/heads/master'
              run: |
                  git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
                  git fetch origin gh-pages
                  ./scripts/deploy_pages.sh

            - name: Build @duckdb/benchmarks
              shell: bash
              run: |
                rm -r packages/duckdb-wasm/node_modules/*
                yarn workspace @duckdb/benchmarks build
                yarn workspace @duckdb/benchmarks bench:system:sort:int
                yarn workspace @duckdb/benchmarks bench:system:join:2
                yarn workspace @duckdb/benchmarks bench:system:join:3
                yarn workspace @duckdb/benchmarks bench:system:regex
                yarn workspace @duckdb/benchmarks bench:system:sum:int
                yarn workspace @duckdb/benchmarks bench:system:sum:csv

            - name: Upload reports
              uses: actions/upload-artifact@v3
              with:
                  name: reports_micro
                  path: |
                      ./reports/benchmark_system_sort_int.json
                      ./reports/benchmark_system_join_2.json
                      ./reports/benchmark_system_join_3.json
                      ./reports/benchmark_system_regex.json
                      ./reports/benchmark_system_sum_csv.json
                      ./reports/benchmark_system_sum_int.json
                  retention-days: 1

            - name: Preparation TPCH 0.01
              if: github.ref == 'refs/heads/master'
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      git config --global --add safe.directory '*'
                      mkdir -p ./lib/build/wasm/release ./reports
                      yarn install --frozen-lockfile
                      ./scripts/generate_tpch_tbl.sh 0.01
                      ./scripts/generate_tpch_arrow.sh 0.01
                      ./scripts/generate_tpch_duckdb.sh 0.01
                      ./scripts/generate_tpch_sqlite.sh 0.01

            - name: Benchmarks TPCH 0.01
              if: github.ref == 'refs/heads/master'
              shell: bash
              run: |
                yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.01
                yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.01
                yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.01
                yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.01

            - name: Upload reports TPCH 0.01
              if: github.ref == 'refs/heads/master'
              uses: actions/upload-artifact@v3
              with:
                  name: reports_tpch_0_01
                  path: |
                      ./reports/benchmark_system_tpch_001_duckdb.json
                      ./reports/benchmark_system_tpch_001_sqljs.json
                      ./reports/benchmark_system_tpch_001_arquero.json
                      ./reports/benchmark_system_tpch_001_lovefield.json
                  retention-days: 1

            - name: Preparation TPCH 0.1
              if: github.ref == 'refs/heads/master'
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      git config --global --add safe.directory '*'
                      mkdir -p ./lib/build/wasm/release ./reports
                      yarn install --frozen-lockfile
                      ./scripts/generate_tpch_tbl.sh 0.1
                      ./scripts/generate_tpch_arrow.sh 0.1
                      ./scripts/generate_tpch_duckdb.sh 0.1
                      ./scripts/generate_tpch_sqlite.sh 0.1

            - name: Benchmarks TPCH 0.1
              if: github.ref == 'refs/heads/master'
              shell: bash
              run: |
                yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.1
                yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.1
                yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.1
                yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.1

            - name: Upload reports TPCH 0.1
              if: github.ref == 'refs/heads/master'
              uses: actions/upload-artifact@v3
              with:
                  name: reports_tpch_0_1
                  path: |
                      ./reports/benchmark_system_tpch_01_duckdb.json
                      ./reports/benchmark_system_tpch_01_sqljs.json
                      ./reports/benchmark_system_tpch_01_arquero.json
                      ./reports/benchmark_system_tpch_01_lovefield.json
                  retention-days: 1

            - name: Preparation TPCH 0.25
              if: github.ref == 'refs/heads/master'
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      git config --global --add safe.directory '*'
                      mkdir -p ./lib/build/wasm/release ./reports
                      yarn install --frozen-lockfile
                      ./scripts/generate_tpch_tbl.sh 0.25
                      ./scripts/generate_tpch_arrow.sh 0.25
                      ./scripts/generate_tpch_duckdb.sh 0.25
                      ./scripts/generate_tpch_sqlite.sh 0.25

            - name: Benchmarks TPCH 0.25
              if: github.ref == 'refs/heads/master'
              shell: bash
              run: |
                yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.25
                yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.25
                yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.25
                yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.25

            - name: Upload reports TPCH 0.25
              if: github.ref == 'refs/heads/master'
              uses: actions/upload-artifact@v3
              with:
                  name: reports_tpch_0_25
                  path: |
                      ./reports/benchmark_system_tpch_025_duckdb.json
                      ./reports/benchmark_system_tpch_025_sqljs.json
                      ./reports/benchmark_system_tpch_025_arquero.json
                      ./reports/benchmark_system_tpch_025_lovefield.json
                  retention-days: 1

            - name: Preparation TPCH 0.5
              if: github.ref == 'refs/heads/master'
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      git config --global --add safe.directory '*'
                      mkdir -p ./lib/build/wasm/release ./reports
                      yarn install --frozen-lockfile
                      ./scripts/generate_tpch_tbl.sh 0.5
                      ./scripts/generate_tpch_arrow.sh 0.5
                      ./scripts/generate_tpch_duckdb.sh 0.5
                      ./scripts/generate_tpch_sqlite.sh 0.5

            - name: Benchmarks TPCH 0.5
              if: github.ref == 'refs/heads/master'
              shell: bash
              run: |
                yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.5
                yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.5
                yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.5
                yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.5

            - name: Upload reports TPCH 0.5
              if: github.ref == 'refs/heads/master'
              uses: actions/upload-artifact@v3
              with:
                  name: reports_tpch_0_5
                  path: |
                      ./reports/benchmark_system_tpch_05_duckdb.json
                      ./reports/benchmark_system_tpch_05_sqljs.json
                      ./reports/benchmark_system_tpch_05_arquero.json
                      ./reports/benchmark_system_tpch_05_lovefield.json
                  retention-days: 1

    merge_reports:
        name: Merge benchmark reports
        runs-on: ubuntu-latest
        if: github.ref == 'refs/heads/master'
        needs:
            - js_libs
        steps:
            - uses: actions/checkout@v3
              with:
                  submodules: 'recursive'

            - uses: actions/download-artifact@v3
              with:
                  name: dataprep
                  path: ./target/release/

            - uses: actions/download-artifact@v3
              with:
                  name: reports_micro
                  path: ./reports/

            - uses: actions/download-artifact@v3
              with:
                  name: reports_tpch_0_01
                  path: ./reports/

            - uses: actions/download-artifact@v3
              with:
                  name: reports_tpch_0_1
                  path: ./reports/

            - uses: actions/download-artifact@v3
              with:
                  name: reports_tpch_0_25
                  path: ./reports/

            - uses: actions/download-artifact@v3
              with:
                  name: reports_tpch_0_5
                  path: ./reports/

            - name: Merge reports
              shell: bash
              run: |
                mkdir duckdb
                cd duckdb
                wget https://github.com/duckdb/duckdb/releases/download/v0.8.1/duckdb_cli-linux-amd64.zip
                unzip duckdb_cli-linux-amd64.zip
                ./duckdb -c "COPY (FROM read_json_auto('../reports/*.json', union_by_name=1)) TO 'benchmarks.json'"
                ./duckdb -c "COPY (FROM read_json_auto('../reports/*.json', union_by_name=1)) TO 'benchmarks.csv'"

            - name: Upload artifact
              uses: actions/upload-artifact@v3
              with:
                  name: reports_various
                  path: |
                      ./reports/benchmarks.arrow
                      ./duckdb/benchmarks.json
                      ./duckdb/benchmarks.csv
                  retention-days: 1

            - name: Merge benchmark reports
              uses: duckdb/duckdb-wasm-ci-env@v0.11
              with:
                  script: |-
                      chmod +x ./target/release/dataprep
                      ./target/release/dataprep merge-benchmarks -r ./reports/

            - name: Upload artifact
              uses: actions/upload-artifact@v3
              with:
                  name: reports_all
                  path: |
                      ./reports/benchmarks.arrow
                  retention-days: 1

            - name: Deploy benchmark reports
              if: github.ref == 'refs/heads/master'
              run: |
                  git config --global user.name 'github-actions[bot]'
                  git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com'
                  git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
                  git fetch origin gh-pages
                  ./scripts/deploy_benchmarks.sh