From 07e3f321a1466d02c3cee3f4dd9cec8d9333b2ee Mon Sep 17 00:00:00 2001 From: richarddavison <89518095+richarddavison@users.noreply.github.com> Date: Sun, 14 Jan 2024 23:05:32 +0100 Subject: [PATCH] Update CI (#68) Update CI --- .github/workflows/ci.yml | 40 +++++++++++--------- .github/workflows/release.yml | 71 +++++++++++++++++++++++------------ Makefile | 40 +++++++++++--------- 3 files changed, 92 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a942a271b9..1ba9e8676e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,8 +41,16 @@ jobs: include: - os: ubuntu-latest platform: linux + arch: x86_64 + - os: ubuntu-latest + platform: linux + arch: aarch64 + - os: macos-latest + platform: darwin + arch: x86_64 - os: macos-latest platform: darwin + arch: aarch64 # - os: windows-latest # platform: windows runs-on: ${{ matrix.os }} @@ -55,44 +63,40 @@ jobs: with: cache: yarn node-version: lts/* - - name: Install linux dependencies - if: contains(matrix.platform, 'linux') + - name: Install Linux dependencies + if: matrix.platform == 'linux' run: | sudo apt-get -y update sudo apt-get -y install make nodejs sudo snap install zig --classic --beta - - name: Setup Rust - uses: dtolnay/rust-toolchain@v1 - with: - toolchain: nightly - name: Install MacOS dependencies - if: contains(matrix.platform, 'darwin') + if: matrix.platform == 'darwin' env: HOMEBREW_NO_AUTO_UPDATE: 1 run: | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" brew install zig make - name: Install Windows dependencies - if: contains(matrix.platform, 'windows') + if: matrix.platform == 'windows' + shell: pwsh run: | Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser iex "& {$(irm get.scoop.sh)} -RunAsAdmin" scoop install make zig - shell: pwsh - - name: Install JavaScript deps + - name: Install JavaScript dependencies run: | corepack enable yarn - - name: Run Darwin tests - if: contains(matrix.platform, 'darwin') + - name: Setup Rust + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: nightly + - name: Run tests + if: matrix.arch != 'aarch64' || matrix.platform != 'linux' run: | make test-ci - - name: Run Linux X86 tests - if: contains(matrix.platform, 'linux') - run: | - make CURRENT_TARGET=x86_64-unknown-linux-musl test-ci - - name: Run Linux ARM tests - if: contains(matrix.platform, 'linux') + - name: Run tests + if: matrix.arch == 'aarch64' && matrix.platform == 'linux' env: CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5d6f2e0a11..dd12907052 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,8 +12,16 @@ jobs: include: - os: ubuntu-latest platform: linux + arch: x86_64 + - os: ubuntu-latest + platform: linux + arch: aarch64 + - os: macos-latest + platform: darwin + arch: x86_64 - os: macos-latest platform: darwin + arch: aarch64 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -24,54 +32,69 @@ jobs: with: cache: yarn node-version: lts/* - - name: Install linux dependencies - if: contains(matrix.platform, 'linux') + - name: Install Linux dependencies + if: matrix.platform == 'linux' run: | sudo apt-get -y update - sudo apt-get -y install make + sudo apt-get -y install make nodejs sudo snap install zig --classic --beta - - name: Setup Rust - uses: dtolnay/rust-toolchain@v1 - with: - toolchain: nightly - name: Install MacOS dependencies - if: contains(matrix.platform, 'darwin') + if: matrix.platform == 'darwin' env: HOMEBREW_NO_AUTO_UPDATE: 1 run: | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" brew install zig make - - name: Install JavaScript deps + - name: Install Windows dependencies + if: matrix.platform == 'windows' + shell: pwsh + run: | + Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + iex "& {$(irm get.scoop.sh)} -RunAsAdmin" + scoop install make zig + - name: Install JavaScript dependencies run: | corepack enable yarn + - name: Setup Rust + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: nightly - name: Run tests + if: matrix.arch != 'aarch64' || matrix.platform != 'linux' run: | make test-ci - - name: Make stdlib & libs - if: contains(matrix.platform, 'linux') + - name: Run tests + if: matrix.arch == 'aarch64' && matrix.platform == 'linux' + env: + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 run: | - make stdlib && make libs - - name: Build OSX binaries - if: contains(matrix.platform, 'darwin') + sudo apt-get install -y \ + qemu-system-arm \ + qemu-efi-aarch64 \ + qemu-utils \ + qemu-user + + make CURRENT_TARGET=aarch64-unknown-linux-musl test-ci + - name: Build binaries run: | - make release-osx - - name: Build Linux binaries - if: contains(matrix.platform, 'linux') + make release + - name: Build Lambda binaries + if: matrix.platform == 'linux' run: | - make release-linux - - name: Upload changelog - if: contains(matrix.platform, 'linux') - uses: actions/upload-artifact@v4 - with: - name: changelog - path: CHANGELOG.md + make release-lambda - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: artifacts-${{ matrix.platform }} path: | *.zip + - name: Upload changelog + if: matrix.platform == 'linux' && matrix.arch == 'x86_64' + uses: actions/upload-artifact@v4 + with: + name: changelog + path: CHANGELOG.md release: permissions: contents: write diff --git a/Makefile b/Makefile index 069f437e7b..33a4e61192 100644 --- a/Makefile +++ b/Makefile @@ -9,21 +9,22 @@ BUILD_DIR = ./target/release BUNDLE_DIR = bundle ZSTD_LIB_ARGS = UNAME=Linux ZSTD_LIB_COMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 CC_ARM = zig cc -target aarch64-linux-musl -flto -CC_X86 = zig cc -target x86_64-linux-musl -flto +CC_X64 = zig cc -target x86_64-linux-musl -flto AR = zig ar TS_SOURCES = $(wildcard src/js/*.ts) $(wildcard src/js/@llrt/*.ts) $(wildcard tests/*.ts) STD_JS_FILE = $(BUNDLE_DIR)/@llrt/std.js -RELEASE_ARCH_NAME_x86 = x86_64 +RELEASE_ARCH_NAME_x64 = x86_64 RELEASE_ARCH_NAME_arm64 = arm64 -RELEASE_TARGETS = arm64 x86 -RELEASE_ZIPS = $(addprefix llrt-lambda-,$(RELEASE_TARGETS)) +LAMBDA_PREFIX = llrt-lambda +RELEASE_TARGETS = arm64 x64 +RELEASE_ZIPS = $(addprefix $(LAMBDA_PREFIX)-,$(RELEASE_TARGETS)) ifeq ($(OS),Windows_NT) DETECTED_OS := Windows - ARCH := x86 + ARCH := x64 else DETECTED_OS := $(shell uname | tr A-Z a-z) ARCH := $(shell uname -m) @@ -31,22 +32,24 @@ endif CURRENT_TARGET ?= $(TARGET_$(DETECTED_OS)_$(ARCH)) -lambda: | libs $(RELEASE_ZIPS) +lambda-all: clean-js | libs $(RELEASE_ZIPS) +release-all: clean-js | lambda-all llrt-linux-x64.zip llrt-linux-arm64.zip llrt-darwin-x64.zip llrt-darwin-arm64.zip -release: clean-js | lambda llrt-linux-x86.zip llrt-linux-arm64.zip llrt-macos-x86.zip llrt-macos-arm64.zip +release-lambda: clean-js | libs-$(ARCH) $(LAMBDA_PREFIX)-$(DETECTED_OS)-$(ARCH).zip +release: clean-js | llrt-$(DETECTED_OS)-$(ARCH).zip -release-linux: clean-js | lambda llrt-linux-x86.zip llrt-linux-arm64.zip -release-osx: clean-js | llrt-macos-x86.zip llrt-macos-arm64.zip +release-linux: clean-js | lambda-all llrt-linux-x64.zip llrt-linux-arm64.zip +release-darwin: clean-js | llrt-darwin-x64.zip llrt-darwin-arm64.zip -llrt-macos-x86.zip: js +llrt-darwin-x64.zip: js cargo $(BUILD_ARG) --target $(TARGET_darwin_x86_64) zip -j $@ target/$(TARGET_darwin_x86_64)/release/llrt -llrt-macos-arm64.zip: js +llrt-darwin-arm64.zip: js cargo $(BUILD_ARG) --target $(TARGET_darwin_arm64) zip -j $@ target/$(TARGET_darwin_arm64)/release/llrt -llrt-linux-x86.zip: js +llrt-linux-x64.zip: js cargo $(BUILD_ARG) --target $(TARGET_linux_x86_64) zip -j $@ target/$(TARGET_linux_x86_64)/release/llrt @@ -160,9 +163,12 @@ test-ci: clean-js | toolchain js cargo $(TOOLCHAIN) -Z panic-abort-tests test --target $(CURRENT_TARGET) cargo $(TOOLCHAIN) run -r --target $(CURRENT_TARGET) -- test -d bundle -libs: lib/zstd.h +libs-arm64: lib/arm64/libzstd.a lib/zstd.h +libs-x64: lib/x64/libzstd.a lib/zstd.h -lib/zstd.h: | lib/arm64/libzstd.a lib/x86/libzstd.a +libs: | libs-arm64 libs-x64 + +lib/zstd.h: cp zstd/lib/zstd.h $@ lib/arm64/libzstd.a: @@ -171,10 +177,10 @@ lib/arm64/libzstd.a: cd zstd/lib && make clean && make -j lib-nomt CC="$(CC_ARM)" AR="$(AR)" $(ZSTD_LIB_ARGS) cp zstd/lib/libzstd.a $@ -lib/x86/libzstd.a: +lib/x64/libzstd.a: mkdir -p $(dir $@) rm -f zstd/lib/-.o - cd zstd/lib && make clean && make -j lib-nomt CC="$(CC_X86)" AR="$(AR)" $(ZSTD_LIB_ARGS) + cd zstd/lib && make clean && make -j lib-nomt CC="$(CC_X64)" AR="$(AR)" $(ZSTD_LIB_ARGS) cp zstd/lib/libzstd.a $@ bench: @@ -184,4 +190,4 @@ bench: deploy: cd example/infrastructure && yarn deploy --require-approval never -.PHONY: toolchain clean-js release-linux release-osx lambda stdlib test test-ci run js run-release build release clean flame deploy \ No newline at end of file +.PHONY: libs libs-arm64 libs-x64 toolchain clean-js release-linux release-darwin lambda stdlib test test-ci run js run-release build release clean flame deploy \ No newline at end of file