Update CI (#68)

Update CI
awslabs · Jan 14, 2024 · 07e3f32 · 07e3f32
1 parent 9fcea0c
commit 07e3f32
Show file tree

Hide file tree

Showing 3 changed files with 92 additions and 59 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -41,8 +41,16 @@ jobs:
         include:
           - os: ubuntu-latest
             platform: linux
+            arch: x86_64
+          - os: ubuntu-latest
+            platform: linux
+            arch: aarch64
+          - os: macos-latest
+            platform: darwin
+            arch: x86_64
           - os: macos-latest
             platform: darwin
+            arch: aarch64
           # - os: windows-latest
           #   platform: windows
     runs-on: ${{ matrix.os }}
@@ -55,44 +63,40 @@ jobs:
         with:
           cache: yarn
           node-version: lts/*
-      - name: Install linux dependencies
-        if: contains(matrix.platform, 'linux')
+      - name: Install Linux dependencies
+        if: matrix.platform == 'linux'
         run: |
           sudo apt-get -y update
           sudo apt-get -y install make nodejs
           sudo snap install zig --classic --beta
-      - name: Setup Rust
-        uses: dtolnay/rust-toolchain@v1
-        with:
-          toolchain: nightly
       - name: Install MacOS dependencies
-        if: contains(matrix.platform, 'darwin')
+        if: matrix.platform == 'darwin'
         env:
           HOMEBREW_NO_AUTO_UPDATE: 1
         run: |
           /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
           brew install zig make
       - name: Install Windows dependencies
-        if: contains(matrix.platform, 'windows')
+        if: matrix.platform == 'windows'
+        shell: pwsh
         run: |
           Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
           iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
           scoop install make zig
-        shell: pwsh
-      - name: Install JavaScript deps
+      - name: Install JavaScript dependencies
         run: |
           corepack enable
           yarn
-      - name: Run Darwin tests
-        if: contains(matrix.platform, 'darwin')
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: nightly
+      - name: Run tests
+        if: matrix.arch != 'aarch64' || matrix.platform != 'linux'
         run: |
           make test-ci
-      - name: Run Linux X86 tests
-        if: contains(matrix.platform, 'linux')
-        run: |
-          make CURRENT_TARGET=x86_64-unknown-linux-musl test-ci
-      - name: Run Linux ARM tests
-        if: contains(matrix.platform, 'linux')
+      - name: Run tests
+        if: matrix.arch == 'aarch64' && matrix.platform == 'linux'
         env:
           CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64
         run: |

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -12,8 +12,16 @@ jobs:
         include:
           - os: ubuntu-latest
             platform: linux
+            arch: x86_64
+          - os: ubuntu-latest
+            platform: linux
+            arch: aarch64
+          - os: macos-latest
+            platform: darwin
+            arch: x86_64
           - os: macos-latest
             platform: darwin
+            arch: aarch64
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -24,54 +32,69 @@ jobs:
         with:
           cache: yarn
           node-version: lts/*
-      - name: Install linux dependencies
-        if: contains(matrix.platform, 'linux')
+      - name: Install Linux dependencies
+        if: matrix.platform == 'linux'
         run: |
           sudo apt-get -y update
-          sudo apt-get -y install make
+          sudo apt-get -y install make nodejs
           sudo snap install zig --classic --beta
-      - name: Setup Rust
-        uses: dtolnay/rust-toolchain@v1
-        with:
-          toolchain: nightly
       - name: Install MacOS dependencies
-        if: contains(matrix.platform, 'darwin')
+        if: matrix.platform == 'darwin'
         env:
           HOMEBREW_NO_AUTO_UPDATE: 1
         run: |
           /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
           brew install zig make
-      - name: Install JavaScript deps
+      - name: Install Windows dependencies
+        if: matrix.platform == 'windows'
+        shell: pwsh
+        run: |
+          Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
+          iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
+          scoop install make zig
+      - name: Install JavaScript dependencies
         run: |
           corepack enable
           yarn
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: nightly
       - name: Run tests
+        if: matrix.arch != 'aarch64' || matrix.platform != 'linux'
         run: |
           make test-ci
-      - name: Make stdlib & libs
-        if: contains(matrix.platform, 'linux')
+      - name: Run tests
+        if: matrix.arch == 'aarch64' && matrix.platform == 'linux'
+        env:
+          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64
         run: |
-          make stdlib && make libs
-      - name: Build OSX binaries
-        if: contains(matrix.platform, 'darwin')
+          sudo apt-get install -y \
+            qemu-system-arm \
+            qemu-efi-aarch64 \
+            qemu-utils \
+            qemu-user
+
+          make CURRENT_TARGET=aarch64-unknown-linux-musl test-ci
+      - name: Build binaries
         run: |
-          make release-osx
-      - name: Build Linux binaries
-        if: contains(matrix.platform, 'linux')
+          make release
+      - name: Build Lambda binaries
+        if: matrix.platform == 'linux'
         run: |
-          make release-linux
-      - name: Upload changelog
-        if: contains(matrix.platform, 'linux')
-        uses: actions/upload-artifact@v4
-        with:
-          name: changelog
-          path: CHANGELOG.md
+          make release-lambda
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
           name: artifacts-${{ matrix.platform }}
           path: |
             *.zip
+      - name: Upload changelog
+        if: matrix.platform == 'linux' && matrix.arch == 'x86_64'
+        uses: actions/upload-artifact@v4
+        with:
+          name: changelog
+          path: CHANGELOG.md
   release:
     permissions:
       contents: write

diff --git a/Makefile b/Makefile
@@ -9,44 +9,47 @@ BUILD_DIR = ./target/release
 BUNDLE_DIR = bundle
 ZSTD_LIB_ARGS = UNAME=Linux ZSTD_LIB_COMPRESSION=0 ZSTD_LIB_DICTBUILDER=0
 CC_ARM = zig cc -target aarch64-linux-musl -flto
-CC_X86 = zig cc -target x86_64-linux-musl -flto
+CC_X64 = zig cc -target x86_64-linux-musl -flto
 AR = zig ar
 
 TS_SOURCES = $(wildcard src/js/*.ts) $(wildcard src/js/@llrt/*.ts) $(wildcard tests/*.ts)
 STD_JS_FILE = $(BUNDLE_DIR)/@llrt/std.js
 
-RELEASE_ARCH_NAME_x86 = x86_64
+RELEASE_ARCH_NAME_x64 = x86_64
 RELEASE_ARCH_NAME_arm64 = arm64
 
-RELEASE_TARGETS = arm64 x86
-RELEASE_ZIPS = $(addprefix llrt-lambda-,$(RELEASE_TARGETS))
+LAMBDA_PREFIX = llrt-lambda
+RELEASE_TARGETS = arm64 x64
+RELEASE_ZIPS = $(addprefix $(LAMBDA_PREFIX)-,$(RELEASE_TARGETS))
 
 ifeq ($(OS),Windows_NT)
     DETECTED_OS := Windows
-	ARCH := x86
+	ARCH := x64
 else
     DETECTED_OS := $(shell uname | tr A-Z a-z)
 	ARCH := $(shell uname -m)
 endif
 
 CURRENT_TARGET ?= $(TARGET_$(DETECTED_OS)_$(ARCH))
 
-lambda: | libs $(RELEASE_ZIPS)
+lambda-all: clean-js | libs $(RELEASE_ZIPS)
+release-all: clean-js | lambda-all llrt-linux-x64.zip llrt-linux-arm64.zip llrt-darwin-x64.zip llrt-darwin-arm64.zip
 
-release: clean-js | lambda llrt-linux-x86.zip llrt-linux-arm64.zip llrt-macos-x86.zip llrt-macos-arm64.zip
+release-lambda: clean-js |  libs-$(ARCH) $(LAMBDA_PREFIX)-$(DETECTED_OS)-$(ARCH).zip
+release: clean-js | llrt-$(DETECTED_OS)-$(ARCH).zip
 
-release-linux: clean-js | lambda llrt-linux-x86.zip llrt-linux-arm64.zip
-release-osx: clean-js | llrt-macos-x86.zip llrt-macos-arm64.zip
+release-linux: clean-js | lambda-all llrt-linux-x64.zip llrt-linux-arm64.zip
+release-darwin: clean-js | llrt-darwin-x64.zip llrt-darwin-arm64.zip
 
-llrt-macos-x86.zip: js
+llrt-darwin-x64.zip: js
 	cargo $(BUILD_ARG) --target $(TARGET_darwin_x86_64)
 	zip -j $@ target/$(TARGET_darwin_x86_64)/release/llrt
 
-llrt-macos-arm64.zip: js
+llrt-darwin-arm64.zip: js
 	cargo $(BUILD_ARG) --target $(TARGET_darwin_arm64)
 	zip -j $@ target/$(TARGET_darwin_arm64)/release/llrt
 
-llrt-linux-x86.zip: js
+llrt-linux-x64.zip: js
 	cargo $(BUILD_ARG) --target $(TARGET_linux_x86_64)
 	zip -j $@ target/$(TARGET_linux_x86_64)/release/llrt
 
@@ -160,9 +163,12 @@ test-ci: clean-js | toolchain js
 	cargo $(TOOLCHAIN) -Z panic-abort-tests test --target $(CURRENT_TARGET)
 	cargo $(TOOLCHAIN) run -r --target $(CURRENT_TARGET) -- test -d bundle
 
-libs: lib/zstd.h
+libs-arm64: lib/arm64/libzstd.a lib/zstd.h
+libs-x64: lib/x64/libzstd.a lib/zstd.h
 
-lib/zstd.h: | lib/arm64/libzstd.a lib/x86/libzstd.a
+libs: | libs-arm64 libs-x64
+
+lib/zstd.h:
 	cp zstd/lib/zstd.h $@
 
 lib/arm64/libzstd.a: 
@@ -171,10 +177,10 @@ lib/arm64/libzstd.a:
 	cd zstd/lib && make clean && make -j lib-nomt CC="$(CC_ARM)" AR="$(AR)" $(ZSTD_LIB_ARGS)
 	cp zstd/lib/libzstd.a $@
 
-lib/x86/libzstd.a:
+lib/x64/libzstd.a:
 	mkdir -p $(dir $@)
 	rm -f zstd/lib/-.o
-	cd zstd/lib && make clean && make -j lib-nomt CC="$(CC_X86)" AR="$(AR)" $(ZSTD_LIB_ARGS)
+	cd zstd/lib && make clean && make -j lib-nomt CC="$(CC_X64)" AR="$(AR)" $(ZSTD_LIB_ARGS)
 	cp zstd/lib/libzstd.a $@ 
 
 bench:
@@ -184,4 +190,4 @@ bench:
 deploy:
 	cd example/infrastructure && yarn deploy --require-approval never
 
-.PHONY: toolchain clean-js release-linux release-osx lambda stdlib test test-ci run js run-release build release clean flame deploy
+.PHONY: libs libs-arm64 libs-x64 toolchain clean-js release-linux release-darwin lambda stdlib test test-ci run js run-release build release clean flame deploy