Skip to content

Native JNI S3 Paddle #961

Native JNI S3 Paddle

Native JNI S3 Paddle #961

name: Native JNI S3 Paddle
on:
workflow_dispatch:
schedule:
- cron: '0 10 * * *'
jobs:
build-paddle-jni-cpu:
if: github.repository == 'deepjavalibrary/djl'
runs-on: ${{ matrix.operating-system }}
strategy:
matrix:
operating-system: [ macos-latest, windows-latest ]
steps:
- uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
distribution: 'corretto'
java-version: 11
- uses: actions/cache@v3
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
restore-keys: |
${{ runner.os }}-gradle-
- name: Release JNI prep
run: ./gradlew :engines:paddlepaddle:paddlepaddle-native:compileJNI
- name: Upload compiled jni library
uses: actions/upload-artifact@v3
if: always()
with:
name: jnilib-${{ runner.os }}
path: engines/paddlepaddle/paddlepaddle-native/jnilib
# A few things tried to build GPU jni (2.0.2) shared library:
# 1. cpu JNI .so file works fine with cu101 and cu102
# 2. cu101 and cu102 generate identical .so file, no point to build both of them
# 3. centos:7 + GCC 8 is broken: symbol not found
# 4. amazonlinux:2 + GCC 7.3.1 is broken: symbol not found (CPU works fine)
# 5. ubuntu:18.04 + GCC 7.5.1 works fine
# 6. ubuntu:16.04 + GCC 8 works fine (old pipeline use this, assume this works)
build-paddle-jni-linux:
if: github.repository == 'deepjavalibrary/djl'
runs-on: ubuntu-latest
container: amazonlinux:2
steps:
- name: Install Environment
run: |
yum -y update
yum -y groupinstall "Development Tools"
yum -y install patch cmake3
ln -sf /usr/bin/cmake3 /usr/bin/cmake
pip3 install awscli --upgrade
- uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
distribution: 'corretto'
java-version: 11
- uses: actions/cache@v3
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
restore-keys: |
${{ runner.os }}-gradle-
- name: Release JNI prep
run: |
./gradlew :engines:paddlepaddle:paddlepaddle-native:compileJNI
- name: Upload compiled jni library
uses: actions/upload-artifact@v3
if: always()
with:
name: jnilib-${{ runner.os }}
path: engines/paddlepaddle/paddlepaddle-native/jnilib
publish:
if: github.repository == 'deepjavalibrary/djl'
runs-on: ubuntu-latest
needs: [ build-paddle-jni-cpu, build-paddle-jni-linux ]
steps:
- uses: actions/checkout@v3
- name: Download compiledJNI Mac
uses: actions/download-artifact@v3
with:
name: jnilib-macOS
path: jnilib
- name: Download compiledJNI Windows
uses: actions/download-artifact@v3
with:
name: jnilib-Windows
path: jnilib
- name: Download compiledJNI Linux CPU
uses: actions/download-artifact@v3
with:
name: jnilib-Linux
path: jnilib
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1-node16
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Copy files to S3 with the AWS CLI
run: |
PADDLE_VERSION="$(cat gradle.properties | awk -F '=' '/paddlepaddle_version/ {print $2}')"
aws s3 sync jnilib s3://djl-ai/publish/paddlepaddle-${PADDLE_VERSION}/jnilib
aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/paddlepaddle-${PADDLE_VERSION}/jnilib*"