feat: add basic vm implementation (#9)

* feat: add basic vm implementation - fix run-checks CI breaking on main - consolidate all bins into one, `monocore` - add working example, `microvm_shell` - implement rlimit for vm config - add makefiles for building project and examples, esp. for macos entitlement - add tests and doc examples - increase pre-commit allowed file size - improve description of project in readme * fix: ci issues with libkrunfw soname by creating symlinks
appcypher · Oct 15, 2024 · 001f173 · 001f173
1 parent dc33da5
commit 001f173
Show file tree

Hide file tree

Showing 28 changed files with 1,794 additions and 197 deletions.
diff --git a/.github/workflows/tests_and_checks.yml b/.github/workflows/tests_and_checks.yml
@@ -5,6 +5,13 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ '**' ]
+  workflow_dispatch:
+    inputs:
+      force_build_libkrun:
+        description: 'Force build libkrun'
+        required: false
+        type: boolean
+        default: false
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -14,30 +21,45 @@ jobs:
   check-libkrun-changes:
     runs-on: ubuntu-latest
     outputs:
-      should_build: ${{ steps.check_files.outputs.should_build }}
+      should_build: ${{ steps.check_build.outputs.should_build }}
       libkrun_hash: ${{ steps.hash_libkrun.outputs.hash }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 2
 
-      - name: Check if relevant files have changed
-        id: check_files
+      - name: Hash build_libkrun.sh
+        id: hash_libkrun
+        run: |
+          HASH=$(sha256sum build_libkrun.sh | awk '{ print $1 }')
+          echo "hash=$HASH" >> $GITHUB_OUTPUT
+          echo "Calculated hash: $HASH"
+
+      - name: Check if cache exists
+        id: check_cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            build/libkrunfw/libkrunfw*.so*
+            build/libkrun/target/release/libkrun*.so*
+          key: ${{ runner.os }}-libkrun-${{ steps.hash_libkrun.outputs.hash }}
+
+      - name: Check if build is needed
+        id: check_build
         run: |
-          if [ $(git rev-list --count HEAD) -lt 2 ]; then
+          if [ "${{ steps.check_cache.outputs.cache-hit }}" == 'true' ]; then
+            echo "Cache hit, no need to build"
+            echo "should_build=false" >> $GITHUB_OUTPUT
+          elif [ $(git rev-list --count HEAD) -lt 2 ]; then
             echo "Only one commit in the repository, building libkrun"
             echo "should_build=true" >> $GITHUB_OUTPUT
           else
             git diff --name-only HEAD^ HEAD | grep -q 'build_libkrun.sh' && echo "should_build=true" >> $GITHUB_OUTPUT || echo "should_build=false" >> $GITHUB_OUTPUT
           fi
 
-      - name: Hash build_libkrun.sh
-        id: hash_libkrun
-        run: echo "hash=$(sha256sum build_libkrun.sh | awk '{ print $1 }')" >> $GITHUB_OUTPUT
-
   build-libkrun:
     needs: check-libkrun-changes
-    if: needs.check-libkrun-changes.outputs.should_build == 'true'
+    if: needs.check-libkrun-changes.outputs.should_build == 'true' || github.event.inputs.force_build_libkrun == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
@@ -127,9 +149,21 @@ jobs:
         with:
           command: check bans licenses sources
 
+      # A hack to make `ld` find the libkrunfw because right now it seems to only look for libkrunfw.so.x
+      # and not libkrunfw.so or libkrunfw.so.x.x.x even though their SONAME is libkrunfw.so.x
+      - name: Create symlinks
+        run: |
+          cd build/libkrunfw
+          objdump -p libkrunfw.so.4.4.2 | grep SONAME # sanity check
+          ln -sf libkrunfw.so.4.4.2 libkrunfw.so.4
+          ln -sf libkrunfw.so.4 libkrunfw.so
+
       - name: Test Release
         if: ${{ matrix.rust-toolchain == 'stable'  && github.event_name == 'push' }}
-        run: cargo build --release
+        run: |
+          LIBRARY_PATH=${{ github.workspace }}/build/libkrunfw:${{ github.workspace }}/build/libkrun/target/release:${{ env.LIBRARY_PATH }} \
+          LD_LIBRARY_PATH=${{ github.workspace }}/build/libkrunfw:${{ github.workspace }}/build/libkrun/target/release:${{ env.LD_LIBRARY_PATH }} \
+          cargo build --release
 
   run-tests:
     needs: [check-libkrun-changes, build-libkrun]
@@ -161,15 +195,26 @@ jobs:
             build/libkrun/target/release/libkrun*.so*
           key: ${{ runner.os }}-libkrun-${{ needs.check-libkrun-changes.outputs.libkrun_hash }}
 
+      - name: Print Cache
+        run: tree -L 2 build/
+
       - name: Install Rust Toolchain
         uses: actions-rs/toolchain@v1
         with:
           override: true
           toolchain: ${{ matrix.rust-toolchain }}
 
+      # A hack to make `ld` find the libkrunfw because right now it seems to only look for libkrunfw.so.x
+      # and not libkrunfw.so or libkrunfw.so.x.x.x even though their SONAME is libkrunfw.so.x
+      - name: Create symlinks
+        run: |
+          cd build/libkrunfw
+          objdump -p libkrunfw.so.4.4.2 | grep SONAME # sanity check
+          ln -sf libkrunfw.so.4.4.2 libkrunfw.so.4
+          ln -sf libkrunfw.so.4 libkrunfw.so
+
       - name: Run Tests
         run: |
+          LIBRARY_PATH=${{ github.workspace }}/build/libkrunfw:${{ github.workspace }}/build/libkrun/target/release:${{ env.LIBRARY_PATH }} \
+          LD_LIBRARY_PATH=${{ github.workspace }}/build/libkrunfw:${{ github.workspace }}/build/libkrun/target/release:${{ env.LD_LIBRARY_PATH }} \
           cargo test --all-features
-        env:
-          LIBRARY_PATH: ${{ github.workspace }}/build/libkrunfw:${{ github.workspace }}/build/libkrun/target/release:${{ env.LIBRARY_PATH }}
-          LD_LIBRARY_PATH: ${{ github.workspace }}/build/libkrunfw:${{ github.workspace }}/build/libkrun/target/release:${{ env.LD_LIBRARY_PATH }}
diff --git a/.gitignore b/.gitignore
@@ -12,5 +12,6 @@ private
 .history
 .DS_Store
 
-deps/
+# Build dirs
 build/
+monocore/build/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -52,6 +52,7 @@ repos:
       - id: check-yaml
       - id: check-json
       - id: check-added-large-files
+        args: ['--maxkb=5120']
       - id: detect-private-key
       - id: check-executables-have-shebangs
       - id: check-toml
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/README.md b/README.md
@@ -4,26 +4,30 @@
   </a> -->
 
   <h1 align="center">monocore</h1>
-<!--
+
   <p>
-    <a href="https://crates.io/crates/monocore">
+    <!-- <a href="https://crates.io/crates/monocore">
       <img src="https://img.shields.io/crates/v/monocore?label=crates" alt="Crate">
-    </a>
+    </a> -->
     <a href="https://github.com/appcypher/monocore/actions?query=">
       <img src="https://github.com/appcypher/monocore/actions/workflows/tests_and_checks.yml/badge.svg" alt="Build Status">
     </a>
     <a href="https://github.com/appcypher/monocore/blob/main/LICENSE">
       <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" alt="License">
     </a>
-    <a href="https://docs.rs/monocore">
+    <!-- <a href="https://docs.rs/monocore">
       <img src="https://img.shields.io/static/v1?label=Docs&message=docs.rs&color=blue" alt="Docs">
-    </a>
-  </p> -->
+    </a> -->
+  </p>
 </div>
 
-**`monocore`** is an open-source self-hostable platform for orchestrating lightweight virtual machines.
+**`monocore`** is your gateway to spin up AI-driven sandboxed environments in no time.
+
+Tired of clunky cloud setups? `monocore` is an open-source, self-hostable platform that lets you orchestrate microVMs for your AI agents—right from your local dev machine. With just a single command, you're up and running.
 
-It is specifically designed to quickly provision sandboxed environments for executing code and storing other artifacts generated by **AI agents** _and people_.
+Forget the headaches of replication. `monocore` is distributed by design, so your VMs are seamlessly taken care of, letting you focus on building and experimenting without limits.
+
+Want the power of fly.io but with the freedom of self-hosting? That's where `monocore` comes in. No lock-ins, no unnecessary overhead—just lightweight VMs, ready when you are.
 
 > [!WARNING]
 > This project is in early development and is not yet ready for production use.
@@ -33,45 +37,79 @@ It is specifically designed to quickly provision sandboxed environments for exec
 ## Outline
 
 - [Development](#development)
+- [Contributing](#contributing)
 - [License](#license)
 
 ## Development
 
-If you want to contribute to monocore, you will need to build the project from source.
+Follow these steps to set up monocore for development:
+
+### Prerequisites
+
+- [Git][git_home]
+- [Rust toolchain (latest stable version)][rustup_home]
+- On macOS: [Homebrew][brew_home]
+
+### Setup
+
+1. **Clone the repository**
+
+   ```sh
+   git clone https://github.com/appcypher/monocore
+   cd monocore
+   ```
+
+2. **Build libkrun**
+
+   monocore uses a modified version of [libkrun][libkrun-repo] for its microVMs.
+
+   ```sh
+   ./build_libkrun.sh
+   ```
+
+   > **Note for macOS users:** Install `krunvm` before building libkrun:
+   >
+   > ```sh
+   > brew tap slp/tap
+   > brew install krunvm
+   > ```
+
+3. **Build and install monocore**
+
+   ```sh
+   cd monocore # Ensure you are in the monocore subdirectory
+   make
+   sudo make install
+   ```
 
-### 1. Getting the code
+4. **Run examples**
 
-```sh
-git clone https://github.com/appcypher/monocore
-cd monocore
-```
+   ```sh
+   cd monocore # Ensure you are in the monocore subdirectory
+   make example microvm_shell
+   ```
 
-### 2. Building libkrun
+## Contributing
 
-monocore uses a modified version of [libkrun][libkrun-repo] as the backend for its microVMs, so you will need to build it first.
 
-To do so, run the following command in the `monocore` directory:
+1. **Read the [CONTRIBUTING.md](./CONTRIBUTING.md) file**
 
-```sh
-./build_libkrun.sh
-```
+   This file contains information about the coding style, commit message conventions,
+   and other guidelines that you should follow when contributing to monocore.
 
-> [!NOTE]
-> On macOS, you will need to install `krunvm` using Homebrew before you can build libkrun.
->
-> ```sh
-> brew tap slp/tap
-> brew install krunvm
-> ```
+2. **Install pre-commit hooks**
 
-### 3. Building monocore
+   ```sh
+   pre-commit install
+   ```
 
-```sh
-cargo build --release
-```
+   You will need to have `pre-commit` installed to use these hooks.
 
 ## License
 
 This project is licensed under the [Apache License 2.0](./LICENSE).
 
 [libkrun-repo]: https://github.com/containers/libkrun
+[brew_home]: https://brew.sh/
+[rustup_home]: https://rustup.rs/
+[git_home]: https://git-scm.com/
diff --git a/build_libkrun.sh b/build_libkrun.sh
@@ -13,7 +13,7 @@ LIBKRUN_REPO="https://github.com/appcypher/libkrun.git"
 LIB_DIR="/usr/local/lib"
 LIB64_DIR="/usr/local/lib64"
 NO_CLEANUP=false
-FORCE_BUILD=false  # Added variable for force build
+FORCE_BUILD=false
 
 # Color variables
 RED="\033[1;31m"

diff --git a/monocore/Cargo.toml b/monocore/Cargo.toml
@@ -9,16 +9,8 @@ name = "monocore"
 path = "lib/lib.rs"
 
 [[bin]]
-name = "mono"
-path = "bin/mono.rs"
-
-[[bin]]
-name = "monoproxy"
-path = "bin/monoproxy.rs"
-
-[[bin]]
-name = "monod"
-path = "bin/monod.rs"
+name = "monocore"
+path = "bin/monocore.rs"
 
 [dependencies]
 anyhow.workspace = true
@@ -48,6 +40,7 @@ tracing.workspace = true
 tracing-subscriber.workspace = true
 typed-builder = "0.20.0"
 typed-path = "0.9.2"
+xattr = "1.3.1"
 
 [dev-dependencies]
 test-log = "0.2.16"