builder_cache (#29)

* mounting a volume to buildkit

* squash last 12 commits

* remove toml

* remove volume

* retrigger tests

* making import export dynamic

* set the startup flag

* Add tests support for startup & refactoring

* testing

* testing

* testing

* testing

* testing

* testing

* testing

* testing

* testing

* testing

* was sh all along

* fix startup string to bool

* removing ts for testing as building it alone fails

* runtime error while building c

* now go failed

* rust failed

* just py,rb,node

* just py,rb,node

* trigger all the tests

* trigger all the tests

* trigger all the tests

* trigger all the tests

* trigger all the tests

* trigger all the tests startup with cli, i am soooo dumb

* i think this might work now

* final testing

* retrigger

* what happens if i trigger tests with only runtime build

* retrigger with backtrace added

* add diff registry for run time/dev to isolate and retrigger tests with both open

* disable backtrace like before, this test should fail

* triggering only the startup tests with backtrace enabled

* enable backtrace,both issues resolved, very demure, very mindful

* cleanup and enable all tests

* only add startup-rootless and startup-client as tests

* Seperated tests

.github/workflows/test.yml → .github/workflows/langTest.yml

@@ -34,4 +34,4 @@ jobs:
         run: ./test/test.sh rootless
 
       - name: Test Daemon
-        run: ./test/test.sh client
+        run: ./test/test.sh client

.github/workflows/startuptest.yml

@@ -0,0 +1,38 @@
+name: Build and Test MetaCall Builder for startup flow
+# Seperate because the space is limited in the github actions
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - "v*.*.*"
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      # TODO:
+      # - name: Lint
+      #   run: docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.59.1 golangci-lint run -v --enable-all
+
+      - name: Build
+        run: docker compose build
+
+      - name: Test Binary
+        env:
+          BUILDER_ARGS: "runtime rb"
+        run: docker compose up --exit-code-from binary binary
+
+      - name: Test Startup-Rootless
+        run: ./test/test.sh rootless startup
+
+      - name: Test Startup-Daemon
+        run: ./test/test.sh client startup

Dockerfile

@@ -32,9 +32,13 @@ FROM moby/buildkit:master-rootless AS builder_rootless
 
 COPY --from=builder_binary --chown=user:user /builder /home/user/builder
 
+# TO-DO : Replace with copying from scripts locally
 RUN printf '#!/bin/sh\n\
 export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\
-/home/user/builder $@ | buildctl-daemonless.sh build --output type=image,name=registry:5000/metacall/builder_output,push=true,registry.insecure=true\n'\
+/home/user/builder $@ | buildctl-daemonless.sh build \
+--export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \
+--import-cache type=registry,ref=${IMPORT_REGISTRY},registry.insecure=true \
+--output type=image,name=registry:5000/metacall/builder_output_$1,push=true,registry.insecure=true\n'\
 >> /home/user/builder.sh \
     && chmod 700 /home/user/builder.sh \
     && chmod 700 /home/user/builder
@@ -46,8 +50,12 @@ COPY --from=builder_binary --chown=root:root /builder /home/builder
 
 RUN apk add --no-cache docker
 
+# TO-DO : Replace with copying from scripts locally
 RUN printf '#!/bin/sh\n\
-/home/builder $@ | buildctl --addr="docker-container://metacall_builder_buildkit" build --output type=image,name=registry:5000/metacall/builder_output,push=true,registry.insecure=true\n'\
+/home/builder $@ | buildctl --addr="docker-container://metacall_builder_buildkit" build \
+--export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \
+--import-cache type=registry,ref=${IMPORT_REGISTRY},registry.insecure=true \
+--output type=image,name=registry:5000/metacall/builder_output_$1,push=true,registry.insecure=true\n'\
 >> /home/builder.sh \
     && chmod 700 /home/builder.sh \
     && chmod 700 /home/builder

cmd/builder/flags.go

@@ -6,8 +6,13 @@ import (
 
 type RuntimeImageFlags struct {
 	MetacallCli bool
+	Startup bool
 }
 
 func (i *RuntimeImageFlags) Set(cmd *cobra.Command) {
 	cmd.Flags().BoolVar(&i.MetacallCli, "cli", false, "set to also get metacall cli in the runtime image")
 }
+
+func (i *RuntimeImageFlags) SetStartup(cmd *cobra.Command) {
+	cmd.Flags().BoolVar(&i.Startup, "startup", false, "startup flag to be used for building image with all languages")
+}

cmd/builder/runtime.go

@@ -21,15 +21,21 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command {
 		Use:   "runtime",
 		Short: "Build runtime image for MetaCall",
 		RunE: func(cmd *cobra.Command, args []string) error {
+
 			if o.RuntimeImageFlags.MetacallCli {
 				args = append(args, "node")
 			}
+
+			if o.RuntimeImageFlags.Startup {
+				args = staging.GetAllLanguages()
+			}
+
 			base := cmd.Context().Value(baseKey{}).(llb.State)
 
 			devBaseLang := staging.DevBase(base, branch, args)
 			devImage := staging.MergeStates(devBaseLang)
 
-			runtimeLang:= staging.RuntimeBase(base, branch, args)
+			runtimeLang := staging.RuntimeBase(base, branch, args)
 			runtimeImage := staging.MergeStates(runtimeLang)
 
 			if o.RuntimeImageFlags.MetacallCli {
@@ -40,14 +46,14 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command {
 			if err != nil {
 				return err
 			}
-
 			cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, runtime))
 			return nil
 
 		},
 		Example: `"builder runtime -b develop --cli nodejs typescript go rust wasm java c cobol"`,
 	}
 	o.RuntimeImageFlags.Set(cmd)
+	o.RuntimeImageFlags.SetStartup(cmd)
 
 	return cmd
 }

docker-compose.yml

@@ -1,8 +1,5 @@
 version: "3.7"
 
-# volumes:
-#   buildkit:
-
 services:
   binary:
     image: metacall/builder_binary
@@ -21,6 +18,10 @@ services:
       dockerfile: Dockerfile
       target: builder_rootless
     entrypoint: /home/user/builder.sh ${BUILDER_ARGS:-}
+    # entrypoint:  /bin/sh -c "sleep infinity"
+    environment:
+      IMPORT_REGISTRY: ${IMPORT_REGISTRY:-registry:5000/metacall/builder_startup} # Default won't be needed once we copy the script
+      EXPORT_REGISTRY: ${EXPORT_REGISTRY:-registry:5000/metacall/builder_dump}
     security_opt:
       - "seccomp=unconfined"
       - "apparmor=unconfined"
@@ -37,6 +38,9 @@ services:
       dockerfile: Dockerfile
       target: builder_client
     entrypoint: /home/builder.sh ${BUILDER_ARGS:-}
+    environment:
+      IMPORT_REGISTRY: ${IMPORT_REGISTRY:-testing}
+      EXPORT_REGISTRY: ${EXPORT_REGISTRY:-testing}
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       # - buildkit:/run/user/1000/buildkit/buildkitd.sock
@@ -51,10 +55,6 @@ services:
     container_name: metacall_builder_buildkit
     restart: always
     privileged: true
-    # volumes:
-    #   - /var/run/docker.sock:/var/run/docker.sock
-    #   - /run/containerd/containerd.sock:/run/containerd/containerd.sock
-    #   - buildkit:/run/buildkit/buildkitd.sock
     healthcheck:
       test: buildctl debug workers
       interval: 10s

pkg/env/env.go

@@ -137,8 +137,8 @@ func (e Env) MetaCallBuild(arg string) Env {
 }
 
 func (e Env) MetacallRuntime(arg string) Env {
-	e.state = e.state.Run(llb.Shlexf("bash /usr/local/metacall/tools/metacall-runtime.sh base ports clean %v", arg)).
-		Root() // TODO: Add backtrace
+	e.state = e.state.Run(llb.Shlexf("bash /usr/local/metacall/tools/metacall-runtime.sh base backtrace ports clean %v", arg)).
+		Root()
 
 	return e
 }

pkg/staging/stages.go

@@ -6,9 +6,8 @@ import (
 )
 
 var languageMap = map[string]string{
-	"base": "base",
-	"py":   "python",
-	"rb":   "ruby",
+	"py": "python",
+	"rb": "ruby",
 	// "netcore":   "netcore",
 	// "netcore2":  "netcore2",
 	// "netcore5":  "netcore5",
@@ -22,15 +21,15 @@ var languageMap = map[string]string{
 	// "v8rep52":   "v8rep52",
 	// "v8rep51":   "v8rep51",
 	"node": "nodejs",
-	"ts":   "typescript",
+	// "ts":   "typescript",
 	// "file":       "file",
 	// "rpc":        "rpc",
-	"wasm": "wasm",
-	"java": "java",
-	"c":    "c",
-	"cob":  "cobol",
-	"go":   "go",
-	"rs":   "rust",
+	// "wasm": "wasm",
+	// "java": "java",
+	// // "c":    "c",
+	// "cob":  "cobol",
+	// "go":   "go",
+	// "rs":   "rust",
 	// "backtrace"	: "backtrace",
 	// "sandbox"	: "sandbox",
 }
@@ -135,3 +134,11 @@ func MergeStates(individualLangStates map[string]llb.State) llb.State {
 	}
 	return llb.Merge(states)
 }
+
+func GetAllLanguages() []string {
+	langs := []string{}
+	for lang := range languageMap {
+		langs = append(langs, lang)
+	}
+	return langs
+}

scripts/builder.sh

@@ -0,0 +1,20 @@
+#!/bin/sh\n\
+export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\
+
+if [ -z "$EXPORT_REGISTRY" ]; then
+    EXPORT_REGISTRY="registry:5000/metacall/builder_cache"
+fi
+
+if [ -z "$IMPORT_REGISTRY" ]; then
+    IMPORT_REGISTRY="registry:5000/metacall/builder_startup"
+fi
+
+if [ -z "$BUILDER_BINARY" ]; then
+    BUILDER_BINARY="/home/user/builder"
+fi
+
+
+${BUILDER_BINARY} $@ | buildctl-daemonless.sh build \
+--export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \
+--import-cache type=registry,ref=${IMPORT_REGISTRY},registry.insecure=true \
+--output type=image,name=registry:5000/metacall/builder_output,push=true,registry.insecure=true\n'\

test/test.sh

@@ -16,34 +16,46 @@ else
 fi
 
 DOCKER_SERVICE=${1:-rootless}
+TEST_TYPE=${2:-default}
 
 build() {
 	${DOCKER_CMD} up --exit-code-from ${DOCKER_SERVICE} ${DOCKER_SERVICE}
 }
 
-test() {
-	build
-
+setupRegistry(){
+
 	${DOCKER_CMD} up -d registry
 
 	while [ ! "$(docker inspect --format '{{json .State.Health.Status }}' metacall_builder_registry)" = "\"healthy\"" ]; do
 		sleep 5
 	done
+}
 
-	DOCKER_OUTPUT=`docker run --rm -v ./test/suites:/test -t localhost:5000/metacall/builder_output sh -c "metacallcli test/$1"`
+cleanup(){
+	${DOCKER_CMD} down
+}
+
+test() {
+
+	build
+	setupRegistry
+
+	first_arg=$(echo $BUILDER_ARGS | cut -d ' ' -f 1)
+
+	DOCKER_OUTPUT=`docker run --rm -v ./test/suites:/test -t localhost:5000/metacall/builder_output_$first_arg sh -c "metacallcli test/$1"`
 	DOCKER_OUTPUT=`echo ${DOCKER_OUTPUT} | tr -d '\r\n'`
 	EXPECTED_OUTPUT=`echo $2 | tr -d '\r\n'`
+	TEST_NAME=`echo $1`
 
 	if [ "${DOCKER_OUTPUT}" = "${EXPECTED_OUTPUT}" ]; then
-		echo "Test passed: $1"
+		echo "Test passed: ${TEST_NAME}"
 	else
-		echo "Failed to run test: $1"
+		echo "Failed to run test: ${TEST_NAME}"
 		echo "Expected output was: '${EXPECTED_OUTPUT}'"
 		echo "Test output was: '${DOCKER_OUTPUT}'"
 		exit 1
 	fi
 
-	${DOCKER_CMD} down
 }
 
 # TODO:
@@ -57,12 +69,44 @@ test() {
 # 	done
 # done
 
-# Build the dev image with NodeJS language
-echo "Building dev mode with NodeJS language."
-export BUILDER_ARGS="dev node"
-test node/test.js "0123456789"
+defaultTests(){
+	# Build the dev image with NodeJS language
+	echo "Building dev mode with NodeJS language."
+	export BUILDER_ARGS="dev node"
+	export IMPORT_REGISTRY="registry:5000/metacall/builder_cache"
+	export EXPORT_REGISTRY="registry:5000/metacall/builder_cache"
+	test node/test.js "0123456789"
+	cleanup
+
+	# Build the cli image with languages all together
+	echo "Building cli mode with all languages."
+	export BUILDER_ARGS="runtime --cli py node rb"
+	export IMPORT_REGISTRY="registry:5000/metacall/builder_cache"
+	export EXPORT_REGISTRY="registry:5000/metacall/builder_cache"
+	test node/test.js "0123456789"
+	cleanup
+}
+
+startupTests(){
+	# Build the startup image with all languages
+	echo "Building all languages in startup mode."
+	export BUILDER_ARGS="runtime --cli --startup"
+	export EXPORT_REGISTRY="registry:5000/metacall/builder_startup"
+	export IMPORT_REGISTRY="registry:5000/metacall/builder_startup"
+	test node/test.js "0123456789"
+
+
+	# Testing the cache registry
+	echo "Building cli mode with node and py languages."
+	export BUILDER_ARGS="runtime --cli py node"
+	export IMPORT_REGISTRY="registry:5000/metacall/builder_startup"
+	export EXPORT_REGISTRY="registry:5000/metacall/builder_dump" # To not able to rewrite the cache 
+	test node/test.js "0123456789" # Should be quicker since all caches are already built
+	cleanup
+}
 
-# Build the cli image with languages all together
-echo "Building cli mode with all languages."
-export BUILDER_ARGS="runtime --cli py node rb"
-test node/test.js "0123456789"
+if [ "${TEST_TYPE}" = "default" ]; then
+	defaultTests
+elif [ "${TEST_TYPE}" = "startup" ]; then
+	startupTests
+fi