Support TestContainers PoC

.cspell/custom-dictionary.txt

@@ -431,4 +431,6 @@ frrist
 swaggo
 isbadactor
 installationid
-firstbacalhauimage
+firstbacalhauimage
+RYUK
+buildvcs

.github/workflows/testcontainers-integration-tests.yml

@@ -0,0 +1,38 @@
+name: Test Containers Integration Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  repository_dispatch:
+    types: [ok-to-test]
+
+jobs:
+  test-build-and-run-testcontainers:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # Detect Go version from test_integration/go.mod
+      - name: Detect Go version from test_integration/go.mod
+        id: detect-test-go-version
+        run: |
+          go_version=$(grep '^go ' test_integration/go.mod | awk '{print $2}')
+          echo "Go version detected for integration tests: $go_version"
+          echo "GOLANG_TEST_VERSION=$go_version" >> $GITHUB_ENV
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GOLANG_TEST_VERSION }}
+          cache: true
+
+      - name: Run Go tests in integration test directory
+        run: |
+          # TODO: make WebUI happy when compiling
+          mkdir -p webui/build
+          echo "hello test" > webui/build/index.html
+          cd test_integration
+          go test -v -count=1 ./...

.gitignore

@@ -65,3 +65,6 @@ dist/
 .arg
 .ruff_cache
 .aider*
+
+# Integration Tests
+test_integration/assets/dockerfiles/bacalhau_bin

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
   - id: detect-aws-credentials
     args: [--allow-missing-credentials]
   - id: detect-private-key
-    exclude: 'testdata/.*|test-integration/certificates/.*'
+    exclude: 'testdata/.*|test_integration/assets/dockerfiles/certificates/.*|docker-compose-deployment/certificates/.*'
   - id: check-yaml
   - id: check-json
 - repo: https://github.com/astral-sh/ruff-pre-commit
@@ -36,4 +36,5 @@ repos:
   - id: "go-test-build-header"
     name: "Go test '//go:build' header present"
     entry: ".gitprecommit/go_test_build_header.sh"
+    exclude: 'test_integration/.*|'
     language: "script"

go.work

@@ -2,4 +2,5 @@ go 1.23.0
 
 use (
 	.
+	test_integration
 )

test_integration/README.md

@@ -0,0 +1,42 @@
+# Integration Test Suite for Bacalhau
+
+This test suite is designed to perform integration testing for Bacalhau using real docker containers, simulating a production-like environment.
+
+## Key Features
+
+- Uses TestContainers Go library for managing Docker environments
+- Implements Testify Go testing suite for structured and extensible tests
+- Compiles project binary and builds custom Docker images for testing
+- Provides a base suite with common functionality, easily extendable for specific test cases
+- Simulates real production usage by executing commands in a jumpbox container
+
+## Architecture
+
+1. **Base Suite**: Compiles the project binary and builds several Docker images:
+    - Compute node
+    - Requester node
+    - Jumpbox node
+
+2. **TestContainers**: Used to set up and manage a Docker Compose stack for each test suite.
+
+3. **Test Structure**:
+    - Each test suite inherits from the base suite
+    - Suites have their own Docker Compose stack, shared by all tests within the suite
+    - Tests within a suite run in series
+    - Different suites can run in parallel if needed
+
+4. **Test Execution**:
+    - Tests use TestContainers' exec-in-container functionality
+    - Commands are run in the jumpbox container, simulating real-world usage
+    - No mocking is used, providing high-fidelity test results
+
+## Run Test Suite:
+
+To run test suite, you will need to have docker daemon on your local machine, as well as docker compose,
+
+Then:
+
+```shell
+cd test_integration
+go test -v -count=1 ./...
+```

test_integration/assets/docker_compose_files/docker-compose-with-minio-and-registry.yml

@@ -0,0 +1,102 @@
+x-common-env-variables: &common-env-variables
+  NETWORK_AUTH_TOKEN: "i_am_very_secret_token"
+  BACALHAU_API_PORT: "1234"
+  MINIO_ROOT_USER: "minioadmin"
+  MINIO_ROOT_PASSWORD: "minioadminpass"
+  AWS_ACCESS_KEY_ID: "minioadmin"
+  AWS_SECRET_ACCESS_KEY: "minioadminpass"
+
+networks:
+  bacalhau-network:
+    driver: bridge
+
+services:
+  bacalhau-minio-node:
+    image: quay.io/minio/minio
+    command: server /data --console-address ":9001"
+    restart: always
+    networks:
+      - bacalhau-network
+    environment: *common-env-variables
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+      start_period: 2s
+
+  bacalhau-container-img-registry-node:
+    image: "placeholder-registry-image"
+    restart: always
+    networks:
+      - bacalhau-network
+    environment:
+      REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY: /var/lib/registry
+      REGISTRY_HTTP_ADDR: "0.0.0.0:5000"
+      REGISTRY_HTTP_TLS_CERTIFICATE: "/certs/bacalhau-container-img-registry-node.crt"
+      REGISTRY_HTTP_TLS_KEY: "/certs/bacalhau-container-img-registry-node.key"
+    healthcheck:
+      test: [ "CMD-SHELL", "nc -zv localhost 5000" ]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+      start_period: 2s
+
+  bacalhau-requester-node:
+    image: "placeholder-requester-image"
+    networks:
+      - bacalhau-network
+    environment: *common-env-variables
+    depends_on:
+      bacalhau-minio-node:
+        condition: service_healthy
+    privileged: true
+    command:
+      - /bin/bash
+      - -c
+      - |
+        bacalhau config set "orchestrator.auth.token" "$${NETWORK_AUTH_TOKEN}" && bacalhau serve --orchestrator -c api.port=$${BACALHAU_API_PORT}
+    healthcheck:
+      test: [ "CMD-SHELL", "nc -zv localhost 1234" ]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+      start_period: 2s
+
+  bacalhau-compute-node:
+    image: "placeholder-compute-image"
+    privileged: true
+    networks:
+      - bacalhau-network
+    depends_on:
+      bacalhau-requester-node:
+        condition: service_healthy
+      bacalhau-container-img-registry-node:
+        condition: service_healthy
+    environment:
+      <<: *common-env-variables
+      REQUESTER_NODE_LINK: 'bacalhau-requester-node'
+    healthcheck:
+      test: [ "CMD-SHELL", "nc -zv localhost 1234" ]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+      start_period: 2s
+
+  bacalhau-jumpbox-node:
+    image: "placeholder-jumpbox-image"
+    privileged: true
+    networks:
+      - bacalhau-network
+    depends_on:
+      bacalhau-requester-node:
+        condition: service_healthy
+      bacalhau-compute-node:
+        condition: service_healthy
+      bacalhau-container-img-registry-node:
+        condition: service_healthy
+    environment:
+      <<: *common-env-variables
+      BACALHAU_API_HOST: 'bacalhau-requester-node'
+      BACALHAU_COMPUTE_NODE_HOST: 'bacalhau-compute-node'
+      BACALHAU_MINIO_NODE_HOST: 'bacalhau-minio-node'

test_integration/assets/dockerfiles/Dockerfile-ComputeNode

@@ -0,0 +1,25 @@
+# Use the docker:dind image as the base image
+FROM docker:dind
+
+# Set the working directory
+WORKDIR /app
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Update CA certificates
+RUN update-ca-certificates
+
+# Download and execute the Bash script from the given URL
+COPY ./bacalhau_bin /usr/local/bin/bacalhau
+RUN chmod +x /usr/local/bin/bacalhau
+
+COPY compute_node_image_setup.sh compute_node_image_setup.sh
+ENTRYPOINT ["/usr/bin/env"]
+CMD ./compute_node_image_setup.sh

test_integration/assets/dockerfiles/Dockerfile-DockerImageRegistryNode

@@ -0,0 +1,24 @@
+FROM registry:2
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Create a directory to store certificates to be used by the registry
+RUN mkdir /certs
+
+# Copy the certificate and key from the local directory to /certs
+COPY certificates/generated_assets/bacalhau-container-img-registry-node.crt /certs/
+COPY certificates/generated_assets/bacalhau-container-img-registry-node.key /certs/
+
+# Ensure proper permissions for certs
+RUN chmod 600 /certs/bacalhau-container-img-registry-node.key
+RUN chmod 644 /certs/bacalhau-container-img-registry-node.crt
+
+# Expose the registry's default port
+EXPOSE 5000 443

test_integration/assets/dockerfiles/Dockerfile-JumpboxNode

@@ -0,0 +1,28 @@
+# Use the docker:dind image as the base image
+FROM docker:dind
+
+# Set the working directory
+WORKDIR /app
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Update CA certificates
+RUN update-ca-certificates
+
+# Download and execute the Bash script from the given URL
+COPY ./bacalhau_bin /usr/local/bin/bacalhau
+RUN chmod +x /usr/local/bin/bacalhau
+
+# Download the minio cli binary, make it executable, and move it to /usr/local/bin
+RUN curl -o /tmp/mc https://dl.min.io/client/mc/release/linux-amd64/mc \
+    && chmod +x /tmp/mc \
+    && mv /tmp/mc /usr/local/bin/
+
+ENTRYPOINT ["dockerd-entrypoint.sh"]

test_integration/assets/dockerfiles/Dockerfile-RequesterNode

@@ -0,0 +1,23 @@
+# Use the docker:dind image as the base image
+FROM docker:dind
+
+# Set the working directory
+WORKDIR /app
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Update CA certificates
+RUN update-ca-certificates
+
+# Download and execute the Bash script from the given URL
+COPY ./bacalhau_bin /usr/local/bin/bacalhau
+RUN chmod +x /usr/local/bin/bacalhau
+
+ENTRYPOINT ["dockerd-entrypoint.sh"]

test_integration/assets/dockerfiles/certificates/README.md

@@ -0,0 +1,9 @@
+# Certificate Generation
+
+The script in the folder allows you to generate certificates that are signed by a root CA, and provide the
+CN and SAN for these leaf certs. The generated certs will be added to the `generated_assets` directory.
+
+Usage: `./generate_leaf_certs.sh <CN_and_SAN>`
+```shell
+./generate_leaf_certs.sh my-bacalhau-requester-node
+```