Merge pull request #856 from kradalby/integration-v2

Integration tests v2

.github/workflows/test-integration-cli.yml

@@ -3,7 +3,7 @@ name: CI
 on: [pull_request]
 
 jobs:
-  integration-test:
+  integration-test-cli:
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/test-integration-derp.yml

@@ -3,7 +3,7 @@ name: CI
 on: [pull_request]
 
 jobs:
-  integration-test:
+  integration-test-derp:
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/test-integration-general.yml

@@ -3,7 +3,7 @@ name: CI
 on: [pull_request]
 
 jobs:
-  integration-test:
+  integration-test-general:
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/test-integration-oidc.yml

@@ -3,7 +3,7 @@ name: CI
 on: [pull_request]
 
 jobs:
-  integration-test:
+  integration-test-oidc:
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/test-integration-v2-general.yml

@@ -0,0 +1,40 @@
+name: CI
+
+on: [pull_request]
+
+jobs:
+  integration-test-v2-general:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v14.1
+        with:
+          files: |
+            *.nix
+            go.*
+            **/*.go
+            integration_test/
+            config-example.yaml
+
+      - uses: cachix/install-nix-action@v16
+        if: steps.changed-files.outputs.any_changed == 'true'
+
+      - name: Run general integration tests
+        if: steps.changed-files.outputs.any_changed == 'true'
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 240
+          max_attempts: 5
+          retry_on: error
+          command: nix develop --command -- make test_integration_v2_general

Makefile

@@ -66,6 +66,16 @@ test_integration_oidc:
 		-v /var/run/docker.sock:/var/run/docker.sock golang:1 \
 		go test -failfast -timeout 30m -count=1 -run IntegrationOIDC ./...
 
+test_integration_v2_general:
+	docker run \
+		-t --rm \
+		-v ~/.cache/hs-integration-go:/go \
+		--name headscale-test-suite \
+		-v $$PWD:$$PWD -w $$PWD/integration \
+		-v /var/run/docker.sock:/var/run/docker.sock \
+		golang:1 \
+		go test ./... -timeout 15m -v
+
 coverprofile_func:
 	go tool cover -func=coverage.out
 

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.5
 	github.com/ccding/go-stun/stun v0.0.0-20200514191101-4dc67bcdb029
+	github.com/cenkalti/backoff/v4 v4.1.3
 	github.com/coreos/go-oidc/v3 v3.3.0
 	github.com/deckarep/golang-set/v2 v2.1.0
 	github.com/efekarakus/termcolor v1.0.1
@@ -54,7 +55,6 @@ require (
 	github.com/akutz/memconn v0.1.0 // indirect
 	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/containerd/console v1.0.3 // indirect
 	github.com/containerd/continuity v0.3.0 // indirect

integration/control.go

@@ -0,0 +1,13 @@
+package integration
+
+import v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
+
+type ControlServer interface {
+	Shutdown() error
+	GetHealthEndpoint() string
+	GetEndpoint() string
+	WaitForReady() error
+	CreateNamespace(namespace string) error
+	CreateAuthKey(namespace string) (*v1.PreAuthKey, error)
+	ListNodes(namespace string) ([]*v1.Machine, error)
+}

integration/dockertestutil/config.go

@@ -0,0 +1,40 @@
+package dockertestutil
+
+import (
+	"os"
+
+	"github.com/ory/dockertest/v3/docker"
+)
+
+func IsRunningInContainer() bool {
+	if _, err := os.Stat("/.dockerenv"); err != nil {
+		return false
+	}
+
+	return true
+}
+
+func DockerRestartPolicy(config *docker.HostConfig) {
+	// set AutoRemove to true so that stopped container goes away by itself on error *immediately*.
+	// when set to false, containers remain until the end of the integration test.
+	config.AutoRemove = false
+	config.RestartPolicy = docker.RestartPolicy{
+		Name: "no",
+	}
+}
+
+func DockerAllowLocalIPv6(config *docker.HostConfig) {
+	if config.Sysctls == nil {
+		config.Sysctls = make(map[string]string, 1)
+	}
+	config.Sysctls["net.ipv6.conf.all.disable_ipv6"] = "0"
+}
+
+func DockerAllowNetworkAdministration(config *docker.HostConfig) {
+	config.CapAdd = append(config.CapAdd, "NET_ADMIN")
+	config.Mounts = append(config.Mounts, docker.HostMount{
+		Type:   "bind",
+		Source: "/dev/net/tun",
+		Target: "/dev/net/tun",
+	})
+}

integration/dockertestutil/execute.go

@@ -0,0 +1,94 @@
+package dockertestutil
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/ory/dockertest/v3"
+)
+
+const dockerExecuteTimeout = time.Second * 10
+
+var (
+	ErrDockertestCommandFailed  = errors.New("dockertest command failed")
+	ErrDockertestCommandTimeout = errors.New("dockertest command timed out")
+)
+
+type ExecuteCommandConfig struct {
+	timeout time.Duration
+}
+
+type ExecuteCommandOption func(*ExecuteCommandConfig) error
+
+func ExecuteCommandTimeout(timeout time.Duration) ExecuteCommandOption {
+	return ExecuteCommandOption(func(conf *ExecuteCommandConfig) error {
+		conf.timeout = timeout
+
+		return nil
+	})
+}
+
+func ExecuteCommand(
+	resource *dockertest.Resource,
+	cmd []string,
+	env []string,
+	options ...ExecuteCommandOption,
+) (string, string, error) {
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	execConfig := ExecuteCommandConfig{
+		timeout: dockerExecuteTimeout,
+	}
+
+	for _, opt := range options {
+		if err := opt(&execConfig); err != nil {
+			return "", "", fmt.Errorf("execute-command/options: %w", err)
+		}
+	}
+
+	type result struct {
+		exitCode int
+		err      error
+	}
+
+	resultChan := make(chan result, 1)
+
+	// Run your long running function in it's own goroutine and pass back it's
+	// response into our channel.
+	go func() {
+		exitCode, err := resource.Exec(
+			cmd,
+			dockertest.ExecOptions{
+				Env:    append(env, "HEADSCALE_LOG_LEVEL=disabled"),
+				StdOut: &stdout,
+				StdErr: &stderr,
+			},
+		)
+		resultChan <- result{exitCode, err}
+	}()
+
+	// Listen on our channel AND a timeout channel - which ever happens first.
+	select {
+	case res := <-resultChan:
+		if res.err != nil {
+			return stdout.String(), stderr.String(), res.err
+		}
+
+		if res.exitCode != 0 {
+			// Uncomment for debugging
+			// log.Println("Command: ", cmd)
+			// log.Println("stdout: ", stdout.String())
+			// log.Println("stderr: ", stderr.String())
+
+			return stdout.String(), stderr.String(), ErrDockertestCommandFailed
+		}
+
+		return stdout.String(), stderr.String(), nil
+	case <-time.After(execConfig.timeout):
+
+		return stdout.String(), stderr.String(), ErrDockertestCommandTimeout
+	}
+}

integration/dockertestutil/network.go

@@ -0,0 +1,62 @@
+package dockertestutil
+
+import (
+	"errors"
+
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+)
+
+var ErrContainerNotFound = errors.New("container not found")
+
+func GetFirstOrCreateNetwork(pool *dockertest.Pool, name string) (*dockertest.Network, error) {
+	networks, err := pool.NetworksByName(name)
+	if err != nil || len(networks) == 0 {
+		if _, err := pool.CreateNetwork(name); err == nil {
+			// Create does not give us an updated version of the resource, so we need to
+			// get it again.
+			networks, err := pool.NetworksByName(name)
+			if err != nil {
+				return nil, err
+			}
+
+			return &networks[0], nil
+		}
+	}
+
+	return &networks[0], nil
+}
+
+func AddContainerToNetwork(
+	pool *dockertest.Pool,
+	network *dockertest.Network,
+	testContainer string,
+) error {
+	containers, err := pool.Client.ListContainers(docker.ListContainersOptions{
+		All: true,
+		Filters: map[string][]string{
+			"name": {testContainer},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	err = pool.Client.ConnectNetwork(network.Network.ID, docker.NetworkConnectionOptions{
+		Container: containers[0].ID,
+	})
+	if err != nil {
+		return err
+	}
+
+	// TODO(kradalby): This doesnt work reliably, but calling the exact same functions
+	// seem to work fine...
+	// if container, ok := pool.ContainerByName("/" + testContainer); ok {
+	// 	err := container.ConnectToNetwork(network)
+	// 	if err != nil {
+	// 		return err
+	// 	}
+	// }
+
+	return nil
+}