Update docs (#1317)

docs update
smartcontractkit · Nov 11, 2024 · 0e411f1 · 0e411f1
1 parent 7f2e997
commit 0e411f1
Show file tree

Hide file tree

Showing 14 changed files with 179 additions and 33 deletions.
diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md
@@ -15,10 +15,10 @@
     - [Configuration](./framework/configuration.md)
     - [Test Configuration](./framework/test_configuration_overrides.md)
     - [Components Persistence](framework/components/state.md)
+    - [Components Cleanup](framework/components/cleanup.md)
     - [Components Caching](framework/components/caching.md)
     - [External Environment](framework/components/external.md)
     - [Secrets]()
-    - [Docker](framework/docker.md)
     - [Observability Stack](framework/observability/observability_stack.md)
       - [Metrics](framework/observability/metrics.md)
       - [Logs](framework/observability/logs.md)

diff --git a/book/src/framework/components/cleanup.md b/book/src/framework/components/cleanup.md
@@ -0,0 +1,23 @@
+# Components Cleanup
+
+Managing state is challenging, especially in end-to-end testing, we use [ryuk](https://golang.testcontainers.org/features/garbage_collector/#ryuk) and following simple rules:
+- If `TESTCONTAINERS_RYUK_DISABLED=true`, no cleanup occurs — containers, volumes, and networks remain on your machine.
+
+  Feel free to use `ctf d rm` to remove containers when you are ready.
+- If `TESTCONTAINERS_RYUK_DISABLED` is unset, the test environment will be automatically cleaned up a few seconds after the test completes.
+
+
+Keep in mind that all components are mapped to [static ports](state.md), so without cleanup, only one environment can run at a time.
+
+This design choice simplifies debugging.
+
+A simplified command is available to prune unused volumes, containers, and build caches. Use it when you’re running low on space on your machine.
+```
+ctf d c
+```
+
+<div class="warning">
+
+The framework manages cleanup for both on-chain and off-chain Docker components. However, if your test involves actions like configuring Chainlink jobs, it's best practice to make these actions idempotent, so they can be applied reliably in any environment.
+
+</div>
diff --git a/book/src/framework/docker.md b/book/src/framework/docker.md
diff --git a/book/src/framework/first_test.md b/book/src/framework/first_test.md
@@ -46,7 +46,7 @@ Run the test
 CTF_CONFIGS=smoke.toml go test -v -run TestMe
 ```
 
-Remove containers
+Remove containers (read more about cleanup [here](components/cleanup.md))
 ```
 ctf d rm
 ```

diff --git a/book/src/framework/getting_started.md b/book/src/framework/getting_started.md
@@ -37,7 +37,7 @@ Allow it to run in `System Settings -> Security Settings` (OS X)
 ![img.png](images/img.png)
 
 
-Create an `.envrc` file and do `source .envrc`
+Create an `.envrc` file and do `source .envrc` (we recommend to use [direnv](https://direnv.net/), so you don't need to load it every time)
 ```
 export TESTCONTAINERS_RYUK_DISABLED=true # do not remove containers while we develop locally
 ```

diff --git a/book/src/overview.md b/book/src/overview.md
@@ -1,19 +1,17 @@
 # Intro
 
-The Chainlink Testing Framework (CTF) is a blockchain development framework written in Go. 
 
-Its primary purpose is to help Chainlink developers create extensive integration, e2e, performance, and chaos tests to ensure the stability of the Chainlink project.
 
-It can also be helpful to those who just want to use Chainlink oracles in their projects to help test their contracts, or even for those that aren't using Chainlink.
+The Chainlink Testing Framework is a toolset designed for end-to-end testing of Chainlink products, focusing on functionality, resiliency, and performance.
 
-This documentation is primarily for:
-- Engineers looking to write end-to-end tests
-- Non-technical users or external developers who want to [setup](framework/interactive.md) `Chainlink` nodes locally
+This documentation is intended for:
+- Chainlink engineers writing end-to-end tests in [Golang](https://go.dev/)
+- Engineers using other languages who want to integrate with the Chainlink platform
 
 To get started with writing tests, refer to the [Framework](./framework/getting_started.md) chapter, where we guide you from basic to more complex scenarios.
 
+If you want to build integration with Chainlink not in [Golang](https://go.dev/), please refer to our [Interactive](framework/interactive.md) chapter.
+
 [Repository](https://github.com/smartcontractkit/chainlink-testing-framework) contains two major pieces:
 - [Framework](framework/overview.md)
 - [Libraries](libraries.md)
-
-If you're a non-technical user or want to build integration with Chainlink not in `Golang`, please refer to our [Interactive](framework/interactive.md) chapter.
diff --git a/framework/.changeset/v0.2.1.md b/framework/.changeset/v0.2.1.md
@@ -0,0 +1,4 @@
+- Add upgrade tests
+- Polish docs, add "cleanup" section
+- Add official CL images for 2.16, 2.17 + ARM versions
+- Do not fail if there is no default DB volume when cleanup
diff --git a/framework/cmd/docker.go b/framework/cmd/docker.go
@@ -9,13 +9,30 @@ import (
 	"strings"
 )
 
-func cleanDockerResources() error {
+func rmTestContainers() error {
 	framework.L.Info().Str("label", "framework=ctf").Msg("Cleaning up docker containers")
 	// Bash command for removing Docker containers and networks with "framework=ctf" label
 	cmd := exec.Command("bash", "-c", `
 		docker ps -aq --filter "label=framework=ctf" | xargs -r docker rm -f && \
 		docker network ls --filter "label=framework=ctf" -q | xargs -r docker network rm && \
-		docker volume rm postgresql_data
+		docker volume rm postgresql_data || true
+	`)
+	framework.L.Debug().Msg("Running command")
+	if framework.L.GetLevel() == zerolog.DebugLevel {
+		fmt.Println(cmd.String())
+	}
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("error running clean command: %s", string(output))
+	}
+	return nil
+}
+
+func cleanUpDockerResources() error {
+	framework.L.Info().Msg("Cleaning up docker resources")
+	// Bash command for removing Docker containers and networks with "framework=ctf" label
+	cmd := exec.Command("bash", "-c", `
+		docker system prune -a --volumes -f
 	`)
 	framework.L.Debug().Msg("Running command")
 	if framework.L.GetLevel() == zerolog.DebugLevel {

diff --git a/framework/cmd/interactive.go b/framework/cmd/interactive.go
@@ -93,7 +93,7 @@ func createComponentsFromForm(form *nodeSetForm) error {
 func cleanup(form *nodeSetForm) error {
 	var err error
 	f := func() {
-		err = cleanDockerResources()
+		err = rmTestContainers()
 	}
 	err = spinner.New().
 		Title("Removing docker resources..").
@@ -143,7 +143,11 @@ Docker Desktop (https://www.docker.com/products/docker-desktop/)
 			huh.NewSelect[string]().
 				Title("Choose Chainlink node version").
 				Options(
-					huh.NewOption("public.ecr.aws/chainlink/chainlink:v2.17.0", "public.ecr.aws/chainlink/chainlink:v2.17.0")).
+					huh.NewOption("public.ecr.aws/chainlink/chainlink:v2.17.0-arm64", "public.ecr.aws/chainlink/chainlink:v2.17.0-arm64"),
+					huh.NewOption("public.ecr.aws/chainlink/chainlink:v2.17.0", "public.ecr.aws/chainlink/chainlink:v2.17.0"),
+					huh.NewOption("public.ecr.aws/chainlink/chainlink:v2.16.0-arm64", "public.ecr.aws/chainlink/chainlink:v2.16.0-arm64"),
+					huh.NewOption("public.ecr.aws/chainlink/chainlink:v2.16.0", "public.ecr.aws/chainlink/chainlink:v2.16.0"),
+				).
 				Value(&f.CLVersion),
 			huh.NewConfirm().
 				Title("Do you need to spin up an observability stack?").

diff --git a/framework/cmd/main.go b/framework/cmd/main.go
@@ -63,12 +63,24 @@ func main() {
 				Aliases: []string{"d"},
 				Usage:   "Control docker containers marked with 'framework=ctf' label",
 				Subcommands: []*cli.Command{
+					{
+						Name:    "clean",
+						Aliases: []string{"c"},
+						Usage:   "Cleanup all docker resources: volumes, images, build caches",
+						Action: func(c *cli.Context) error {
+							err := cleanUpDockerResources()
+							if err != nil {
+								return fmt.Errorf("failed to clean Docker resources: %w", err)
+							}
+							return nil
+						},
+					},
 					{
 						Name:    "remove",
 						Aliases: []string{"rm"},
 						Usage:   "Remove Docker containers and networks with 'framework=ctf' label",
 						Action: func(c *cli.Context) error {
-							err := cleanDockerResources()
+							err := rmTestContainers()
 							if err != nil {
 								return fmt.Errorf("failed to clean Docker resources: %w", err)
 							}

diff --git a/framework/examples/myproject/.envrc b/framework/examples/myproject/.envrc
@@ -1,7 +1,7 @@
 export TESTCONTAINERS_RYUK_DISABLED=true
 export PRIVATE_KEY="..."
 # load test
-export LOKI_TENANT_ID=promtail
-export LOKI_URL=http://localhost:3030/loki/api/v1/push
+#export LOKI_TENANT_ID=promtail
+#export LOKI_URL=http://localhost:3030/loki/api/v1/push
 #export RESTY_DEBUG=true
-export CTF_LOG_LEVEL=debug
+export CTF_LOG_LEVEL=info
diff --git a/framework/examples/myproject/chaos_test.go b/framework/examples/myproject/chaos_test.go
@@ -18,6 +18,11 @@ type CfgChaos struct {
 	NodeSet            *ns.Input         `toml:"nodeset" validate:"required"`
 }
 
+func verifyServices(t *testing.T, c []*clclient.ChainlinkClient) {
+	_, _, err := c[0].ReadBridges()
+	require.NoError(t, err)
+}
+
 func TestChaos(t *testing.T) {
 	in, err := framework.Load[CfgChaos](t)
 	require.NoError(t, err)
@@ -32,13 +37,24 @@ func TestChaos(t *testing.T) {
 	c, err := clclient.NewCLDefaultClients(out.CLNodes, framework.L)
 	require.NoError(t, err)
 
-	t.Run("run the cluster and simulate slow network", func(t *testing.T) {
-		// example commands for Pumba:
-		// stop --duration=1s --restart re2:node0                                            # stop one container for 1s and restart
-		// netem --tc-image=gaiadocker/iproute2 --duration=1m delay --time=300 re2:node.*   # slow network
-		_, err = chaos.ExecPumba("stop --duration=1s --restart re2:node0", 1*time.Second)
+	t.Run("run the cluster and test various chaos scenarios", func(t *testing.T) {
+		// Here are examples of using Pumba (https://github.com/alexei-led/pumba)
+		// for simplicity we allow users to run commands "as is", read their docs to learn more
+		// second parameter is experiment wait time
+
+		// Restart the container
+		_, err = chaos.ExecPumba("stop --duration=20s --restart re2:node0", 30*time.Second)
 		require.NoError(t, err)
-		_, _, err = c[0].ReadBridges()
+		verifyServices(t, c)
+
+		// Simulate poor network with 1s delay
+		_, err = chaos.ExecPumba("netem --tc-image=gaiadocker/iproute2 --duration=1m delay --time=1000 re2:node.*", 30*time.Second)
 		require.NoError(t, err)
+		verifyServices(t, c)
+
+		// Stress container CPU (TODO: it is not portable, works only in CI or Linux VM, cgroups are required)
+		//_, err = chaos.ExecPumba(`stress --stress-image=alexeiled/stress-ng:latest-ubuntu --duration=30s --stressors="--cpu 1 --vm 2 --vm-bytes 1G" node0`, 30*time.Second)
+		//require.NoError(t, err)
+		//verifyServices(t, c)
 	})
 }
diff --git a/framework/examples/myproject/go.mod b/framework/examples/myproject/go.mod
@@ -8,7 +8,7 @@ replace (
 )
 
 require (
-	github.com/smartcontractkit/chainlink-testing-framework/framework v0.1.10
+	github.com/smartcontractkit/chainlink-testing-framework/framework v0.2.0
 	github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2
 	github.com/stretchr/testify v1.9.0
 )
@@ -27,6 +27,7 @@ require (
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
+	github.com/atotto/clipboard v0.1.4 // indirect
 	github.com/aws/aws-sdk-go v1.45.25 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.31.0 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.27.39 // indirect
@@ -42,16 +43,26 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.31.3 // indirect
 	github.com/aws/smithy-go v1.21.0 // indirect
+	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/bytedance/sonic v1.12.3 // indirect
 	github.com/bytedance/sonic/loader v0.2.0 // indirect
 	github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect
 	github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 // indirect
+	github.com/catppuccin/go v0.2.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/charmbracelet/bubbles v0.20.0 // indirect
+	github.com/charmbracelet/bubbletea v1.1.1 // indirect
+	github.com/charmbracelet/huh v0.6.0 // indirect
+	github.com/charmbracelet/huh/spinner v0.0.0-20241028115900-20a4d21717a8 // indirect
+	github.com/charmbracelet/lipgloss v0.13.0 // indirect
+	github.com/charmbracelet/x/ansi v0.2.3 // indirect
+	github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect
+	github.com/charmbracelet/x/term v0.2.0 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/coder/websocket v1.8.12 // indirect
@@ -61,6 +72,7 @@ require (
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/dockercfg v0.3.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dennwc/varint v1.0.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -71,6 +83,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/edsrzf/mmap-go v1.1.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/ethereum/go-ethereum v1.14.11 // indirect
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
 	github.com/fatih/color v1.16.0 // indirect
@@ -142,14 +155,18 @@ require (
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-localereader v0.0.1 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/miekg/dns v1.1.56 // indirect
 	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.1 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
@@ -160,6 +177,9 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
+	github.com/muesli/cancelreader v0.2.2 // indirect
+	github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
@@ -169,6 +189,7 @@ require (
 	github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
@@ -182,7 +203,9 @@ require (
 	github.com/prometheus/exporter-toolkit v0.10.1-0.20230714054209-2f4150c63f97 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/prometheus/prometheus v0.47.2-0.20231010075449-4b9c19fe5510 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rs/zerolog v1.33.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
 	github.com/sercand/kuberesolver/v5 v5.1.1 // indirect
 	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
@@ -202,7 +225,9 @@ require (
 	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/urfave/cli/v2 v2.27.5 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.7 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect