try jd

.github/workflows/framework-golden-tests-private.yml

@@ -0,0 +1,78 @@
+name: Framework Golden Tests Examples (Private)
+on:
+  push:
+
+jobs:
+  test:
+    defaults:
+      run:
+        working-directory: framework/examples/myproject_cll
+    env:
+      LOKI_TENANT_ID: promtail
+      LOKI_URL: http://localhost:3030/loki/api/v1/push
+      # this is not the best practice, and it must be fixed, run your tests WITHOUT IT!
+      # however, on current latest image we must use this flag
+      CTF_IGNORE_CRITICAL_LOGS: true
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          - name: TestJD
+            config: jd.toml
+            count: 1
+            timeout: 10m
+    name: ${{ matrix.test.name }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      - name: Configure AWS credentials using OIDC
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
+        with:
+          role-to-assume: ${{ secrets.PUBLIC_AWS_ECR_ROLE }}
+          aws-region: us-east-1
+      - name: Authenticate to ECR Public
+        id: login-ecr-public
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
+        with:
+          registry-type: public
+      - name: Check for changes in Framework
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        id: changes
+        with:
+          filters: |
+            src:
+              - 'framework/**'
+              - '.github/workflows/framework-golden-tests.yml'
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.22.8
+      - name: Cache Go modules
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: go-modules-${{ hashFiles('framework/examples/myproject_cll/go.sum') }}-${{ runner.os }}-framework-golden-examples
+          restore-keys: |
+            go-modules-${{ runner.os }}-framework-golden-examples
+            go-modules-${{ runner.os }}
+      - name: Install dependencies
+        run: go mod download
+      - name: Run System Tests
+        if: steps.changes.outputs.src == 'true'
+        env:
+          CTF_CONFIGS: ${{ matrix.test.config }}
+        run: |
+          go test -timeout ${{ matrix.test.timeout }} -v -count ${{ matrix.test.count }} -run ${{ matrix.test.name }}
+      - name: Upload Logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: container-logs-${{ matrix.test.name }}
+          path: framework/examples/myproject_cll/logs
+          retention-days: 1

book/src/framework/components/state.md

@@ -40,7 +40,7 @@ One node set is enough for any kind of testing, if you need more nodes consider
 
 ## Custom ports
 
-You can also define a custom set of ports for any node
+You can also define a custom set of ports for any node.
 ```toml
 [nodeset]
   nodes = 5
@@ -53,6 +53,7 @@ You can also define a custom set of ports for any node
 
     [nodeset.node_specs.node]
       # here we defined 2 new ports to listen and mapped them to our host machine
-      custom_ports = [14000, 14001]
+      # syntax is "host:docker", if you provide only host port then we map 1-to-1
+      custom_ports = ["14000:15000", "20000"]
       image = "public.ecr.aws/chainlink/chainlink:v2.16.0"
 ```

framework/.changeset/v0.3.2.md

@@ -0,0 +1,5 @@
+- Allow exposing CL node ports in host:docker format
+- Expose default test private keys as framework constants
+- Rename CHAINLINK_IMAGE to CTF_CHAINLINK_IMAGE to avoid CI collisions
+- Add CTF_JD_IMAGE env var
+- Add JobDistributor component

framework/components/clnode/clnode.go

@@ -112,6 +112,54 @@ func NewNode(in *Input, pgOut *postgres.Output) (*Output, error) {
 	return out, nil
 }
 
+// generatePortBindings generates exposed ports and port bindings
+// exposes default CL node port
+// exposes custom_ports in format "host:docker" or map 1-to-1 if only "host" port is provided
+func generatePortBindings(in *Input) ([]string, nat.PortMap, error) {
+	httpPort := fmt.Sprintf("%s/tcp", DefaultHTTPPort)
+	portBindings := nat.PortMap{
+		nat.Port(httpPort): []nat.PortBinding{
+			{
+				HostIP:   "0.0.0.0",
+				HostPort: fmt.Sprintf("%d/tcp", in.Node.HTTPPort),
+			},
+		},
+	}
+	customPorts := make([]string, 0)
+	for _, p := range in.Node.CustomPorts {
+		if strings.Contains(p, CustomPortSeparator) {
+			pp := strings.Split(p, CustomPortSeparator)
+			if len(pp) != 2 {
+				return nil, nil, errors.New("custom_ports has ':' but you must provide both ports")
+			}
+			customPorts = append(customPorts, fmt.Sprintf("%s/tcp", pp[1]))
+
+			dockerPort := nat.Port(fmt.Sprintf("%s/tcp", pp[1]))
+			hostPort := fmt.Sprintf("%s/tcp", pp[0])
+			portBindings[dockerPort] = []nat.PortBinding{
+				{
+					HostIP:   "0.0.0.0",
+					HostPort: hostPort,
+				},
+			}
+		} else {
+			customPorts = append(customPorts, fmt.Sprintf("%s/tcp", p))
+
+			dockerPort := nat.Port(fmt.Sprintf("%s/tcp", p))
+			hostPort := fmt.Sprintf("%s/tcp", p)
+			portBindings[dockerPort] = []nat.PortBinding{
+				{
+					HostIP:   "0.0.0.0",
+					HostPort: hostPort,
+				},
+			}
+		}
+	}
+	exposedPorts := []string{httpPort}
+	exposedPorts = append(exposedPorts, customPorts...)
+	return exposedPorts, portBindings, nil
+}
+
 func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 	ctx := context.Background()
 
@@ -148,52 +196,17 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 		return nil, err
 	}
 
-	httpPort := fmt.Sprintf("%s/tcp", DefaultHTTPPort)
 	var containerName string
 	if in.Node.Name != "" {
 		containerName = in.Node.Name
 	} else {
 		containerName = framework.DefaultTCName("node")
 	}
 
-	portBindings := nat.PortMap{
-		nat.Port(httpPort): []nat.PortBinding{
-			{
-				HostIP:   "0.0.0.0",
-				HostPort: fmt.Sprintf("%d/tcp", in.Node.HTTPPort),
-			},
-		},
-	}
-	customPorts := make([]string, 0)
-	for _, p := range in.Node.CustomPorts {
-		if strings.Contains(p, CustomPortSeparator) {
-			pp := strings.Split(p, CustomPortSeparator)
-			if len(pp) != 2 {
-				return nil, errors.New("custom_ports has ':' but you must provide both ports")
-			}
-			customPorts = append(customPorts, fmt.Sprintf("%s/tcp", pp[1]))
-			portBindings[nat.Port(fmt.Sprintf("%s/tcp", pp[1]))] = []nat.PortBinding{
-				{
-					HostIP:   "0.0.0.0",
-					HostPort: fmt.Sprintf("%s/tcp", pp[0]),
-				},
-			}
-			framework.L.Warn().Str("PortHost", pp[0]).Str("PortInternal", pp[1]).Send()
-		} else {
-			customPorts = append(customPorts, fmt.Sprintf("%s/tcp", p))
-			framework.L.Warn().Str("Port", p).Send()
-			portBindings[nat.Port(fmt.Sprintf("%s/tcp", p))] = []nat.PortBinding{
-				{
-					HostIP:   "0.0.0.0",
-					HostPort: fmt.Sprintf("%s/tcp", p),
-				},
-			}
-		}
+	exposedPorts, portBindings, err := generatePortBindings(in)
+	if err != nil {
+		return nil, err
 	}
-	exposedPorts := []string{httpPort}
-	exposedPorts = append(exposedPorts, customPorts...)
-	framework.L.Warn().Any("ExposedPorts", exposedPorts).Send()
-
 	req := tc.ContainerRequest{
 		AlwaysPullImage: in.Node.PullImage,
 		Image:           in.Node.Image,

framework/components/jd/jd.go

@@ -0,0 +1,113 @@
+package jd
+
+import (
+	"context"
+	"fmt"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/go-connections/nat"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	tc "github.com/testcontainers/testcontainers-go"
+	tcwait "github.com/testcontainers/testcontainers-go/wait"
+	"os"
+)
+
+const (
+	TmpImageName            = "jd-local"
+	GRPCPort         string = "42242"
+	CSAEncryptionKey string = "!PASsword000!"
+	WSRPCPort        string = "8080"
+)
+
+type Input struct {
+	Image            string  `toml:"image"`
+	GRPCPort         string  `toml:"grpc_port"`
+	WSRPCPort        string  `toml:"wsrpc_port"`
+	DBURL            string  `toml:"db_url"`
+	CSAEncryptionKey string  `toml:"csa_encryption_key"`
+	DockerFilePath   string  `toml:"docker_file"`
+	DockerContext    string  `toml:"docker_ctx"`
+	Out              *Output `toml:"out"`
+}
+
+type Output struct {
+	UseCache       bool   `toml:"use_cache"`
+	HostGRPCUrl    string `toml:"grpc_url"`
+	DockerGRPCUrl  string `toml:"docker_internal_grpc_url"`
+	HostWSRPCUrl   string `toml:"wsrpc_url"`
+	DockerWSRPCUrl string `toml:"docker_internal_wsrpc_url"`
+}
+
+func defaults(in *Input) {
+	if in.GRPCPort == "" {
+		in.GRPCPort = GRPCPort
+	}
+	if in.WSRPCPort == "" {
+		in.WSRPCPort = WSRPCPort
+	}
+	if in.CSAEncryptionKey == "" {
+		in.CSAEncryptionKey = CSAEncryptionKey
+	}
+}
+
+func NewJD(in *Input) (*Output, error) {
+	if in.Out != nil && in.Out.UseCache {
+		return in.Out, nil
+	}
+	ctx := context.Background()
+	defaults(in)
+	jdImg := os.Getenv("CTF_JD_IMAGE")
+	if jdImg != "" {
+		in.Image = jdImg
+	}
+	containerName := framework.DefaultTCName("jd")
+	bindPort := fmt.Sprintf("%s/tcp", in.GRPCPort)
+	req := tc.ContainerRequest{
+		Name:     containerName,
+		Image:    in.Image,
+		Labels:   framework.DefaultTCLabels(),
+		Networks: []string{framework.DefaultNetworkName},
+		NetworkAliases: map[string][]string{
+			framework.DefaultNetworkName: {containerName},
+		},
+		ExposedPorts: []string{bindPort},
+		HostConfigModifier: func(h *container.HostConfig) {
+			h.PortBindings = framework.MapTheSamePort(bindPort)
+		},
+		Env: map[string]string{
+			"DATABASE_URL":              in.DBURL,
+			"PORT":                      in.GRPCPort,
+			"NODE_RPC_PORT":             in.WSRPCPort,
+			"CSA_KEY_ENCRYPTION_SECRET": in.CSAEncryptionKey,
+		},
+		WaitingFor: tcwait.ForAll(
+			tcwait.ForListeningPort(nat.Port(fmt.Sprintf("%s/tcp", in.GRPCPort))),
+		),
+	}
+	if req.Image == "" {
+		req.Image = TmpImageName
+		if err := framework.BuildImage(in.DockerContext, in.DockerFilePath, req.Image); err != nil {
+			return nil, err
+		}
+		req.KeepImage = false
+	}
+	c, err := tc.GenericContainer(ctx, tc.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	host, err := framework.GetHost(c)
+	if err != nil {
+		return nil, err
+	}
+	out := &Output{
+		UseCache:       true,
+		HostGRPCUrl:    fmt.Sprintf("http://%s:%s", host, in.GRPCPort),
+		DockerGRPCUrl:  fmt.Sprintf("http://%s:%s", containerName, in.GRPCPort),
+		HostWSRPCUrl:   fmt.Sprintf("ws://%s:%s", host, in.WSRPCPort),
+		DockerWSRPCUrl: fmt.Sprintf("ws://%s:%s", containerName, in.WSRPCPort),
+	}
+	in.Out = out
+	return out, nil
+}

framework/components/jd/jd_test.go

@@ -0,0 +1,30 @@
+package jd_test
+
+import (
+	"github.com/davecgh/go-spew/spew"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/jd"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/postgres"
+	"github.com/stretchr/testify/require"
+	"os"
+	"sync"
+	"testing"
+)
+
+// here we only test that we can boot up JD
+// client examples are under "examples" dir
+// since JD is private this env var should be set locally and in CI
+func TestJD(t *testing.T) {
+	err := framework.DefaultNetwork(&sync.Once{})
+	require.NoError(t, err)
+	pgOut, err := postgres.NewPostgreSQL(&postgres.Input{
+		Image: "postgres:12.0",
+	})
+	require.NoError(t, err)
+	out, err := jd.NewJD(&jd.Input{
+		DBURL: pgOut.JDDockerInternalURL,
+		Image: os.Getenv("CTF_JD_IMAGE"),
+	})
+	require.NoError(t, err)
+	spew.Dump(out)
+}