ci: make redis cluster default for integration tests (#1083)

Dockerfile

@@ -11,7 +11,7 @@ ARG NODE_VERSION=lts-alpine
 #
 
 # Build Image
-FROM node:${NODE_VERSION} as builder
+FROM node:${NODE_VERSION} AS builder
 
 WORKDIR /opt/app
 

config/default.json

@@ -86,10 +86,11 @@
   },
   "PROXY_CACHE": {
     "enabled": true,
-    "type": "redis",
+    "type": "redis-cluster",
     "proxyConfig": {
-      "host": "localhost",
-      "port": 6379
+      "cluster": [
+        { "host": "localhost", "port": 6379 }
+      ]
     }
   },
   "API_DOC_ENDPOINTS_ENABLED": true,

docker-compose.yml

@@ -1,9 +1,20 @@
-version: "3.7"
-
 networks:
   cl-mojaloop-net:
     name: cl-mojaloop-net
 
+# @see https://uninterrupted.tech/blog/hassle-free-redis-cluster-deployment-using-docker/
+x-redis-node: &REDIS_NODE
+  image: docker.io/bitnami/redis-cluster:6.2.14
+  environment: &REDIS_ENVS
+    ALLOW_EMPTY_PASSWORD: yes
+    REDIS_CLUSTER_DYNAMIC_IPS: no
+    REDIS_CLUSTER_ANNOUNCE_IP: ${REDIS_CLUSTER_ANNOUNCE_IP}
+    REDIS_NODES: localhost:6379 localhost:6380 localhost:6381 localhost:6382 localhost:6383 localhost:6384
+  healthcheck:
+    test: [ "CMD", "redis-cli", "ping" ]
+    timeout: 2s
+  network_mode: host
+
 services:
   central-ledger:
     image: mojaloop/central-ledger:local
@@ -31,10 +42,14 @@ services:
       - CLEDG_MONGODB__DISABLED=false
     networks:
       - cl-mojaloop-net
+    extra_hosts:
+      - "redis-node-0:host-gateway"
     depends_on:
       - mysql
       - kafka
       - objstore
+      - redis-node-0
+      # - redis
     healthcheck:
       test: ["CMD", "sh", "-c" ,"apk --no-cache add curl", "&&", "curl", "http://localhost:3001/health"]
       timeout: 20s
@@ -95,20 +110,77 @@ services:
       start_period: 40s
       interval: 30s
 
-  redis:
-    image: redis:6.2.4-alpine
-    restart: "unless-stopped"
+  redis-node-0:
+    <<: *REDIS_NODE
+    container_name: cl_redis-node-0
     environment:
-      - ALLOW_EMPTY_PASSWORD=yes
-      - REDIS_PORT=6379
-      - REDIS_REPLICATION_MODE=master
-      - REDIS_TLS_ENABLED=no
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
+      <<: *REDIS_ENVS
+      REDIS_CLUSTER_CREATOR: yes
+      REDIS_PORT_NUMBER: 6379
+    depends_on:
+      - redis-node-1
+      - redis-node-2
+      - redis-node-3
+      - redis-node-4
+      - redis-node-5
+  redis-node-1:
+    <<: *REDIS_NODE
+    container_name: cl_redis-node-1
+    environment:
+      <<: *REDIS_ENVS
+      REDIS_PORT_NUMBER: 6380
     ports:
-      - "6379:6379"
-    networks:
-      - cl-mojaloop-net
+      - "16380:16380"
+  redis-node-2:
+    <<: *REDIS_NODE
+    container_name: cl_redis-node-2
+    environment:
+      <<: *REDIS_ENVS
+      REDIS_PORT_NUMBER: 6381
+    ports:
+      - "16381:16381"
+  redis-node-3:
+    <<: *REDIS_NODE
+    container_name: cl_redis-node-3
+    environment:
+      <<: *REDIS_ENVS
+      REDIS_PORT_NUMBER: 6382
+    ports:
+      - "16382:16382"
+  redis-node-4:
+    <<: *REDIS_NODE
+    container_name: cl_redis-node-4
+    environment:
+      <<: *REDIS_ENVS
+      REDIS_PORT_NUMBER: 6383
+    ports:
+      - "16383:16383"
+  redis-node-5:
+    <<: *REDIS_NODE
+    container_name: cl_redis-node-5
+    environment:
+      <<: *REDIS_ENVS
+      REDIS_PORT_NUMBER: 6384
+    ports:
+      - "16384:16384"
+
+##  To be used with proxyCache.type === 'redis'
+#  redis:
+#    image: redis:6.2.4-alpine
+#    restart: "unless-stopped"
+#    environment:
+#      <<: *REDIS_ENVS
+#      REDIS_CLUSTER_CREATOR: yes
+#    depends_on:
+#      - redis-node-1
+#      - redis-node-2
+#      - redis-node-3
+#      - redis-node-4
+#      - redis-node-5
+#    ports:
+#      - "6379:6379"
+#    networks:
+#      - cl-mojaloop-net
 
   mockserver:
     image: jamesdbloom/mockserver

docker/central-ledger/default.json

@@ -84,10 +84,11 @@
   },
   "PROXY_CACHE": {
     "enabled": true,
-    "type": "redis",
+    "type": "redis-cluster",
     "proxyConfig": {
-      "host": "redis",
-      "port": 6379
+      "cluster": [
+        { "host": "redis-node-0", "port": 6379 }
+      ]
     }
   },
   "KAFKA": {

docker/config-modifier/configs/central-ledger.js

@@ -16,6 +16,7 @@ module.exports = {
     enabled: true,
     type: 'redis',
     proxyConfig: {
+      cluster: undefined,
       host: 'redis',
       port: 6379
     }

docker/env.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# Retrieve the external IP address of the host machine (on macOS)
+# or the IP address of the docker0 interface (on Linux)
+get_external_ip() {
+  if [ "$(uname)" = "Linux" ]; then
+    echo "$(ip addr show docker0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)"
+  else
+    # Need to find a way to support Windows here
+    echo "$(route get ifconfig.me | grep interface | sed -e 's/.*: //' | xargs ipconfig getifaddr)"
+  fi
+}
+
+# set/override dynamic variables
+export REDIS_CLUSTER_ANNOUNCE_IP=$(get_external_ip)

package.json

@@ -63,7 +63,7 @@
     "migrate:current": "npx knex migrate:currentVersion $npm_package_config_knex",
     "seed:run": "npx knex seed:run $npm_package_config_knex",
     "docker:build": "docker build --build-arg NODE_VERSION=\"$(cat .nvmrc)-alpine\" -t mojaloop/central-ledger:local .",
-    "docker:up": "docker-compose -f docker-compose.yml up",
+    "docker:up": ". ./docker/env.sh && docker-compose -f docker-compose.yml up",
     "docker:up:backend": "docker-compose up -d ml-api-adapter mysql mockserver kafka kowl temp_curl",
     "docker:up:int": "docker compose up -d kafka init-kafka objstore mysql",
     "docker:script:populateTestData": "sh ./test/util/scripts/populateTestData.sh",
@@ -96,7 +96,7 @@
     "@mojaloop/central-services-stream": "11.3.1",
     "@mojaloop/database-lib": "11.0.6",
     "@mojaloop/event-sdk": "14.1.1",
-    "@mojaloop/inter-scheme-proxy-cache-lib": "^2.2.0",
+    "@mojaloop/inter-scheme-proxy-cache-lib": "2.3.0",
     "@mojaloop/ml-number": "11.2.4",
     "@mojaloop/object-store-lib": "12.0.3",
     "@now-ims/hapi-now-auth": "2.1.0",

test-integration.Dockerfile

@@ -2,7 +2,7 @@
 ARG NODE_VERSION=lts-alpine
 
 # Build Image
-FROM node:${NODE_VERSION} as builder
+FROM node:${NODE_VERSION} AS builder
 
 USER root
 

test.Dockerfile

@@ -2,7 +2,7 @@
 ARG NODE_VERSION=lts-alpine
 
 # Build Image
-FROM node:${NODE_VERSION} as builder
+FROM node:${NODE_VERSION} AS builder
 
 USER root
 

test/scripts/test-integration.sh

@@ -18,10 +18,13 @@ TTK_FUNC_TEST_EXIT_CODE=1
 ## Make reports directory
 mkdir ./test/results
 
+## Set environment variables
+source ./docker/env.sh
+
 ## Start backend services
 echo "==> Starting Docker backend services"
-docker compose pull mysql kafka init-kafka redis
-docker compose up -d mysql kafka init-kafka redis
+docker compose pull mysql kafka init-kafka redis-node-0
+docker compose up -d mysql kafka init-kafka redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5
 docker compose ps
 npm run wait-4-docker