From 964f2ad05b0681996b4c9839f82c9ab1fbf3121d Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Thu, 26 Apr 2018 17:16:21 -0400 Subject: [PATCH] Expose knobs for tuning the Kafka replication & retained bytes --- larger.env | 3 +++ persistent-template.yml | 30 +++++++++++++++++++++++++----- template.yml | 20 ++++++++++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/larger.env b/larger.env index 298f738..80f6cb1 100644 --- a/larger.env +++ b/larger.env @@ -29,3 +29,6 @@ WHISK_ACTIONS_INVOKES_CONCURRENT=100 WHISK_ACTIONS_INVOKES_CONCURRENT_IN_SYSTEM=5000 WHISK_ACTIONS_INVOKES_PER_MINUTE=600 WHISK_TRIGGERS_FIRES_PER_MINUTE=600 +WHISK_KAFKA_REPLICATION_FACTOR=2 +WHISK_KAFKA_TOPICS_INVOKER_RETENTION_BYTES=536870912 +WHISK_KAFKA_TOPICS_COMPLETED_RETENTION_BYTES=536870912 diff --git a/persistent-template.yml b/persistent-template.yml index b973d04..3e6eb44 100644 --- a/persistent-template.yml +++ b/persistent-template.yml @@ -133,16 +133,16 @@ parameters: displayName: Number of Zookeper cluster nodes (odd number of nodes is recomended) required: true value: "1" -- name: KAFKA_NODE_COUNT - description: Number of Kafka cluster nodes which will be deployed - displayName: Number of Kafka cluster nodes - required: true - value: "1" - name: ZOOKEEPER_VOLUME_CAPACITY description: Volume space available for Zookeeper data, e.g. 512Mi, 2Gi. displayName: Zookeeper Volume Capacity required: true value: 1Gi +- name: KAFKA_NODE_COUNT + description: Number of Kafka cluster nodes which will be deployed + displayName: Number of Kafka cluster nodes + required: true + value: "1" - name: KAFKA_VOLUME_CAPACITY description: Volume space available for Kafka data, e.g. 512Mi, 2Gi. displayName: Kafka Volume Capacity @@ -194,6 +194,18 @@ parameters: description: Default memory an action requests if unspecified. value: "256 m" required: true +- name: WHISK_KAFKA_REPLICATION_FACTOR + description: Replication factor for all OpenWhisk topics + value: "1" + required: true +- name: WHISK_KAFKA_TOPICS_COMPLETED_RETENTION_BYTES + description: Maximum bytes to retain for the completed activations topic for each Controller. Each kafka node will consume this many bytes times the number of Controllers just for the completed topics. So, if this is set to 1GB and you have 3 Controllers then expect up to 3GB of disk space used by the Controller topics. + value: "268435456" + required: true +- name: WHISK_KAFKA_TOPICS_INVOKER_RETENTION_BYTES + description: Maximum bytes to retain for each Invoker topic. Each Kafka node will consume this many bytes times the number of Invokers just for the Invoker topics. So, if this is set to 1GB and you have 10 Invokers then expect up to 10GB of disk space used by the Invoker topics. + value: "268435456" + required: true - name: OPENWHISK_VERSION description: The DockerHub tag for openwhisk/{controller,invoker} value: "rhdemo-6ee19949" @@ -690,6 +702,10 @@ objects: value: "${WHISK_ACTIONS_MEMORY_MIN}" - name: "CONFIG_whisk_memory_std" value: "${WHISK_ACTIONS_MEMORY_STD}" + - name: "CONFIG_whisk_kafka_replicationFactor" + value: "${WHISK_KAFKA_REPLICATION_FACTOR}" + - name: "CONFIG_whisk_kafka_topics_completed_retentionBytes" + value: "${WHISK_KAFKA_TOPICS_COMPLETED_RETENTION_BYTES}" # extra JVM arguments - name: "JAVA_OPTS" @@ -1125,6 +1141,10 @@ objects: value: "${WHISK_ACTIONS_MEMORY_MIN}" - name: "CONFIG_whisk_memory_std" value: "${WHISK_ACTIONS_MEMORY_STD}" + - name: "CONFIG_whisk_kafka_replicationFactor" + value: "${WHISK_KAFKA_REPLICATION_FACTOR}" + - name: "CONFIG_whisk_kafka_topics_invoker_retentionBytes" + value: "${WHISK_KAFKA_TOPICS_INVOKER_RETENTION_BYTES}" - name: "DOCKER_IMAGE_PREFIX" valueFrom: configMapKeyRef: diff --git a/template.yml b/template.yml index 6314a86..a342302 100644 --- a/template.yml +++ b/template.yml @@ -179,6 +179,18 @@ parameters: description: Default memory an action requests if unspecified. value: "256 m" required: true +- name: WHISK_KAFKA_REPLICATION_FACTOR + description: Replication factor for all OpenWhisk topics + value: "1" + required: true +- name: WHISK_KAFKA_TOPICS_COMPLETED_RETENTION_BYTES + description: Maximum bytes to retain for the completed activations topic for each Controller. Each kafka node will consume this many bytes times the number of Controllers just for the completed topics. So, if this is set to 1GB and you have 3 Controllers then expect up to 3GB of disk space used by the Controller topics. + value: "268435456" + required: true +- name: WHISK_KAFKA_TOPICS_INVOKER_RETENTION_BYTES + description: Maximum bytes to retain for each Invoker topic. Each Kafka node will consume this many bytes times the number of Invokers just for the Invoker topics. So, if this is set to 1GB and you have 10 Invokers then expect up to 10GB of disk space used by the Invoker topics. + value: "268435456" + required: true - name: OPENWHISK_VERSION description: The DockerHub tag for openwhisk/{controller,invoker} value: "rhdemo-6ee19949" @@ -675,6 +687,10 @@ objects: value: "${WHISK_ACTIONS_MEMORY_MIN}" - name: "CONFIG_whisk_memory_std" value: "${WHISK_ACTIONS_MEMORY_STD}" + - name: "CONFIG_whisk_kafka_replicationFactor" + value: "${WHISK_KAFKA_REPLICATION_FACTOR}" + - name: "CONFIG_whisk_kafka_topics_completed_retentionBytes" + value: "${WHISK_KAFKA_TOPICS_COMPLETED_RETENTION_BYTES}" # extra JVM arguments - name: "JAVA_OPTS" @@ -1104,6 +1120,10 @@ objects: value: "${WHISK_ACTIONS_MEMORY_MIN}" - name: "CONFIG_whisk_memory_std" value: "${WHISK_ACTIONS_MEMORY_STD}" + - name: "CONFIG_whisk_kafka_replicationFactor" + value: "${WHISK_KAFKA_REPLICATION_FACTOR}" + - name: "CONFIG_whisk_kafka_topics_invoker_retentionBytes" + value: "${WHISK_KAFKA_TOPICS_INVOKER_RETENTION_BYTES}" - name: "DOCKER_IMAGE_PREFIX" valueFrom: configMapKeyRef: