From 3e559ae7e1146dbc025732cc2199873fba8d8ceb Mon Sep 17 00:00:00 2001 From: Giacomo Guiulfo Date: Wed, 19 Sep 2018 14:48:41 -0700 Subject: [PATCH] [incubator/kafka]: add reassignPartitions to topic configuration (#7623) * [incubator/kafka]: add reassignPartitions to topic configuration Signed-off-by: Giacomo Guiulfo * [incubator/kafka]: bump chart to 0.10.0 Signed-off-by: Giacomo Guiulfo --- incubator/kafka/Chart.yaml | 2 +- incubator/kafka/README.md | 2 +- incubator/kafka/templates/configmap-config.yaml | 14 +++++++++++++- incubator/kafka/values.yaml | 8 ++++++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/incubator/kafka/Chart.yaml b/incubator/kafka/Chart.yaml index 2564adbbc3d5..aadf80b0abad 100755 --- a/incubator/kafka/Chart.yaml +++ b/incubator/kafka/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 description: Apache Kafka is publish-subscribe messaging rethought as a distributed commit log. name: kafka -version: 0.9.6 +version: 0.10.0 appVersion: 4.1.2 keywords: - kafka diff --git a/incubator/kafka/README.md b/incubator/kafka/README.md index 8bcd5ddce27e..e3d10b99f291 100644 --- a/incubator/kafka/README.md +++ b/incubator/kafka/README.md @@ -115,7 +115,7 @@ following configurable parameters: | `prometheus.operator.enabled` | True if using the Prometheus Operator, False if not | `false` | | `prometheus.operator.serviceMonitor.namespace` | Namespace which Prometheus is running in. Default to kube-prometheus install. | `monitoring` | | `prometheus.operator.serviceMonitor.selector` | Default to kube-prometheus install (CoreOS recommended), but should be set according to Prometheus install | `{ prometheus: kube-prometheus }` | -| `topics` | List of topics to create & configure. Can specify name, partitions, replicationFactor, config. See values.yaml | `[]` (Empty list) | +| `topics` | List of topics to create & configure. Can specify name, partitions, replicationFactor, reassignPartitions, config. See values.yaml | `[]` (Empty list) | | `zookeeper.enabled` | If True, installs Zookeeper Chart | `true` | | `zookeeper.resources` | Zookeeper resource requests and limits | `{}` | | `zookeeper.env` | Environmental variables provided to Zookeeper Zookeeper | `{ZK_HEAP_SIZE: "1G"}` | diff --git a/incubator/kafka/templates/configmap-config.yaml b/incubator/kafka/templates/configmap-config.yaml index d02954bd2e49..062041c26ea1 100644 --- a/incubator/kafka/templates/configmap-config.yaml +++ b/incubator/kafka/templates/configmap-config.yaml @@ -26,7 +26,19 @@ data: done echo "Applying runtime configuration using {{ .Values.image }}:{{ .Values.imageTag }}" {{- range $n, $topic := .Values.topics }} - {{- if and $topic.partitions $topic.replicationFactor }} + {{- if and $topic.partitions $topic.replicationFactor $topic.reassignPartitions }} + cat << EOF > {{ $topic.name }}-increase-replication-factor.json + {"version":1, "partitions":[ + {{- $partitions := (int $topic.partitions) }} + {{- $replicas := (int $topic.replicationFactor) }} + {{- range $i := until $partitions }} + {"topic":"{{ $topic.name }}","partition":{{ $i }},"replicas":[{{- range $j := until $replicas }}{{ $j }}{{- if ne $j (sub $replicas 1) }},{{- end }}{{- end }}]}{{- if ne $i (sub $partitions 1) }},{{- end }} + {{- end }} + ]} + EOF + kafka-reassign-partitions --zookeeper {{ $zk }} --reassignment-json-file {{ $topic.name }}-increase-replication-factor.json --execute + kafka-reassign-partitions --zookeeper {{ $zk }} --reassignment-json-file {{ $topic.name }}-increase-replication-factor.json --verify + {{- else if and $topic.partitions $topic.replicationFactor }} kafka-topics --zookeeper {{ $zk }} --create --if-not-exists --force --topic {{ $topic.name }} --partitions {{ $topic.partitions }} --replication-factor {{ $topic.replicationFactor }} {{- else if $topic.partitions }} kafka-topics --zookeeper {{ $zk }} --alter --force --topic {{ $topic.name }} --partitions {{ $topic.partitions }} || true diff --git a/incubator/kafka/values.yaml b/incubator/kafka/values.yaml index 9db504f47df3..6c56e45de679 100644 --- a/incubator/kafka/values.yaml +++ b/incubator/kafka/values.yaml @@ -311,6 +311,8 @@ prometheus: ## Topic creation and configuration. ## The job will be run on a deployment only when the config has been changed. ## - If 'partitions' and 'replicationFactor' are specified we create the topic (with --if-not-exists.) +## - If 'partitions', 'replicationFactor' and 'reassignPartitions' are specified we reassign the partitions to +## increase the replication factor of an existing topic. ## - If 'partitions' is specified we 'alter' the number of partitions. This will ## silently and safely fail if the new setting isn’t strictly larger than the old (i.e. a NOOP.) Do be aware of the ## implications for keyed topics (ref: https://docs.confluent.io/current/kafka/post-deployment.html#admin-operations) @@ -318,9 +320,15 @@ prometheus: ## it will silently and safely fail. ## - If 'config' is specified it's added to the topic configuration. ## +## Note: To increase the 'replicationFactor' of a topic, 'reassignPartitions' must be set to true (see above). +## topics: [] # - name: myExistingTopicConfig # config: "cleanup.policy=compact,delete.retention.ms=604800000" + # - name: myExistingTopicReassignPartitions + # partitions: 8 + # replicationFactor: 5 + # reassignPartitions: true # - name: myExistingTopicPartitions # partitions: 8 # - name: myNewTopicWithConfig