[bitnami/kafka] Release 3.4.1-debian-11-r53

bitnami/kafka/3.4/debian-11/Dockerfile

@@ -8,10 +8,10 @@ ARG TARGETARCH
 
 LABEL com.vmware.cp.artifact.flavor="sha256:1e1b4657a77f0d47e9220f0c37b9bf7802581b93214fff7d1bd2364c8bf22e8e" \
       org.opencontainers.image.base.name="docker.io/bitnami/minideb:bullseye" \
-      org.opencontainers.image.created="2023-07-30T15:51:04Z" \
+      org.opencontainers.image.created="2023-07-31T14:25:29Z" \
       org.opencontainers.image.description="Application packaged by VMware, Inc" \
       org.opencontainers.image.licenses="Apache-2.0" \
-      org.opencontainers.image.ref.name="3.4.1-debian-11-r52" \
+      org.opencontainers.image.ref.name="3.4.1-debian-11-r53" \
       org.opencontainers.image.title="kafka" \
       org.opencontainers.image.vendor="VMware, Inc." \
       org.opencontainers.image.version="3.4.1"

bitnami/kafka/3.4/debian-11/rootfs/opt/bitnami/scripts/kafka-env.sh

@@ -24,26 +24,31 @@ export BITNAMI_DEBUG="${BITNAMI_DEBUG:-false}"
 # By setting an environment variable matching *_FILE to a file path, the prefixed environment
 # variable will be overridden with the value specified in that file
 kafka_env_vars=(
-    ALLOW_PLAINTEXT_LISTENER
     KAFKA_INTER_BROKER_USER
     KAFKA_INTER_BROKER_PASSWORD
+    KAFKA_CONTROLLER_USER
+    KAFKA_CONTROLLER_PASSWORD
     KAFKA_CERTIFICATE_PASSWORD
     KAFKA_TLS_TRUSTSTORE_FILE
     KAFKA_TLS_TYPE
     KAFKA_TLS_CLIENT_AUTH
+    KAFKA_TLS_INTER_BROKER_AUTH
+    KAFKA_TLS_CONTROLLER_AUTH
     KAFKA_OPTS
     KAFKA_CFG_LISTENERS
     KAFKA_CFG_ADVERTISED_LISTENERS
     KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP
     KAFKA_CFG_ZOOKEEPER_CONNECT
+    KAFKA_CFG_CONTROLLER_QUORUM_VOTERS
     KAFKA_CFG_SASL_ENABLED_MECHANISMS
     KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL
     KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL
     KAFKA_CFG_INTER_BROKER_LISTENER_NAME
     KAFKA_CFG_MAX_REQUEST_SIZE
     KAFKA_CFG_MAX_PARTITION_FETCH_BYTES
-    KAFKA_ENABLE_KRAFT
     KAFKA_KRAFT_CLUSTER_ID
+    KAFKA_SKIP_KRAFT_STORAGE_INIT
+    KAFKA_CLIENT_LISTENER_NAME
     KAFKA_ZOOKEEPER_PROTOCOL
     KAFKA_ZOOKEEPER_PASSWORD
     KAFKA_ZOOKEEPER_USER
@@ -74,8 +79,7 @@ export KAFKA_BASE_DIR="${BITNAMI_ROOT_DIR}/kafka"
 export KAFKA_VOLUME_DIR="/bitnami/kafka"
 export KAFKA_DATA_DIR="${KAFKA_VOLUME_DIR}/data"
 export KAFKA_CONF_DIR="${KAFKA_BASE_DIR}/config"
-export KAFKA_CONF_FILE="${KAFKA_CONF_DIR}/kraft/server.properties"
-export KAFKA_ZK_CONF_FILE="${KAFKA_CONF_DIR}/server.properties"
+export KAFKA_CONF_FILE="${KAFKA_CONF_DIR}/server.properties"
 export KAFKA_MOUNTED_CONF_DIR="${KAFKA_VOLUME_DIR}/config"
 export KAFKA_CERTS_DIR="${KAFKA_CONF_DIR}/certs"
 export KAFKA_INITSCRIPTS_DIR="/docker-entrypoint-initdb.d"
@@ -88,28 +92,33 @@ export KAFKA_DAEMON_USER="kafka"
 export KAFKA_DAEMON_GROUP="kafka"
 
 # Kafka runtime settings
-export ALLOW_PLAINTEXT_LISTENER="${ALLOW_PLAINTEXT_LISTENER:-no}"
 export KAFKA_INTER_BROKER_USER="${KAFKA_INTER_BROKER_USER:-user}"
 export KAFKA_INTER_BROKER_PASSWORD="${KAFKA_INTER_BROKER_PASSWORD:-bitnami}"
+export KAFKA_CONTROLLER_USER="${KAFKA_CONTROLLER_USER:-controller_user}"
+export KAFKA_CONTROLLER_PASSWORD="${KAFKA_CONTROLLER_PASSWORD:-bitnami}"
 export KAFKA_CERTIFICATE_PASSWORD="${KAFKA_CERTIFICATE_PASSWORD:-}"
 export KAFKA_TLS_TRUSTSTORE_FILE="${KAFKA_TLS_TRUSTSTORE_FILE:-}"
 export KAFKA_TLS_TYPE="${KAFKA_TLS_TYPE:-JKS}"
 export KAFKA_TLS_CLIENT_AUTH="${KAFKA_TLS_CLIENT_AUTH:-required}"
+export KAFKA_TLS_INTER_BROKER_AUTH="${KAFKA_TLS_INTER_BROKER_AUTH:-$KAFKA_TLS_CLIENT_AUTH}"
+export KAFKA_TLS_CONTROLLER_AUTH="${KAFKA_TLS_CONTROLLER_AUTH:-$KAFKA_TLS_CLIENT_AUTH}"
 export KAFKA_OPTS="${KAFKA_OPTS:-}"
 
 # Kafka configuration overrides
-export KAFKA_CFG_LISTENERS="${KAFKA_CFG_LISTENERS:-PLAINTEXT://:9092,CONTROLLER://:9093}"
-export KAFKA_CFG_ADVERTISED_LISTENERS="${KAFKA_CFG_ADVERTISED_LISTENERS:-PLAINTEXT://:9092}"
+export KAFKA_CFG_LISTENERS="${KAFKA_CFG_LISTENERS:-}"
+export KAFKA_CFG_ADVERTISED_LISTENERS="${KAFKA_CFG_ADVERTISED_LISTENERS:-}"
 export KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP="${KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP:-}"
 export KAFKA_CFG_ZOOKEEPER_CONNECT="${KAFKA_CFG_ZOOKEEPER_CONNECT:-}"
+export KAFKA_CFG_CONTROLLER_QUORUM_VOTERS="${KAFKA_CFG_CONTROLLER_QUORUM_VOTERS:-}"
 export KAFKA_CFG_SASL_ENABLED_MECHANISMS="${KAFKA_CFG_SASL_ENABLED_MECHANISMS:-PLAIN,SCRAM-SHA-256,SCRAM-SHA-512}"
 export KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL="${KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL:-}"
 export KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL="${KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL:-}"
 export KAFKA_CFG_INTER_BROKER_LISTENER_NAME="${KAFKA_CFG_INTER_BROKER_LISTENER_NAME:-}"
 export KAFKA_CFG_MAX_REQUEST_SIZE="${KAFKA_CFG_MAX_REQUEST_SIZE:-}"
 export KAFKA_CFG_MAX_PARTITION_FETCH_BYTES="${KAFKA_CFG_MAX_PARTITION_FETCH_BYTES:-}"
-export KAFKA_ENABLE_KRAFT="${KAFKA_ENABLE_KRAFT:-yes}"
 export KAFKA_KRAFT_CLUSTER_ID="${KAFKA_KRAFT_CLUSTER_ID:-}"
+export KAFKA_SKIP_KRAFT_STORAGE_INIT="${KAFKA_SKIP_KRAFT_STORAGE_INIT:-false}"
+export KAFKA_CLIENT_LISTENER_NAME="${KAFKA_CLIENT_LISTENER_NAME:-}"
 
 # ZooKeeper connection settings
 export KAFKA_ZOOKEEPER_PROTOCOL="${KAFKA_ZOOKEEPER_PROTOCOL:-PLAINTEXT}"

bitnami/kafka/3.4/debian-11/rootfs/opt/bitnami/scripts/kafka/postunpack.sh

@@ -30,6 +30,9 @@ for dir in "$KAFKA_LOG_DIR" "$KAFKA_CONF_DIR" "$KAFKA_MOUNTED_CONF_DIR" "$KAFKA_
 done
 chmod -R g+rwX "$KAFKA_BASE_DIR" "$KAFKA_VOLUME_DIR" "$KAFKA_DATA_DIR" "$KAFKA_INITSCRIPTS_DIR"
 
+# Move the original server.properties, so users can skip initialization logic by mounting their own server.properties directly instead of using the MOUNTED_CONF_DIR
+mv "${KAFKA_CONF_DIR}/server.properties" "${KAFKA_CONF_DIR}/server.properties.original"
+
 # Disable logging to stdout and garbage collection
 # Source: https://logging.apache.org/log4j/log4j-2.4/manual/appenders.html
 replace_in_file "${KAFKA_BASE_DIR}/bin/kafka-server-start.sh" " [-]loggc" " "

bitnami/kafka/3.4/debian-11/rootfs/opt/bitnami/scripts/kafka/run.sh

@@ -16,22 +16,17 @@ set -o pipefail
 # Load Kafka environment variables
 . /opt/bitnami/scripts/kafka-env.sh
 
-if [[ "${KAFKA_CFG_LISTENERS:-}" =~ SASL ]] || [[ "${KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP:-}" =~ SASL ]] || [[ "${KAFKA_ZOOKEEPER_PROTOCOL:-}" =~ SASL ]]; then
+if [[ -f "${KAFKA_CONF_DIR}/kafka_jaas.conf" ]]; then
     export KAFKA_OPTS="-Djava.security.auth.login.config=${KAFKA_CONF_DIR}/kafka_jaas.conf"
 fi
 
-if [[ "${KAFKA_ZOOKEEPER_PROTOCOL:-}" =~ SSL ]]; then
-    ZOOKEEPER_SSL_CONFIG=$(zookeeper_get_tls_config)
-    export KAFKA_OPTS="$KAFKA_OPTS $ZOOKEEPER_SSL_CONFIG"
-fi
-
-flags=("$(kafka_get_conf_file)")
-[[ -z "${KAFKA_EXTRA_FLAGS:-}" ]] || flags=("${flags[@]}" "${KAFKA_EXTRA_FLAGS[@]}")
-START_COMMAND=("$KAFKA_HOME/bin/kafka-server-start.sh" "${flags[@]}" "$@")
+cmd="$KAFKA_HOME/bin/kafka-server-start.sh"
+args=("$KAFKA_CONF_FILE")
+! is_empty_value "${KAFKA_EXTRA_FLAGS:-}" && args=("${args[@]}" "${KAFKA_EXTRA_FLAGS[@]}")
 
 info "** Starting Kafka **"
 if am_i_root; then
-    exec_as_user "$KAFKA_DAEMON_USER" "${START_COMMAND[@]}"
+    exec_as_user "$KAFKA_DAEMON_USER" "$cmd" "${args[@]}" "$@"
 else
-    exec "${START_COMMAND[@]}"
+    exec "$cmd" "${args[@]}" "$@"
 fi

bitnami/kafka/3.4/debian-11/rootfs/opt/bitnami/scripts/kafka/setup.sh

@@ -19,39 +19,42 @@ set -o pipefail
 
 # Map Kafka environment variables
 kafka_create_alias_environment_variables
-if [[ -z "${KAFKA_CFG_BROKER_ID:-}" ]]; then
-    if [[ -n "${BROKER_ID_COMMAND:-}" ]]; then
-        KAFKA_CFG_BROKER_ID="$(eval "${BROKER_ID_COMMAND:-}")"
-        export KAFKA_CFG_BROKER_ID
-    elif ! is_boolean_yes "$KAFKA_ENABLE_KRAFT"; then
-        # By default auto allocate broker ID unless KRaft is enabled
-        export KAFKA_CFG_BROKER_ID=-1
-    fi
-fi
 
-# Set the default tuststore locations
+# Dinamically set node.id/broker.id/controller.quorum.voters if the _COMMAND environment variable is set
+kafka_dynamic_environment_variables
+
+# Set the default tuststore locations before validation
 kafka_configure_default_truststore_locations
-# Ensure Kafka environment variables are valid
-kafka_validate
 # Ensure Kafka user and group exist when running as 'root'
-if am_i_root; then
-    ensure_user_exists "$KAFKA_DAEMON_USER" --group "$KAFKA_DAEMON_GROUP"
-    KAFKA_OWNERSHIP_USER="$KAFKA_DAEMON_USER"
-else
-    KAFKA_OWNERSHIP_USER=""
-fi
+am_i_root && ensure_user_exists "$KAFKA_DAEMON_USER" --group "$KAFKA_DAEMON_GROUP"
 # Ensure directories used by Kafka exist and have proper ownership and permissions
 for dir in "$KAFKA_LOG_DIR" "$KAFKA_CONF_DIR" "$KAFKA_MOUNTED_CONF_DIR" "$KAFKA_VOLUME_DIR" "$KAFKA_DATA_DIR"; do
-    ensure_dir_exists "$dir" "$KAFKA_OWNERSHIP_USER"
+    if am_i_root; then
+        ensure_dir_exists "$dir" "$KAFKA_DAEMON_USER" "$KAFKA_DAEMON_GROUP"
+    else
+        ensure_dir_exists "$dir"
+    fi
 done
 
-# shellcheck disable=SC2148
+# Kafka validation, skipped if server.properties was mounted at either $KAFKA_MOUNTED_CONF_DIR or $KAFKA_CONF_DIR
+[[ ! -f "${KAFKA_MOUNTED_CONF_DIR}/server.properties" && ! -f "$KAFKA_CONF_FILE" ]] && kafka_validate
+# Kafka initialization, skipped if server.properties was mounted at $KAFKA_CONF_DIR
+[[ ! -f "$KAFKA_CONF_FILE" ]] && kafka_initialize
 
-# Ensure Kafka is initialized
-kafka_initialize
-# If KRaft is enabled initialize
-if is_boolean_yes "$KAFKA_ENABLE_KRAFT"; then
-    kraft_initialize
+# Initialise KRaft metadata storage if process.roles configured
+if grep -q "^process.roles=" "$KAFKA_CONF_FILE" && ! is_boolean_yes "$KAFKA_SKIP_KRAFT_STORAGE_INIT" ; then
+    kafka_kraft_storage_initialize
+fi
+# Configure Zookeeper SCRAM users
+if is_boolean_yes "${KAFKA_ZOOKEEPER_BOOTSTRAP_SCRAM_USERS:-}"; then
+    kafka_zookeeper_create_sasl_scram_users
+fi
+# KRaft controllers may get stuck starting when the controller quorum voters are changed.
+# Workaround: Remove quorum-state file when scaling up/down controllers (Waiting proposal KIP-853)
+# https://cwiki.apache.org/confluence/display/KAFKA/KIP-853%3A+KRaft+Voter+Changes
+if [[ -f "${KAFKA_DATA_DIR}/__cluster_metadata-0/quorum-state" ]] && grep -q "^controller.quorum.voters=" "$KAFKA_CONF_FILE" && kafka_kraft_quorum_voters_changed; then
+    warn "Detected inconsitences between controller.quorum.voters and quorum-state, removing it..."
+    rm -f "${KAFKA_DATA_DIR}/__cluster_metadata-0/quorum-state"
 fi
 # Ensure custom initialization scripts are executed
 kafka_custom_init_scripts