Skip to content

Commit

Permalink
Add support for configuring Pulsar listener container concurrency
Browse files Browse the repository at this point in the history
This commit adds configuration property that allows users to configure
Pulsar message listener container concurrency.
  • Loading branch information
vpavic committed Sep 2, 2024
1 parent c06027b commit 7601951
Show file tree
Hide file tree
Showing 7 changed files with 47 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
* @author Alexander Preuß
* @author Phillip Webb
* @author Jonas Geiregat
* @author Vedran Pavic
* @since 3.2.0
*/
@AutoConfiguration
Expand Down Expand Up @@ -187,7 +188,10 @@ ConcurrentPulsarListenerContainerFactory<?> pulsarListenerContainerFactory(
}
pulsarTransactionManager.ifUnique(containerProperties.transactions()::setTransactionManager);
this.propertiesMapper.customizeContainerProperties(containerProperties);
return new ConcurrentPulsarListenerContainerFactory<>(pulsarConsumerFactory, containerProperties);
ConcurrentPulsarListenerContainerFactory<Object> listenerContainerFactory = new ConcurrentPulsarListenerContainerFactory<>(
pulsarConsumerFactory, containerProperties);
this.propertiesMapper.customizeConcurrentPulsarListenerContainerFactory(listenerContainerFactory);
return listenerContainerFactory;
}

@Bean
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -811,6 +811,11 @@ public static class Listener {
*/
private SchemaType schemaType;

/**
* Number of threads used by listener container.
*/
private Integer concurrency;

/**
* Whether to record observations for when the Observations API is available and
* the client supports it.
Expand All @@ -825,6 +830,14 @@ public void setSchemaType(SchemaType schemaType) {
this.schemaType = schemaType;
}

public Integer getConcurrency() {
return this.concurrency;
}

public void setConcurrency(Integer concurrency) {
this.concurrency = concurrency;
}

public boolean isObservationEnabled() {
return this.observationEnabled;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@

import org.springframework.boot.context.properties.PropertyMapper;
import org.springframework.boot.json.JsonWriter;
import org.springframework.pulsar.config.ConcurrentPulsarListenerContainerFactory;
import org.springframework.pulsar.core.PulsarTemplate;
import org.springframework.pulsar.listener.PulsarContainerProperties;
import org.springframework.pulsar.reader.PulsarReaderContainerProperties;
Expand Down Expand Up @@ -198,6 +199,13 @@ private void customizePulsarContainerListenerProperties(PulsarContainerPropertie
map.from(properties::isObservationEnabled).to(containerProperties::setObservationEnabled);
}

<T> void customizeConcurrentPulsarListenerContainerFactory(
ConcurrentPulsarListenerContainerFactory<T> listenerContainerFactory) {
PulsarProperties.Listener properties = this.properties.getListener();
PropertyMapper map = PropertyMapper.get().alwaysApplyingWhenNonNull();
map.from(properties::getConcurrency).to(listenerContainerFactory::setConcurrency);
}

<T> void customizeReaderBuilder(ReaderBuilder<T> readerBuilder) {
PulsarProperties.Reader properties = this.properties.getReader();
PropertyMapper map = PropertyMapper.get().alwaysApplyingWhenNonNull();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2012-2023 the original author or authors.
* Copyright 2012-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -31,6 +31,7 @@
*
* @author Chris Bono
* @author Phillip Webb
* @author Vedran Pavic
*/
final class PulsarReactivePropertiesMapper {

Expand Down Expand Up @@ -93,6 +94,7 @@ private void customizePulsarContainerListenerProperties(ReactivePulsarContainerP
PulsarProperties.Listener properties = this.properties.getListener();
PropertyMapper map = PropertyMapper.get().alwaysApplyingWhenNonNull();
map.from(properties::getSchemaType).to(containerProperties::setSchemaType);
map.from(properties::getConcurrency).to(containerProperties::setConcurrency);
}

void customizeMessageReaderBuilder(ReactiveMessageReaderBuilder<?> builder) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@

import org.springframework.boot.autoconfigure.pulsar.PulsarProperties.Consumer;
import org.springframework.boot.autoconfigure.pulsar.PulsarProperties.Failover.BackupCluster;
import org.springframework.pulsar.config.ConcurrentPulsarListenerContainerFactory;
import org.springframework.pulsar.core.PulsarProducerFactory;
import org.springframework.pulsar.core.PulsarTemplate;
import org.springframework.pulsar.listener.PulsarContainerProperties;
Expand Down Expand Up @@ -272,6 +273,17 @@ void customizeContainerProperties() {
assertThat(containerProperties.transactions().isEnabled()).isTrue();
}

@Test
void customizeConcurrentPulsarListenerContainerFactory() {
PulsarProperties properties = new PulsarProperties();
properties.getListener().setConcurrency(10);
ConcurrentPulsarListenerContainerFactory<?> listenerContainerFactory = mock(
ConcurrentPulsarListenerContainerFactory.class);
new PulsarPropertiesMapper(properties)
.customizeConcurrentPulsarListenerContainerFactory(listenerContainerFactory);
then(listenerContainerFactory).should().setConcurrency(10);
}

@Test
@SuppressWarnings("unchecked")
void customizeReaderBuilder() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -393,9 +393,11 @@ class ListenerProperties {
void bind() {
Map<String, String> map = new HashMap<>();
map.put("spring.pulsar.listener.schema-type", "avro");
map.put("spring.pulsar.listener.concurrency", "10");
map.put("spring.pulsar.listener.observation-enabled", "true");
PulsarProperties.Listener properties = bindProperties(map).getListener();
assertThat(properties.getSchemaType()).isEqualTo(SchemaType.AVRO);
assertThat(properties.getConcurrency()).isEqualTo(10);
assertThat(properties.isObservationEnabled()).isTrue();
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2012-2023 the original author or authors.
* Copyright 2012-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -48,6 +48,7 @@
*
* @author Chris Bono
* @author Phillip Webb
* @author Vedran Pavic
*/
class PulsarReactivePropertiesMapperTests {

Expand Down Expand Up @@ -120,10 +121,12 @@ void customizeContainerProperties() {
PulsarProperties properties = new PulsarProperties();
properties.getConsumer().getSubscription().setType(SubscriptionType.Shared);
properties.getListener().setSchemaType(SchemaType.AVRO);
properties.getListener().setConcurrency(10);
ReactivePulsarContainerProperties<Object> containerProperties = new ReactivePulsarContainerProperties<>();
new PulsarReactivePropertiesMapper(properties).customizeContainerProperties(containerProperties);
assertThat(containerProperties.getSubscriptionType()).isEqualTo(SubscriptionType.Shared);
assertThat(containerProperties.getSchemaType()).isEqualTo(SchemaType.AVRO);
assertThat(containerProperties.getConcurrency()).isEqualTo(10);
}

@Test
Expand Down

0 comments on commit 7601951

Please sign in to comment.