Skip to content
This repository has been archived by the owner on May 13, 2019. It is now read-only.

add filter before send message to queue #120

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

add filter before send message to queue #120

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 21 additions & 7 deletions consumergroup/consumer_group.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,18 @@ type ConsumerGroup struct {
offsetManager OffsetManager
}

// Connects to a consumer group, using Zookeeper for auto-discovery
type ConsumerFilter func(msg *sarama.ConsumerMessage) bool

func JoinConsumerGroup(name string, topics []string, zookeeper []string, config *Config) (cg *ConsumerGroup, err error) {
return joinConsumerGroup(name, topics, zookeeper, nil, config)
}

func JoinConsumerGroupWithFilter(name string, topics []string, filters map[string]ConsumerFilter, zookeeper []string, config *Config) (cg *ConsumerGroup, err error) {
return joinConsumerGroup(name, topics, zookeeper, filters, config)
}

// Connects to a consumer group, using Zookeeper for auto-discovery
func joinConsumerGroup(name string, topics []string, zookeeper []string, filters map[string]ConsumerFilter, config *Config) (cg *ConsumerGroup, err error) {

if name == "" {
return nil, sarama.ConfigurationError("Empty consumergroup name")
Expand Down Expand Up @@ -176,7 +186,7 @@ func JoinConsumerGroup(name string, topics []string, zookeeper []string, config
offsetConfig := OffsetManagerConfig{CommitInterval: config.Offsets.CommitInterval}
cg.offsetManager = NewZookeeperOffsetManager(cg, &offsetConfig)

go cg.topicListConsumer(topics)
go cg.topicListConsumer(topics, filters)

return
}
Expand Down Expand Up @@ -250,7 +260,7 @@ func (cg *ConsumerGroup) FlushOffsets() error {
return cg.offsetManager.Flush()
}

func (cg *ConsumerGroup) topicListConsumer(topics []string) {
func (cg *ConsumerGroup) topicListConsumer(topics []string, filters map[string]ConsumerFilter) {
for {
select {
case <-cg.stopper:
Expand All @@ -271,7 +281,7 @@ func (cg *ConsumerGroup) topicListConsumer(topics []string) {

for _, topic := range topics {
cg.wg.Add(1)
go cg.topicConsumer(topic, cg.messages, cg.errors, stopper)
go cg.topicConsumer(topic, cg.messages, cg.errors, stopper, filters[topic])
}

select {
Expand Down Expand Up @@ -299,7 +309,7 @@ func (cg *ConsumerGroup) topicListConsumer(topics []string) {
}
}

func (cg *ConsumerGroup) topicConsumer(topic string, messages chan<- *sarama.ConsumerMessage, errors chan<- error, stopper <-chan struct{}) {
func (cg *ConsumerGroup) topicConsumer(topic string, messages chan<- *sarama.ConsumerMessage, errors chan<- error, stopper <-chan struct{}, filter ConsumerFilter) {
defer cg.wg.Done()

select {
Expand Down Expand Up @@ -342,7 +352,7 @@ func (cg *ConsumerGroup) topicConsumer(topic string, messages chan<- *sarama.Con
for _, pid := range myPartitions {

wg.Add(1)
go cg.partitionConsumer(topic, pid.ID, messages, errors, &wg, stopper)
go cg.partitionConsumer(topic, pid.ID, messages, errors, &wg, stopper, filter)
}

wg.Wait()
Expand Down Expand Up @@ -374,7 +384,7 @@ func (cg *ConsumerGroup) consumePartition(topic string, partition int32, nextOff
}

// Consumes a partition
func (cg *ConsumerGroup) partitionConsumer(topic string, partition int32, messages chan<- *sarama.ConsumerMessage, errors chan<- error, wg *sync.WaitGroup, stopper <-chan struct{}) {
func (cg *ConsumerGroup) partitionConsumer(topic string, partition int32, messages chan<- *sarama.ConsumerMessage, errors chan<- error, wg *sync.WaitGroup, stopper <-chan struct{}, filter ConsumerFilter) {
defer wg.Done()

select {
Expand Down Expand Up @@ -494,6 +504,10 @@ partitionConsumerLoop:

}

if filter != nil && !filter(message) {
continue partitionConsumerLoop
}

for {
select {
case <-stopper:
Expand Down