Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add locking around partitionIDToState map accesses #1239

Merged
merged 4 commits into from
Dec 6, 2018
Merged

Add locking around partitionIDToState map accesses #1239

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
040860e
Add locks around partitionID map accesses
vprithvi Nov 30, 2018
a6cf8e0
Merge branch 'master' into fix-concurrent-rw
vprithvi Dec 5, 2018
6a8c43b
Address feedback
vprithvi Dec 6, 2018
812b46c
Merge branch 'fix-concurrent-rw' of github.com:jaegertracing/jaeger i…
vprithvi Dec 6, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions cmd/ingester/app/consumer/consumer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ type Consumer struct {
deadlockDetector deadlockDetector

partitionIDToState map[int32]*consumerState
partitionMapLock sync.Mutex
partitionsHeld metrics.Counter
}

Expand Down Expand Up @@ -75,15 +76,18 @@ func (c *Consumer) Start() {
go func() {
c.logger.Info("Starting main loop")
for pc := range c.internalConsumer.Partitions() {
c.partitionMapLock.Lock()
if p, ok := c.partitionIDToState[pc.Partition()]; ok {
// This is a guard against simultaneously draining messages
// from the last time the partition was assigned and
// processing new messages for the same partition, which may lead
// to the cleanup process not completing
p.wg.Wait()
}
c.partitionMetrics(pc.Partition()).startCounter.Inc(1)
c.partitionIDToState[pc.Partition()] = &consumerState{partitionConsumer: pc}
c.partitionIDToState[pc.Partition()].wg.Add(2)
c.partitionMapLock.Unlock()
c.partitionMetrics(pc.Partition()).startCounter.Inc(1)
go c.handleMessages(pc)
go c.handleErrors(pc.Partition(), pc.Errors())
}
Expand All @@ -92,10 +96,12 @@ func (c *Consumer) Start() {

// Close closes the Consumer and underlying sarama consumer
func (c *Consumer) Close() error {
c.partitionMapLock.Lock()
for _, p := range c.partitionIDToState {
c.closePartition(p.partitionConsumer)
p.wg.Wait()
}
c.partitionMapLock.Unlock()
c.deadlockDetector.close()
c.logger.Info("Closing parent consumer")
return c.internalConsumer.Close()
Expand All @@ -105,8 +111,10 @@ func (c *Consumer) handleMessages(pc sc.PartitionConsumer) {
c.logger.Info("Starting message handler", zap.Int32("partition", pc.Partition()))
c.partitionsHeld.Inc(1)
defer c.partitionsHeld.Inc(-1)
c.partitionIDToState[pc.Partition()].wg.Add(1)
defer c.partitionIDToState[pc.Partition()].wg.Done()
c.partitionMapLock.Lock()
wg := &c.partitionIDToState[pc.Partition()].wg
c.partitionMapLock.Unlock()
defer wg.Done()
defer c.closePartition(pc)

msgMetrics := c.newMsgMetrics(pc.Partition())
Expand Down Expand Up @@ -152,8 +160,10 @@ func (c *Consumer) closePartition(partitionConsumer sc.PartitionConsumer) {

func (c *Consumer) handleErrors(partition int32, errChan <-chan *sarama.ConsumerError) {
c.logger.Info("Starting error handler", zap.Int32("partition", partition))
c.partitionIDToState[partition].wg.Add(1)
defer c.partitionIDToState[partition].wg.Done()
c.partitionMapLock.Lock()
wg := &c.partitionIDToState[partition].wg
c.partitionMapLock.Unlock()
defer wg.Done()

errMetrics := c.newErrMetrics(partition)
for err := range errChan {
Expand Down