From 87b2a651b7e95a2daf7ea719e7f25d2b046b23c2 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Sat, 28 Sep 2019 16:51:02 +0300 Subject: [PATCH 01/28] Implemented SubSourceLogic --- src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs | 12 + .../Settings/ConsumerSettings.cs | 13 + .../Consumers/Abstract/SubSourceLogic.cs | 275 ++++++++++++++++++ .../Consumers/Actors/KafkaConsumerActor.cs | 5 + .../Actors/KafkaConsumerActorMetadata.cs | 20 ++ .../Consumers/Concrete/PlainSubSourceStage.cs | 60 ++++ .../Consumers/Exceptions/ConsumerFailed.cs | 6 + 7 files changed, 391 insertions(+) create mode 100644 src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs create mode 100644 src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs diff --git a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs index 4343fc5f..8e1435b4 100644 --- a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs +++ b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs @@ -51,5 +51,17 @@ public static Source, Task> CommittableSource(Con { return Source.FromGraph(new CommittableSourceStage(settings, subscription)); } + + /// + /// The `plainPartitionedSource` is a way to track automatic partition assignment from kafka. + /// When a topic-partition is assigned to a consumer, this source will emit tuples with the assigned topic-partition and a corresponding + /// source of `ConsumerRecord`s. + /// When a topic-partition is revoked, the corresponding source completes. + /// + public static Source<(TopicPartition, Source, NotUsed>), Task> PlainPartitionedSource(ConsumerSettings settings, + IAutoSubscription subscription) + { + return Source.FromGraph(new PlainSubSourceStage(settings, subscription, null, _ => { })); + } } } diff --git a/src/Akka.Streams.Kafka/Settings/ConsumerSettings.cs b/src/Akka.Streams.Kafka/Settings/ConsumerSettings.cs index 4869a1eb..04a6eef1 100644 --- a/src/Akka.Streams.Kafka/Settings/ConsumerSettings.cs +++ b/src/Akka.Streams.Kafka/Settings/ConsumerSettings.cs @@ -51,6 +51,7 @@ public static ConsumerSettings Create(Config config, IDeserializer commitRefreshInterval: config.GetTimeSpan("commit-refresh-interval", TimeSpan.FromMilliseconds(100), allowInfinite: true), stopTimeout: config.GetTimeSpan("stop-timeout", TimeSpan.FromMilliseconds(50)), positionTimeout: config.GetTimeSpan("position-timeout", TimeSpan.FromSeconds(5)), + waitClosePartition: config.GetTimeSpan("wait-close-partition", TimeSpan.FromSeconds(1)), bufferSize: config.GetInt("buffer-size", 50), dispatcherId: config.GetString("use-dispatcher", "akka.kafka.default-dispatcher"), properties: ImmutableDictionary.Empty); @@ -82,6 +83,10 @@ public static ConsumerSettings Create(Config config, IDeserializer /// public TimeSpan PartitionHandlerWarning { get; } /// + /// Time to wait for pending requests when a partition is closed. + /// + public TimeSpan WaitClosePartition { get; } + /// /// When offset committing takes more then this timeout, the warning will be logged /// public TimeSpan CommitTimeWarning { get; } @@ -116,6 +121,7 @@ public static ConsumerSettings Create(Config config, IDeserializer public ConsumerSettings(IDeserializer keyDeserializer, IDeserializer valueDeserializer, TimeSpan pollInterval, TimeSpan pollTimeout, TimeSpan commitTimeout, TimeSpan commitRefreshInterval, TimeSpan stopTimeout, TimeSpan positionTimeout, TimeSpan commitTimeWarning, TimeSpan partitionHandlerWarning, + TimeSpan waitClosePartition, int bufferSize, string dispatcherId, IImmutableDictionary properties) { KeyDeserializer = keyDeserializer; @@ -131,6 +137,7 @@ public ConsumerSettings(IDeserializer keyDeserializer, IDeserializer @@ -177,6 +184,10 @@ public ConsumerSettings WithProperty(string key, string value) => /// When partition assigned events handling takes more then this timeout, the warning will be logged /// public ConsumerSettings WithPartitionHandlerWarning(TimeSpan partitionHandlerWarning) => Copy(partitionHandlerWarning: partitionHandlerWarning); + /// + /// Time to wait for pending requests when a partition is closed. + /// + public ConsumerSettings WithWaitClosePartition(TimeSpan waitClosePartition) => Copy(waitClosePartition: waitClosePartition); /// /// If set to a finite duration, the consumer will re-send the last committed offsets periodically for all assigned partitions. @@ -230,6 +241,7 @@ private ConsumerSettings Copy( TimeSpan? commitRefreshInterval = null, TimeSpan? stopTimeout = null, TimeSpan? positionTimeout = null, + TimeSpan? waitClosePartition = null, int? bufferSize = null, string dispatcherId = null, IImmutableDictionary properties = null) => @@ -243,6 +255,7 @@ private ConsumerSettings Copy( commitTimeWarning: commitTimeWarning ?? this.CommitTimeWarning, commitRefreshInterval: commitRefreshInterval ?? this.CommitRefreshInterval, stopTimeout: stopTimeout ?? this.StopTimeout, + waitClosePartition: waitClosePartition ?? this.WaitClosePartition, positionTimeout: positionTimeout ?? this.PositionTimeout, bufferSize: bufferSize ?? this.BufferSize, dispatcherId: dispatcherId ?? this.DispatcherId, diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs new file mode 100644 index 00000000..b6dfc33c --- /dev/null +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs @@ -0,0 +1,275 @@ +using System; +using System.Collections.Immutable; +using System.Linq; +using System.Threading.Tasks; +using Akka.Actor; +using Akka.Streams.Dsl; +using Akka.Streams.Kafka.Extensions; +using Akka.Streams.Kafka.Helpers; +using Akka.Streams.Kafka.Settings; +using Akka.Streams.Kafka.Stages.Consumers.Actors; +using Akka.Streams.Kafka.Stages.Consumers.Exceptions; +using Akka.Streams.Stage; +using Akka.Streams.Util; +using Akka.Util.Internal; +using Confluent.Kafka; + +namespace Akka.Streams.Kafka.Stages.Consumers.Abstract +{ + /// + /// Stage logic used to produce sub-sources per topic partitions + /// + internal class SubSourceLogic : TimerGraphStageLogic + { + private class CloseRevokedPartitions { } + + private readonly SourceShape<(TopicPartition, Source)> _shape; + private readonly ConsumerSettings _settings; + private readonly IAutoSubscription _subscription; + private readonly IMessageBuilder _messageBuilder; + private readonly Option, Task>>> _getOffsetsOnAssign; + private readonly Action> _onRevoke; + private readonly TaskCompletionSource _completion; + + private readonly int _actorNumber = KafkaConsumerActorMetadata.NextNumber(); + private Action> _partitionAssignedCallback; + private Action> _updatePendingPartitionsAndEmitSubSourcesCallback; + private Action> _partitionRevokedCallback; + private Action<(TopicPartition, Option>)> _subsourceCancelledCallback; + private Action<(TopicPartition, TaskCompletionSource)> _subsourceStartedCallback; + private Action _stageFailCallback; + + /// + /// Kafka has notified us that we have these partitions assigned, but we have not created a source for them yet. + /// + private IImmutableSet _pendingPartitions = ImmutableHashSet.Empty; + + /// + /// We have created a source for these partitions, but it has not started up and is not in subSources yet. + /// + private IImmutableSet _partitionsInStartup = ImmutableHashSet.Empty; + private IImmutableDictionary> _subSources = ImmutableDictionary>.Empty; + + /// + /// Kafka has signalled these partitions are revoked, but some may be re-assigned just after revoking. + /// + private IImmutableSet _partitionsToRevoke = ImmutableHashSet.Empty; + + + protected StageActor SourceActor { get; private set; } + protected IActorRef ConsumerActor { get; private set; } + + /// + /// SubSourceLogic + /// + public SubSourceLogic(SourceShape<(TopicPartition, Source)> shape, ConsumerSettings settings, + IAutoSubscription subscription, Func, IMessageBuilder> messageBuilderFactory, + Option, Task>>> getOffsetsOnAssign, + Action> onRevoke, TaskCompletionSource completion) + : base(shape) + { + _shape = shape; + _settings = settings; + _subscription = subscription; + _messageBuilder = messageBuilderFactory(this); + _getOffsetsOnAssign = getOffsetsOnAssign; + _onRevoke = onRevoke; + _completion = completion; + + _updatePendingPartitionsAndEmitSubSourcesCallback = GetAsyncCallback>(UpdatePendingPartitionsAndEmitSubSources); + _partitionAssignedCallback = GetAsyncCallback>(HandlePartitionsAssigned); + _partitionRevokedCallback = GetAsyncCallback>(HandlePartitionsRevoked); + _stageFailCallback = GetAsyncCallback(FailStage); + _subsourceCancelledCallback = GetAsyncCallback<(TopicPartition, Option>)>(HandleSubsourceCancelled); + _subsourceStartedCallback = GetAsyncCallback<(TopicPartition, TaskCompletionSource)>(HandleSubsourceStarted); + + SetHandler(shape.Outlet, onPull: EmitSubSourcesForPendingPartitions, onDownstreamFinish: PerformShutdown); + } + + public override void PostStop() + { + ConsumerActor.Tell(new KafkaConsumerActorMetadata.Internal.Stop(), SourceActor.Ref); + + OnShutdown(); + + base.PostStop(); + } + + protected override void OnTimer(object timerKey) + { + if (timerKey is CloseRevokedPartitions) + { + Log.Debug("#{} Closing SubSources for revoked partitions: {}", _actorNumber, _partitionsToRevoke.JoinToString(", ")); + + _onRevoke(_partitionsToRevoke); + _pendingPartitions = _pendingPartitions.Except(_partitionsToRevoke); + _partitionsInStartup = _partitionsInStartup.Except(_partitionsToRevoke); + _partitionsToRevoke.ForEach(tp => _subSources[tp].SetResult(Done.Instance)); + _subSources = _subSources.RemoveRange(_partitionsToRevoke); + _partitionsToRevoke = _partitionsToRevoke.Clear(); + } + } + + private async void HandlePartitionsAssigned(IImmutableSet assigned) + { + var formerlyUnknown = assigned.Except(_partitionsToRevoke); + + if (Log.IsDebugEnabled && formerlyUnknown.Any()) + { + Log.Debug("#{} Assigning new partitions: {}", _actorNumber, formerlyUnknown.JoinToString(", ")); + } + + // make sure re-assigned partitions don't get closed on CloseRevokedPartitions timer + _partitionsToRevoke = _partitionsToRevoke.Except(assigned); + + if (!_getOffsetsOnAssign.HasValue) + { + UpdatePendingPartitionsAndEmitSubSources(formerlyUnknown); + } + else + { + try + { + var offsets = await _getOffsetsOnAssign.Value(assigned); + + SeekAndEmitSubSources(formerlyUnknown, offsets); + } + catch (Exception ex) + { + _stageFailCallback(new ConsumerFailed($"{_actorNumber} Failed to fetch offset for partitions: {formerlyUnknown.JoinToString(", ")}", ex)); + } + } + } + + private async void SeekAndEmitSubSources(IImmutableSet formerlyUnknown, IImmutableSet offsets) + { + try + { + await ConsumerActor.Ask(new KafkaConsumerActorMetadata.Internal.Seek(offsets), TimeSpan.FromSeconds(10)); + + UpdatePendingPartitionsAndEmitSubSources(formerlyUnknown); + } + catch (AskTimeoutException ex) + { + _stageFailCallback(new ConsumerFailed($"{_actorNumber} Consumer failed during seek for partitions: {offsets.JoinToString(", ")}")); + } + } + + private void HandlePartitionsRevoked(IImmutableSet revoked) + { + _partitionsToRevoke = _partitionsToRevoke.Union(revoked.Select(r => r.TopicPartition)); + + ScheduleOnce(new CloseRevokedPartitions(), _settings.WaitClosePartition); + } + + private void HandleSubsourceCancelled((TopicPartition, Option>) obj) + { + var (topicPartition, firstUnconsumed) = obj; + + _subSources = _subSources.Remove(topicPartition); + _partitionsInStartup = _partitionsInStartup.Remove(topicPartition); + _pendingPartitions = _pendingPartitions.Add(topicPartition); + + if (firstUnconsumed.HasValue) + { + var topicPartitionOffset = new TopicPartitionOffset(topicPartition, firstUnconsumed.Value.Offset); + Log.Debug("#{} Seeking {} to {} after partition SubSource cancelled", _actorNumber, topicPartition, topicPartitionOffset.Offset); + + SeekAndEmitSubSources(formerlyUnknown: ImmutableHashSet.Empty, offsets: ImmutableList.Create(topicPartitionOffset).ToImmutableHashSet()); + } + else + { + EmitSubSourcesForPendingPartitions(); + } + } + + private void HandleSubsourceStarted((TopicPartition, TaskCompletionSource) obj) + { + var (topicPartition, taskCompletionSource) = obj; + + if (!_partitionsInStartup.Contains(topicPartition)) + { + // Partition was revoked while starting up. Kill! + taskCompletionSource.SetResult(Done.Instance); + } + else + { + _subSources.SetItem(topicPartition, taskCompletionSource); + _partitionsInStartup.Remove(topicPartition); + } + } + + private void UpdatePendingPartitionsAndEmitSubSources(IImmutableSet formerlyUnknownPartitions) + { + _pendingPartitions = _pendingPartitions.Union(formerlyUnknownPartitions.Where(tp => !_partitionsInStartup.Contains(tp))); + + EmitSubSourcesForPendingPartitions(); + } + + private void EmitSubSourcesForPendingPartitions() + { + if (_pendingPartitions.Any() && IsAvailable(_shape.Outlet)) + { + var topicPartition = _pendingPartitions.First(); + + _pendingPartitions = _pendingPartitions.Skip(1).ToImmutableHashSet(); + _partitionsInStartup = _partitionsInStartup.Add(topicPartition); + + var subSourceStage = new SubSourceStage(topicPartition, ConsumerActor, _subsourceStartedCallback, _subsourceCancelledCallback, _messageBuilder, _actorNumber); + var subsource = Source.FromGraph(subSourceStage); + + Push(_shape.Outlet, (topicPartition, subsource)); + + EmitSubSourcesForPendingPartitions(); + } + } + + /// + /// Makes this logic task finished + /// + protected void OnShutdown() + { + _completion.TrySetResult(NotUsed.Instance); + } + + private void PerformShutdown() + { + SetKeepGoing(true); + + // TODO from alpakka: we should wait for subsources to be shutdown and next shutdown main stage + _subSources.Values.ForEach(task => task.SetResult(Done.Instance)); + + if (!IsClosed(_shape.Outlet)) + Complete(_shape.Outlet); + + SourceActor.Become(args => + { + var (actor, message) = args; + if (message is Terminated terminated && terminated.ActorRef.Equals(ConsumerActor)) + { + OnShutdown(); + CompleteStage(); + } + }); + + Materializer.ScheduleOnce(_settings.StopTimeout, () => ConsumerActor.Tell(new KafkaConsumerActorMetadata.Internal.Stop())); + } + + private class SubSourceStage : GraphStage> + { + public override SourceShape Shape { get; } + + public SubSourceStage(TopicPartition topicPartition, IActorRef consumerActor, + Action<(TopicPartition, TaskCompletionSource)> subSourceStartedCallback, + Action<(TopicPartition, Option>)> subSourceCancelledCallback, + IMessageBuilder messageBuilder, + int actorNumber) + { + // TODO + } + + + protected override GraphStageLogic CreateLogic(Attributes inheritedAttributes) => throw new NotImplementedException(); + } + } +} \ No newline at end of file diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Actors/KafkaConsumerActor.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Actors/KafkaConsumerActor.cs index c47833d1..f8f71e7e 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Actors/KafkaConsumerActor.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Actors/KafkaConsumerActor.cs @@ -147,6 +147,11 @@ protected override bool Receive(object message) } return true; + case KafkaConsumerActorMetadata.Internal.Seek seek: + seek.Offsets.ForEach(topicPartitionOffset => _consumer.Seek(topicPartitionOffset)); + Sender.Tell(Done.Instance); + return true; + case KafkaConsumerActorMetadata.Internal.Committed committed: _commitRefreshing.Committed(committed.Offsets); return true; diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Actors/KafkaConsumerActorMetadata.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Actors/KafkaConsumerActorMetadata.cs index e9d32275..b73ab67e 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Actors/KafkaConsumerActorMetadata.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Actors/KafkaConsumerActorMetadata.cs @@ -229,6 +229,26 @@ public SubscribePattern(string topicPattern) /// Stops consuming actor /// public class Stop{ } + + /// + /// Seek + /// + public class Seek + { + /// + /// Seek + /// + /// Offsets to seek + public Seek(IImmutableSet offsets) + { + Offsets = offsets; + } + + /// + /// Offsets to seek + /// + public IImmutableSet Offsets { get; } + } } } } \ No newline at end of file diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs new file mode 100644 index 00000000..d533dbf2 --- /dev/null +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Immutable; +using System.Threading.Tasks; +using Akka.Event; +using Akka.Streams.Dsl; +using Akka.Streams.Kafka.Dsl; +using Akka.Streams.Kafka.Settings; +using Akka.Streams.Kafka.Stages.Consumers.Abstract; +using Akka.Streams.Stage; +using Akka.Streams.Util; +using Confluent.Kafka; + +namespace Akka.Streams.Kafka.Stages.Consumers.Concrete +{ + /// + /// This stage is used for + /// + /// The key type + /// The value type + public class PlainSubSourceStage : KafkaSourceStage, NotUsed>)> + { + /// + /// Consumer settings + /// + public ConsumerSettings Settings { get; } + /// + /// Subscription + /// + public IAutoSubscription Subscription { get; } + /// + /// Function to get offsets from partitions on paritions assigned event + /// + public Option, Task>>> GetOffsetsOnAssign { get; } + /// + /// Partitions revoked event handling action + /// + public Action> OnRevoke { get; } + + /// + /// PlainSubSourceStage + /// + public PlainSubSourceStage(ConsumerSettings settings, IAutoSubscription subscription, + Option, Task>>> getOffsetsOnAssign, + Action> onRevoke) + : base("PlainSubSource") + { + Settings = settings; + Subscription = subscription; + GetOffsetsOnAssign = getOffsetsOnAssign; + OnRevoke = onRevoke; + } + + protected override GraphStageLogic Logic(SourceShape<(TopicPartition, Source, NotUsed>)> shape, + TaskCompletionSource completion, Attributes inheritedAttributes) + { + return new SubSourceLogic>(shape, Settings, Subscription, _ => new PlainMessageBuilder(), + GetOffsetsOnAssign, OnRevoke, completion); + } + } +} \ No newline at end of file diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Exceptions/ConsumerFailed.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Exceptions/ConsumerFailed.cs index 3ef72bf1..12899a13 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Exceptions/ConsumerFailed.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Exceptions/ConsumerFailed.cs @@ -16,5 +16,11 @@ public ConsumerFailed() : this("Consumer actor failed") { } /// /// Message public ConsumerFailed(string message) : base(message) { } + /// + /// Consumer failed + /// + /// Message + /// Inner exception + public ConsumerFailed(string message, Exception innerException) : base(message, innerException) { } } } \ No newline at end of file From 6a4d72c42de18a3a304d62c390cd0ecda7c552f2 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Sat, 28 Sep 2019 16:57:21 +0300 Subject: [PATCH 02/28] Replaced null with Option.None for option parameter --- src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs index 8e1435b4..297da7cc 100644 --- a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs +++ b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs @@ -1,4 +1,6 @@ -using System.Threading.Tasks; +using System; +using System.Collections.Immutable; +using System.Threading.Tasks; using Akka.Actor; using Akka.Streams.Dsl; using Akka.Streams.Kafka.Settings; @@ -7,6 +9,7 @@ using Akka.Streams.Kafka.Messages; using Akka.Streams.Kafka.Stages.Consumers; using Akka.Streams.Kafka.Stages.Consumers.Concrete; +using Akka.Streams.Util; namespace Akka.Streams.Kafka.Dsl { @@ -61,7 +64,9 @@ public static Source, Task> CommittableSource(Con public static Source<(TopicPartition, Source, NotUsed>), Task> PlainPartitionedSource(ConsumerSettings settings, IAutoSubscription subscription) { - return Source.FromGraph(new PlainSubSourceStage(settings, subscription, null, _ => { })); + return Source.FromGraph(new PlainSubSourceStage(settings, subscription, + Option, Task>>>.None, + _ => { })); } } } From 0d58597680c5a5ac094d97bd1759dad1f5fb550b Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Sat, 28 Sep 2019 17:07:32 +0300 Subject: [PATCH 03/28] Fixed xml doc reference --- .../Stages/Consumers/Concrete/PlainSubSourceStage.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs index d533dbf2..0c844918 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs @@ -13,7 +13,7 @@ namespace Akka.Streams.Kafka.Stages.Consumers.Concrete { /// - /// This stage is used for + /// This stage is used for /// /// The key type /// The value type From 6076cb42fd2b0b30c46f8c3892302e479e61e900 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Tue, 1 Oct 2019 14:24:58 +0300 Subject: [PATCH 04/28] Added numbers to logger formatted string --- .../Stages/Consumers/Abstract/SubSourceLogic.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs index b6dfc33c..c2e8ab27 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs @@ -99,7 +99,7 @@ protected override void OnTimer(object timerKey) { if (timerKey is CloseRevokedPartitions) { - Log.Debug("#{} Closing SubSources for revoked partitions: {}", _actorNumber, _partitionsToRevoke.JoinToString(", ")); + Log.Debug("#{0} Closing SubSources for revoked partitions: {1}", _actorNumber, _partitionsToRevoke.JoinToString(", ")); _onRevoke(_partitionsToRevoke); _pendingPartitions = _pendingPartitions.Except(_partitionsToRevoke); @@ -116,7 +116,7 @@ private async void HandlePartitionsAssigned(IImmutableSet assign if (Log.IsDebugEnabled && formerlyUnknown.Any()) { - Log.Debug("#{} Assigning new partitions: {}", _actorNumber, formerlyUnknown.JoinToString(", ")); + Log.Debug("#{0} Assigning new partitions: {1}", _actorNumber, formerlyUnknown.JoinToString(", ")); } // make sure re-assigned partitions don't get closed on CloseRevokedPartitions timer @@ -173,7 +173,7 @@ private void HandleSubsourceCancelled((TopicPartition, Option.Empty, offsets: ImmutableList.Create(topicPartitionOffset).ToImmutableHashSet()); } From 728efc2197503843c3d51a42f4358ed61597888b Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Tue, 1 Oct 2019 15:20:48 +0300 Subject: [PATCH 05/28] Implemented subsource stage logic --- .../Consumers/Abstract/SubSourceLogic.cs | 114 +++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs index c2e8ab27..106a08ae 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Runtime.InteropServices; using System.Threading.Tasks; using Akka.Actor; using Akka.Streams.Dsl; @@ -257,6 +259,14 @@ private void PerformShutdown() private class SubSourceStage : GraphStage> { + private readonly TopicPartition _topicPartition; + private readonly IActorRef _consumerActor; + private readonly Action<(TopicPartition, TaskCompletionSource)> _subSourceStartedCallback; + private readonly Action<(TopicPartition, Option>)> _subSourceCancelledCallback; + private readonly IMessageBuilder _messageBuilder; + private readonly int _actorNumber; + + public Outlet Out { get; } public override SourceShape Shape { get; } public SubSourceStage(TopicPartition topicPartition, IActorRef consumerActor, @@ -265,11 +275,111 @@ public SubSourceStage(TopicPartition topicPartition, IActorRef consumerActor, IMessageBuilder messageBuilder, int actorNumber) { - // TODO + _topicPartition = topicPartition; + _consumerActor = consumerActor; + _subSourceStartedCallback = subSourceStartedCallback; + _subSourceCancelledCallback = subSourceCancelledCallback; + _messageBuilder = messageBuilder; + _actorNumber = actorNumber; + + Out = new Outlet("out"); + Shape = new SourceShape(Out); } - protected override GraphStageLogic CreateLogic(Attributes inheritedAttributes) => throw new NotImplementedException(); + protected override GraphStageLogic CreateLogic(Attributes inheritedAttributes) + { + return new SubSourceStageLogic(Shape, _topicPartition, _consumerActor, _actorNumber, _messageBuilder, + _subSourceStartedCallback, _subSourceCancelledCallback); + } + + private class SubSourceStageLogic : GraphStageLogic + { + private readonly SourceShape _shape; + private readonly TopicPartition _topicPartition; + private readonly IActorRef _consumerActor; + private readonly int _actorNumber; + private readonly IMessageBuilder _messageBuilder; + private readonly Action<(TopicPartition, TaskCompletionSource)> _subSourceStartedCallback; + private KafkaConsumerActorMetadata.Internal.RequestMessages _requestMessages; + private bool _requested = false; + private StageActor _subSourceActor; + private Queue> _buffer = new Queue>(); + + public SubSourceStageLogic(SourceShape shape, TopicPartition topicPartition, IActorRef consumerActor, + int actorNumber, IMessageBuilder messageBuilder, + Action<(TopicPartition, TaskCompletionSource)> subSourceStartedCallback, + Action<(TopicPartition, Option>)> subSourceCancelledCallback) + : base(shape) + { + _shape = shape; + _topicPartition = topicPartition; + _consumerActor = consumerActor; + _actorNumber = actorNumber; + _messageBuilder = messageBuilder; + _subSourceStartedCallback = subSourceStartedCallback; + _requestMessages = new KafkaConsumerActorMetadata.Internal.RequestMessages(0, ImmutableHashSet.Create(topicPartition)); + + SetHandler(shape.Outlet, onPull: Pump, onDownstreamFinish: () => + { + var firstUnconsumed = _buffer.Count > 0 ? new Option>(_buffer.Dequeue()) : Option>.None; + subSourceCancelledCallback((topicPartition, firstUnconsumed)); + + CompleteStage(); + }); + } + + public override void PreStart() + { + Log.Debug("{0} Starting SubSource for partition {1}", _actorNumber, _topicPartition); + + base.PreStart(); + + _subSourceStartedCallback((_topicPartition, new TaskCompletionSource())); + _subSourceActor = GetStageActor(args => + { + var (actor, message) = args; + + switch (message) + { + case KafkaConsumerActorMetadata.Internal.Messages messages: + _requested = false; + + foreach (var consumerMessage in messages.MessagesList) + _buffer.Enqueue(consumerMessage); + + Pump(); + break; + case Status.Failure failure: + FailStage(failure.Cause); + break; + case Terminated terminated when terminated.ActorRef.Equals(_consumerActor): + FailStage(new ConsumerFailed()); + break; + } + }); + + _subSourceActor.Watch(_consumerActor); + } + + private void Pump() + { + if (IsAvailable(_shape.Outlet)) + { + if (_buffer.Count > 0) + { + var message = _buffer.Dequeue(); + Push(_shape.Outlet, _messageBuilder.CreateMessage(message)); + Pump(); + } + else if (!_requested) + { + _requested = true; + _consumerActor.Tell(_requestMessages, _subSourceActor.Ref); + } + } + } + } } } } \ No newline at end of file From 6d72ba0c4e0e1817c648f7aea2fde84a05368883 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Tue, 1 Oct 2019 15:32:09 +0300 Subject: [PATCH 06/28] Fixed base stage xml comment --- .../Stages/Consumers/Abstract/BaseSingleSourceLogic.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs index 01828793..1162f703 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs @@ -18,7 +18,7 @@ namespace Akka.Streams.Kafka.Stages.Consumers.Abstract { /// - /// Shared GraphStageLogic for and + /// Shared GraphStageLogic for and /// /// Key type /// Value type From 47c5a61385c52064144a2f9f379ab6a34547bb6f Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Tue, 1 Oct 2019 16:36:47 +0300 Subject: [PATCH 07/28] Added one test (failing) --- .../PlainPartitionedSourceIntegrationTests.cs | 102 ++++++++++++++++++ .../KafkaIntegrationTests.cs | 14 +++ 2 files changed, 116 insertions(+) create mode 100644 src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs diff --git a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs new file mode 100644 index 00000000..4e696b76 --- /dev/null +++ b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs @@ -0,0 +1,102 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Akka.Streams.Dsl; +using Akka.Streams.Kafka.Dsl; +using Akka.Streams.Kafka.Messages; +using Akka.Streams.Kafka.Settings; +using Akka.Streams.TestKit; +using Confluent.Kafka; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +namespace Akka.Streams.Kafka.Tests.Integration +{ + public class PlainPartitionedSourceIntegrationTests : KafkaIntegrationTests + { + public PlainPartitionedSourceIntegrationTests(ITestOutputHelper output, KafkaFixture fixture) + : base(nameof(PlainPartitionedSourceIntegrationTests), output, fixture) + { + } + + [Fact] + public async Task PlainPartitionedSource_Should_not_lose_any_messages_when_Kafka_node_dies() + { + var topic = CreateTopic(1); + var group = CreateGroup(1); + int partitionsCount = 3; + int totalMessages = 1000 * 10; + + var consumerConfig = CreateConsumerSettings(group) + .WithProperty("metadata.max.age.ms", "100"); // default was 5 * 60 * 1000 (five minutes) + + var (consumeTask, probe) = KafkaConsumer.PlainPartitionedSource(consumerConfig, Subscriptions.Topics(topic)) + .GroupBy(4, tuple => tuple.Item1) + .SelectAsync(8, async tuple => + { + var (topicPartition, source) = tuple; + Log.Info($"Sub-source for {topicPartition}"); + var sourceMessages = await source + .Scan(0, (i, message) => i++) + .Select(i => LogReceivedMessages(topicPartition, i)) + .RunWith(Sink.Last(), Materializer); + + Log.Info($"{topicPartition}: Received {sourceMessages} messages in total"); + return sourceMessages; + }) + .MergeSubstreams() + .As>() + .Scan(0L, (i, subValue) => i + subValue) + .ToMaterialized(this.SinkProbe(), Keep.Both) + .Run(Materializer); + + var produceTask = Source.From(Enumerable.Range(0, totalMessages)) + .Select(LogSentMessages) + .Select(number => + { + if (number == totalMessages / 2) + { + Log.Warning($"Stopping one Kafka container after {number} messages"); + // TODO: Stop one of multiple docker containers in kafka cluster + } + + return number; + }) + .Select(number => new MessageAndMeta() + { + TopicPartition = new TopicPartition(topic, number % partitionsCount), + Message = new Message() { Value = number.ToString() } + }) + .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); + + probe.Request(totalMessages); + + AwaitCondition(() => produceTask.IsCompletedSuccessfully, TimeSpan.FromSeconds(30)); + + probe.Cancel(); + foreach (var i in Enumerable.Range(0, totalMessages)) + { + probe.ExpectNext(i); + } + + AwaitCondition(() => consumeTask.IsCompletedSuccessfully); + } + + private int LogSentMessages(int counter) + { + if (counter % 1000 == 0) + Log.Info($"Sent {counter} messages so far"); + + return counter; + } + + private long LogReceivedMessages(TopicPartition tp, int counter) + { + if (counter % 1000 == 0) + Log.Info($"{tp}: Received {counter} messages so far."); + + return counter; + } + } +} \ No newline at end of file diff --git a/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs index b6541347..e58ca6c8 100644 --- a/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs @@ -9,6 +9,7 @@ using Akka.Streams.Kafka.Settings; using Akka.Streams.TestKit; using Confluent.Kafka; +using FluentAssertions; using Xunit; using Xunit.Abstractions; using Config = Akka.Configuration.Config; @@ -66,6 +67,19 @@ await Source .RunWith(KafkaProducer.PlainSink(producerSettings), Materializer); } + /// + /// Asserts that task will finish successfully until specified timeout. + /// Throws task exception if task failes + /// + protected async Task AssertCompletesSuccessfullyWithing(TimeSpan timeout, Task task) + { + var timeoutTask = Task.Delay(timeout); + + await Task.WhenAny(timeoutTask, task); + + task.IsCompletedSuccessfully.Should().Be(true, $"Timeout {timeout} while waitilng task finish successfully"); + } + protected async Task GivenInitializedTopic(string topic) { using (var producer = ProducerSettings.CreateKafkaProducer()) From c7e2c235d353051dc5a8a6063e5408535b246cc5 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Fri, 4 Oct 2019 17:47:17 +0300 Subject: [PATCH 08/28] Some fixes --- .../PlainPartitionedSourceIntegrationTests.cs | 51 ++++------- .../Consumers/Abstract/SubSourceLogic.cs | 84 ++++++++++++++----- 2 files changed, 81 insertions(+), 54 deletions(-) diff --git a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs index 4e696b76..ca8419a9 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs @@ -25,20 +25,20 @@ public async Task PlainPartitionedSource_Should_not_lose_any_messages_when_Kafka { var topic = CreateTopic(1); var group = CreateGroup(1); - int partitionsCount = 3; - int totalMessages = 1000 * 10; + var totalMessages = 1000 * 10; - var consumerConfig = CreateConsumerSettings(group) - .WithProperty("metadata.max.age.ms", "100"); // default was 5 * 60 * 1000 (five minutes) + var consumerSettings = CreateConsumerSettings(group); - var (consumeTask, probe) = KafkaConsumer.PlainPartitionedSource(consumerConfig, Subscriptions.Topics(topic)) - .GroupBy(4, tuple => tuple.Item1) + await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); + + var (consumeTask, probe) = KafkaConsumer.PlainPartitionedSource(consumerSettings, Subscriptions.Topics(topic)) + .GroupBy(3, tuple => tuple.Item1) .SelectAsync(8, async tuple => { var (topicPartition, source) = tuple; Log.Info($"Sub-source for {topicPartition}"); var sourceMessages = await source - .Scan(0, (i, message) => i++) + .Scan(0, (i, message) => i + 1) .Select(i => LogReceivedMessages(topicPartition, i)) .RunWith(Sink.Last(), Materializer); @@ -50,36 +50,17 @@ public async Task PlainPartitionedSource_Should_not_lose_any_messages_when_Kafka .Scan(0L, (i, subValue) => i + subValue) .ToMaterialized(this.SinkProbe(), Keep.Both) .Run(Materializer); - - var produceTask = Source.From(Enumerable.Range(0, totalMessages)) - .Select(LogSentMessages) - .Select(number => - { - if (number == totalMessages / 2) - { - Log.Warning($"Stopping one Kafka container after {number} messages"); - // TODO: Stop one of multiple docker containers in kafka cluster - } - - return number; - }) - .Select(number => new MessageAndMeta() - { - TopicPartition = new TopicPartition(topic, number % partitionsCount), - Message = new Message() { Value = number.ToString() } - }) - .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); - probe.Request(totalMessages); - - AwaitCondition(() => produceTask.IsCompletedSuccessfully, TimeSpan.FromSeconds(30)); - - probe.Cancel(); - foreach (var i in Enumerable.Range(0, totalMessages)) + AwaitCondition(() => { - probe.ExpectNext(i); - } - + Log.Debug("Expecting next number..."); + var next = probe.RequestNext(TimeSpan.FromSeconds(10)); + Log.Debug("Got requested number: " + next); + return next == totalMessages; + }, TimeSpan.FromSeconds(20)); + + probe.Cancel(); + AwaitCondition(() => consumeTask.IsCompletedSuccessfully); } diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs index 106a08ae..b6a9a993 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs @@ -88,6 +88,47 @@ public SubSourceLogic(SourceShape<(TopicPartition, Source)> s SetHandler(shape.Outlet, onPull: EmitSubSourcesForPendingPartitions, onDownstreamFinish: PerformShutdown); } + public override void PreStart() + { + base.PreStart(); + + SourceActor = GetStageActor(args => + { + switch (args.Item2) + { + case Status.Failure failure: + FailStage(failure.Cause); + break; + + case Terminated terminated when terminated.ActorRef.Equals(ConsumerActor): + FailStage(new ConsumerFailed()); + break; + } + }); + + if (!(Materializer is ActorMaterializer actorMaterializer)) + throw new ArgumentException($"Expected {typeof(ActorMaterializer)} but got {Materializer.GetType()}"); + + var eventHandler = new AsyncCallbacksPartitionEventHandler(_partitionAssignedCallback, _partitionRevokedCallback); + + var extendedActorSystem = actorMaterializer.System.AsInstanceOf(); + ConsumerActor = extendedActorSystem.SystemActorOf(KafkaConsumerActorMetadata.GetProps(SourceActor.Ref, _settings, eventHandler), + $"kafka-consumer-{_actorNumber}"); + + SourceActor.Watch(ConsumerActor); + + switch (_subscription) + { + case TopicSubscription topicSubscription: + ConsumerActor.Tell(new KafkaConsumerActorMetadata.Internal.Subscribe(topicSubscription.Topics), SourceActor.Ref); + break; + + case TopicSubscriptionPattern topicSubscriptionPattern: + ConsumerActor.Tell(new KafkaConsumerActorMetadata.Internal.SubscribePattern(topicSubscriptionPattern.TopicPattern), SourceActor.Ref); + break; + } + } + public override void PostStop() { ConsumerActor.Tell(new KafkaConsumerActorMetadata.Internal.Stop(), SourceActor.Ref); @@ -217,7 +258,8 @@ private void EmitSubSourcesForPendingPartitions() _pendingPartitions = _pendingPartitions.Skip(1).ToImmutableHashSet(); _partitionsInStartup = _partitionsInStartup.Add(topicPartition); - var subSourceStage = new SubSourceStage(topicPartition, ConsumerActor, _subsourceStartedCallback, _subsourceCancelledCallback, _messageBuilder, _actorNumber); + var subSourceStage = new SubSourceStreamStage(topicPartition, ConsumerActor, _subsourceStartedCallback, + _subsourceCancelledCallback, _messageBuilder, _actorNumber); var subsource = Source.FromGraph(subSourceStage); Push(_shape.Outlet, (topicPartition, subsource)); @@ -257,22 +299,22 @@ private void PerformShutdown() Materializer.ScheduleOnce(_settings.StopTimeout, () => ConsumerActor.Tell(new KafkaConsumerActorMetadata.Internal.Stop())); } - private class SubSourceStage : GraphStage> + private class SubSourceStreamStage : GraphStage> { private readonly TopicPartition _topicPartition; private readonly IActorRef _consumerActor; private readonly Action<(TopicPartition, TaskCompletionSource)> _subSourceStartedCallback; private readonly Action<(TopicPartition, Option>)> _subSourceCancelledCallback; - private readonly IMessageBuilder _messageBuilder; + private readonly IMessageBuilder _messageBuilder; private readonly int _actorNumber; - public Outlet Out { get; } - public override SourceShape Shape { get; } + public Outlet Out { get; } + public override SourceShape Shape { get; } - public SubSourceStage(TopicPartition topicPartition, IActorRef consumerActor, + public SubSourceStreamStage(TopicPartition topicPartition, IActorRef consumerActor, Action<(TopicPartition, TaskCompletionSource)> subSourceStartedCallback, Action<(TopicPartition, Option>)> subSourceCancelledCallback, - IMessageBuilder messageBuilder, + IMessageBuilder messageBuilder, int actorNumber) { _topicPartition = topicPartition; @@ -282,32 +324,31 @@ public SubSourceStage(TopicPartition topicPartition, IActorRef consumerActor, _messageBuilder = messageBuilder; _actorNumber = actorNumber; - Out = new Outlet("out"); - Shape = new SourceShape(Out); + Out = new Outlet("out"); + Shape = new SourceShape(Out); } - protected override GraphStageLogic CreateLogic(Attributes inheritedAttributes) { - return new SubSourceStageLogic(Shape, _topicPartition, _consumerActor, _actorNumber, _messageBuilder, + return new SubSourceStreamStageLogic(Shape, _topicPartition, _consumerActor, _actorNumber, _messageBuilder, _subSourceStartedCallback, _subSourceCancelledCallback); } - private class SubSourceStageLogic : GraphStageLogic + private class SubSourceStreamStageLogic : GraphStageLogic { - private readonly SourceShape _shape; + private readonly SourceShape _shape; private readonly TopicPartition _topicPartition; private readonly IActorRef _consumerActor; private readonly int _actorNumber; - private readonly IMessageBuilder _messageBuilder; + private readonly IMessageBuilder _messageBuilder; private readonly Action<(TopicPartition, TaskCompletionSource)> _subSourceStartedCallback; private KafkaConsumerActorMetadata.Internal.RequestMessages _requestMessages; private bool _requested = false; private StageActor _subSourceActor; private Queue> _buffer = new Queue>(); - public SubSourceStageLogic(SourceShape shape, TopicPartition topicPartition, IActorRef consumerActor, - int actorNumber, IMessageBuilder messageBuilder, + public SubSourceStreamStageLogic(SourceShape shape, TopicPartition topicPartition, IActorRef consumerActor, + int actorNumber, IMessageBuilder messageBuilder, Action<(TopicPartition, TaskCompletionSource)> subSourceStartedCallback, Action<(TopicPartition, Option>)> subSourceCancelledCallback) : base(shape) @@ -324,8 +365,6 @@ public SubSourceStageLogic(SourceShape shape, TopicPartition topicPart { var firstUnconsumed = _buffer.Count > 0 ? new Option>(_buffer.Dequeue()) : Option>.None; subSourceCancelledCallback((topicPartition, firstUnconsumed)); - - CompleteStage(); }); } @@ -361,7 +400,14 @@ public override void PreStart() _subSourceActor.Watch(_consumerActor); } - + + public override void PostStop() + { + CompleteStage(); + + base.PostStop(); + } + private void Pump() { if (IsAvailable(_shape.Outlet)) From 8d10f968cdc0439bbef4cfc6f06888160dee1e6f Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Fri, 4 Oct 2019 19:26:51 +0300 Subject: [PATCH 09/28] Updated all stages to use IControl as materialized value --- src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs | 16 +-- .../Extensions/ControlExtensions.cs | 47 ++++++++ src/Akka.Streams.Kafka/Helpers/Control.cs | 110 ++++++++++++++++++ .../Helpers/PromiseControl.cs | 87 ++++++++++++++ .../Abstract/BaseSingleSourceLogic.cs | 49 +++----- .../Abstract/ExternalSingleSourceLogic.cs | 4 +- .../Consumers/Abstract/KafkaSourceStage.cs | 13 +-- .../Abstract/SingleSourceStageLogic.cs | 5 +- .../Concrete/CommittableSourceStage.cs | 7 +- .../ExternalCommittableSourceStage.cs | 7 +- .../Concrete/ExternalPlainSourceStage.cs | 9 +- .../Consumers/Concrete/PlainSourceStage.cs | 9 +- .../Concrete/SourceWithOffsetContextStage.cs | 11 +- 13 files changed, 305 insertions(+), 69 deletions(-) create mode 100644 src/Akka.Streams.Kafka/Extensions/ControlExtensions.cs create mode 100644 src/Akka.Streams.Kafka/Helpers/Control.cs create mode 100644 src/Akka.Streams.Kafka/Helpers/PromiseControl.cs diff --git a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs index b867f937..8e96d2f1 100644 --- a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs +++ b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs @@ -28,7 +28,7 @@ public static class KafkaConsumer /// possible, but when it is, it will make the consumption fully atomic and give "exactly once" semantics that are /// stronger than the "at-least once" semantics you get with Kafka's offset commit functionality. /// - public static Source, Task> PlainSource(ConsumerSettings settings, ISubscription subscription) + public static Source, IControl> PlainSource(ConsumerSettings settings, ISubscription subscription) { return Source.FromGraph(new PlainSourceStage(settings, subscription)); } @@ -37,7 +37,7 @@ public static Source, Task> PlainSource(ConsumerSettin /// Special source that can use an external `KafkaAsyncConsumer`. This is useful when you have /// a lot of manually assigned topic-partitions and want to keep only one kafka consumer. /// - public static Source, Task> PlainExternalSource(IActorRef consumer, IManualSubscription subscription) + public static Source, IControl> PlainExternalSource(IActorRef consumer, IManualSubscription subscription) { return Source.FromGraph(new ExternalPlainSourceStage(consumer, subscription)); } @@ -50,7 +50,7 @@ public static Source, Task> PlainExternalSource(IActor /// If you need to store offsets in anything other than Kafka, should /// be used instead of this API. /// - public static Source, Task> CommittableSource(ConsumerSettings settings, ISubscription subscription) + public static Source, IControl> CommittableSource(ConsumerSettings settings, ISubscription subscription) { return Source.FromGraph(new CommittableSourceStage(settings, subscription)); } @@ -60,7 +60,7 @@ public static Source, Task> CommittableSource(Con /// when an offset is committed based on the record. This can be useful (for example) to store information about which /// node made the commit, what time the commit was made, the timestamp of the record etc. /// - public static Source, Task> CommitWithMetadataSource(ConsumerSettings settings, ISubscription subscription, + public static Source, IControl> CommitWithMetadataSource(ConsumerSettings settings, ISubscription subscription, Func, string> metadataFromRecord) { return Source.FromGraph(new CommittableSourceStage(settings, subscription, metadataFromRecord)); @@ -78,7 +78,7 @@ public static Source, Task> CommitWithMetadataSource and/or /// [ApiMayChange] - public static SourceWithContext, Task> SourceWithOffsetContext( + public static SourceWithContext, IControl> SourceWithOffsetContext( ConsumerSettings settings, ISubscription subscription, Func, string> metadataFromRecord = null) { return Source.FromGraph(new SourceWithOffsetContextStage(settings, subscription, metadataFromRecord)) @@ -89,17 +89,17 @@ public static SourceWithContext, Task> S /// /// The same as but for offset commit support /// - public static Source, Task> CommittableExternalSource(IActorRef consumer, IManualSubscription subscription, + public static Source, IControl> CommittableExternalSource(IActorRef consumer, IManualSubscription subscription, string groupId, TimeSpan commitTimeout) { - return Source.FromGraph, Task>(new ExternalCommittableSourceStage(consumer, groupId, commitTimeout, subscription)); + return Source.FromGraph, IControl>(new ExternalCommittableSourceStage(consumer, groupId, commitTimeout, subscription)); } /// /// Convenience for "at-most once delivery" semantics. /// The offset of each message is committed to Kafka before being emitted downstream. /// - public static Source, Task> AtMostOnceSource(ConsumerSettings settings, ISubscription subscription) + public static Source, IControl> AtMostOnceSource(ConsumerSettings settings, ISubscription subscription) { return CommittableSource(settings, subscription).SelectAsync(1, async message => { diff --git a/src/Akka.Streams.Kafka/Extensions/ControlExtensions.cs b/src/Akka.Streams.Kafka/Extensions/ControlExtensions.cs new file mode 100644 index 00000000..01bd655d --- /dev/null +++ b/src/Akka.Streams.Kafka/Extensions/ControlExtensions.cs @@ -0,0 +1,47 @@ +using System; +using System.Threading.Tasks; +using Akka.Streams.Kafka.Helpers; + +namespace Akka.Streams.Kafka.Extensions +{ + /// + /// ControlExtensions + /// + public static class ControlExtensions + { + /// + /// Stop producing messages from the `Source`, wait for stream completion + /// and shut down the consumer `Source` so that all consumed messages + /// reach the end of the stream. + /// Failures in stream completion will be propagated, the source will be shut down anyway. + /// + public static async Task DrainAndShutdownDefault(this IControl control, Task streamCompletion) + { + TResult result; + + try + { + await control.Stop(); + + result = await streamCompletion; + } + catch (Exception completionError) + { + try + { + await control.Shutdown(); + + return await streamCompletion; + } + catch (Exception) + { + throw completionError; + } + } + + await control.Shutdown(); + + return result; + } + } +} \ No newline at end of file diff --git a/src/Akka.Streams.Kafka/Helpers/Control.cs b/src/Akka.Streams.Kafka/Helpers/Control.cs new file mode 100644 index 00000000..63efe553 --- /dev/null +++ b/src/Akka.Streams.Kafka/Helpers/Control.cs @@ -0,0 +1,110 @@ +using System; +using System.Threading.Tasks; +using Akka.Streams.Kafka.Extensions; +using Confluent.Kafka; + +namespace Akka.Streams.Kafka.Helpers +{ + /// + /// Materialized value of the consumer `Source`. + /// + public interface IControl + { + /// + /// Stop producing messages from the `Source`. This does not stop the underlying kafka consumer + /// and does not unsubscribe from any topics/partitions. + /// + /// Call to close consumer. + /// + Task Stop(); + + /// + /// Shutdown the consumer `Source`. It will wait for outstanding offset + /// commit requests to finish before shutting down. + /// + Task Shutdown(); + + /// + /// Shutdown status. The task will be completed when the stage has been shut down + /// and the underlying has been closed. Shutdown can be triggered + /// from downstream cancellation, errors, or + /// + Task IsShutdown(); + + /// + /// Stop producing messages from the `Source`, wait for stream completion + /// and shut down the consumer `Source` so that all consumed messages + /// reach the end of the stream. + /// Failures in stream completion will be propagated, the source will be shut down anyway. + /// + Task DrainAndShutdown(Task streamCompletion); + } + + /// + /// Combine control and a stream completion signal materialized values into + /// one, so that the stream can be stopped in a controlled way without losing + /// commits. + /// + public class DrainingControl : IControl + { + public IControl Control { get; } + public Task StreamCompletion { get; } + + /// + /// DrainingControl + /// + /// + /// + private DrainingControl(IControl control, Task streamCompletion) + { + Control = control; + StreamCompletion = streamCompletion; + } + + /// + public Task Stop() => Control.Stop(); + + /// + public Task Shutdown() => Control.Shutdown(); + + /// + public Task IsShutdown() => Control.IsShutdown(); + + /// + public Task DrainAndShutdown(Task streamCompletion) => Control.DrainAndShutdown(streamCompletion); + + /// + /// Stop producing messages from the `Source`, wait for stream completion + /// and shut down the consumer `Source` so that all consumed messages + /// reach the end of the stream. + /// + public Task DrainAndShutdown(Task streamCompletion) => Control.DrainAndShutdownDefault(StreamCompletion); + + /// + /// Combine control and a stream completion signal materialized values into + /// one, so that the stream can be stopped in a controlled way without losing + /// commits. + /// + public static DrainingControl Create(IControl control, Task streamCompletion) => new DrainingControl(control, streamCompletion); + } + + /// + /// An implementation of Control to be used as an empty value, all methods return a failed task. + /// + public class NoopControl : IControl + { + private static Exception Exception => new Exception("The correct Consumer.Control has not been assigned, yet."); + + /// + public Task Stop() => Task.FromException(Exception); + + /// + public Task Shutdown() => Task.FromException(Exception); + + /// + public Task IsShutdown() => Task.FromException(Exception); + + /// + public Task DrainAndShutdown(Task streamCompletion) => this.DrainAndShutdownDefault(streamCompletion); + } +} \ No newline at end of file diff --git a/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs b/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs new file mode 100644 index 00000000..069bacb3 --- /dev/null +++ b/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs @@ -0,0 +1,87 @@ +using System; +using System.Threading.Tasks; +using Akka.Streams.Kafka.Extensions; + +namespace Akka.Streams.Kafka.Helpers +{ + /// + /// Used in source logic classes to provide implementation. + /// + /// + class PromiseControl : IControl + { + private readonly SourceShape _shape; + private readonly Action> _completeStageOutlet; + private readonly Action _setStageKeepGoing; + + private readonly TaskCompletionSource _shutdownTaskSource = new TaskCompletionSource(); + private readonly TaskCompletionSource _stopTaskSource = new TaskCompletionSource(); + private readonly Action _stopCallback; + private readonly Action _shutdownCallback; + + public PromiseControl(SourceShape shape, Action> completeStageOutlet, + Action setStageKeepGoing, Func asyncCallbackFactory) + { + _shape = shape; + _completeStageOutlet = completeStageOutlet; + _setStageKeepGoing = setStageKeepGoing; + + _stopCallback = asyncCallbackFactory(PerformStop); + _shutdownCallback = asyncCallbackFactory(PerformShutdown); + } + + /// + public Task Stop() + { + _stopCallback(); + return _stopTaskSource.Task; + } + + /// + public Task Shutdown() + { + _shutdownCallback(); + return _shutdownTaskSource.Task; + } + + /// + public Task IsShutdown() => _shutdownTaskSource.Task; + + /// + public Task DrainAndShutdown(Task streamCompletion) => this.DrainAndShutdownDefault(streamCompletion); + + /// + /// Performs source logic stop + /// + public virtual void PerformStop() + { + _setStageKeepGoing(true); + _completeStageOutlet(_shape.Outlet); + OnStop(); + } + + /// + /// Performs source logic shutdown + /// + public virtual void PerformShutdown() + { + } + + /// + /// Executed on source logic stop + /// + public void OnStop() + { + _stopTaskSource.TrySetResult(Done.Instance); + } + + /// + /// Executed on source logic shutdown + /// + public void OnShutdown() + { + _stopTaskSource.TrySetResult(Done.Instance); + _shutdownTaskSource.TrySetResult(Done.Instance); + } + } +} \ No newline at end of file diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs index 01828793..ab24201e 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Akka.Actor; using Akka.Dispatch; +using Akka.Streams.Kafka.Helpers; using Akka.Streams.Kafka.Settings; using Akka.Streams.Kafka.Stages.Consumers.Actors; using Akka.Streams.Kafka.Stages.Consumers.Exceptions; @@ -26,7 +27,6 @@ namespace Akka.Streams.Kafka.Stages.Consumers.Abstract internal abstract class BaseSingleSourceLogic : GraphStageLogic { private readonly SourceShape _shape; - private readonly TaskCompletionSource _completion; private readonly IMessageBuilder _messageBuilder; private int _requestId = 0; private bool _requested = false; @@ -34,17 +34,27 @@ internal abstract class BaseSingleSourceLogic : GraphStageLogic private readonly ConcurrentQueue> _buffer = new ConcurrentQueue>(); protected IImmutableSet TopicPartitions { get; set; } = ImmutableHashSet.Create(); + + /// + /// Implements for logic control + /// + protected readonly PromiseControl InternalControl; protected StageActor SourceActor { get; private set; } internal IActorRef ConsumerActor { get; private set; } + + /// + /// Implements to provide control over executed source + /// + public virtual IControl Control => InternalControl; - protected BaseSingleSourceLogic(SourceShape shape, TaskCompletionSource completion, Attributes attributes, + protected BaseSingleSourceLogic(SourceShape shape, Attributes attributes, Func, IMessageBuilder> messageBuilderFactory) : base(shape) { _shape = shape; - _completion = completion; _messageBuilder = messageBuilderFactory(this); + InternalControl = new PromiseControl(_shape, Complete, SetKeepGoing, GetAsyncCallback); var supervisionStrategy = attributes.GetAttribute(null); _decider = supervisionStrategy != null ? supervisionStrategy.Decider : Deciders.ResumingDecider; @@ -65,7 +75,7 @@ public override void PreStart() public override void PostStop() { - OnShutdown(); + InternalControl.OnShutdown(); base.PostStop(); } @@ -80,19 +90,6 @@ public override void PostStop() /// protected abstract void ConfigureSubscription(); - /// - /// This is called when stage downstream is finished - /// - protected abstract void PerformShutdown(); - - /// - /// Makes this logic task finished - /// - protected void OnShutdown() - { - _completion.TrySetResult(NotUsed.Instance); - } - /// /// Configures manual subscription /// @@ -129,21 +126,7 @@ private void MessageHandling(Tuple args) break; case Status.Failure failure: - var exception = failure.Cause; - switch (_decider(failure.Cause)) - { - case Directive.Stop: - // Throw - _completion.TrySetException(exception); - FailStage(exception); - break; - case Directive.Resume: - // keep going - break; - case Directive.Restart: - // keep going - break; - } + FailStage(failure.Cause); break; case Terminated terminated: @@ -175,5 +158,7 @@ protected void RequestMessages() Log.Debug($"Requesting messages, requestId: {_requestId}, partitions: {string.Join(", ", TopicPartitions)}"); ConsumerActor.Tell(new KafkaConsumerActorMetadata.Internal.RequestMessages(_requestId, TopicPartitions.ToImmutableHashSet()), SourceActor.Ref); } + + protected abstract void PerformShutdown(); } } \ No newline at end of file diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/ExternalSingleSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/ExternalSingleSourceLogic.cs index 9553e762..81db946c 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/ExternalSingleSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/ExternalSingleSourceLogic.cs @@ -18,9 +18,9 @@ internal class ExternalSingleSourceLogic : BaseSingleSourceLogic private readonly IManualSubscription _subscription; public ExternalSingleSourceLogic(SourceShape shape, IActorRef consumerActor, IManualSubscription subscription, - TaskCompletionSource completion, Attributes attributes, + Attributes attributes, Func, IMessageBuilder> messageBuilderFactory) - : base(shape, completion, attributes, messageBuilderFactory) + : base(shape, attributes, messageBuilderFactory) { _consumerActor = consumerActor; _subscription = subscription; diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/KafkaSourceStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/KafkaSourceStage.cs index 4b766e00..27438dff 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/KafkaSourceStage.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/KafkaSourceStage.cs @@ -1,4 +1,5 @@ using System.Threading.Tasks; +using Akka.Streams.Kafka.Helpers; using Akka.Streams.Stage; namespace Akka.Streams.Kafka.Stages.Consumers.Abstract @@ -12,7 +13,7 @@ namespace Akka.Streams.Kafka.Stages.Consumers.Abstract /// Key type /// Value type /// Stage output messages type - public abstract class KafkaSourceStage : GraphStageWithMaterializedValue, Task> + public abstract class KafkaSourceStage : GraphStageWithMaterializedValue, IControl> { /// /// Name of the stage @@ -44,17 +45,15 @@ protected KafkaSourceStage(string stageName) /// Provides actual stage logic /// /// Shape of the stage - /// Used to specify stage task completion /// Stage attributes /// Stage logic - protected abstract GraphStageLogic Logic(SourceShape shape, TaskCompletionSource completion, Attributes inheritedAttributes); + protected abstract (GraphStageLogic, IControl) Logic(SourceShape shape, Attributes inheritedAttributes); /// - public override ILogicAndMaterializedValue CreateLogicAndMaterializedValue(Attributes inheritedAttributes) + public override ILogicAndMaterializedValue CreateLogicAndMaterializedValue(Attributes inheritedAttributes) { - var completion = new TaskCompletionSource(); - var result = Logic(Shape, completion, inheritedAttributes); - return new LogicAndMaterializedValue(result, completion.Task); + var (logic, materializedValue) = Logic(Shape, inheritedAttributes); + return new LogicAndMaterializedValue(logic, materializedValue); } } } \ No newline at end of file diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SingleSourceStageLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SingleSourceStageLogic.cs index 0c69c219..3d610114 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SingleSourceStageLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SingleSourceStageLogic.cs @@ -27,9 +27,8 @@ internal class SingleSourceStageLogic : BaseSingleSourceLogic shape, ConsumerSettings settings, ISubscription subscription, Attributes attributes, - TaskCompletionSource completion, Func, IMessageBuilder> messageBuilderFactory) - : base(shape, completion, attributes, messageBuilderFactory) + : base(shape, attributes, messageBuilderFactory) { _shape = shape; _settings = settings; @@ -105,7 +104,7 @@ private void ShuttingDownReceive(Tuple args) switch (args.Item2) { case Terminated terminated when terminated.ActorRef.Equals(ConsumerActor): - OnShutdown(); + InternalControl.OnShutdown(); CompleteStage(); break; default: diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/CommittableSourceStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/CommittableSourceStage.cs index 474f2a8a..49b526c2 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/CommittableSourceStage.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/CommittableSourceStage.cs @@ -1,6 +1,7 @@ using System; using System.Threading.Tasks; using Akka.Streams.Kafka.Dsl; +using Akka.Streams.Kafka.Helpers; using Akka.Streams.Kafka.Messages; using Akka.Streams.Kafka.Settings; using Akka.Streams.Kafka.Stages.Consumers.Abstract; @@ -46,12 +47,12 @@ public CommittableSourceStage(ConsumerSettings settings, ISubscription sub /// Provides actual stage logic /// /// Shape of the stage - /// Used to specify stage task completion /// Stage attributes /// Stage logic - protected override GraphStageLogic Logic(SourceShape> shape, TaskCompletionSource completion, Attributes inheritedAttributes) + protected override (GraphStageLogic, IControl) Logic(SourceShape> shape, Attributes inheritedAttributes) { - return new SingleSourceStageLogic>(shape, Settings, Subscription, inheritedAttributes, completion, GetMessageBuilder); + var logic = new SingleSourceStageLogic>(shape, Settings, Subscription, inheritedAttributes, GetMessageBuilder); + return (logic, logic.Control); } private CommittableSourceMessageBuilder GetMessageBuilder(BaseSingleSourceLogic> logic) diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/ExternalCommittableSourceStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/ExternalCommittableSourceStage.cs index ca840a5e..ca774cdd 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/ExternalCommittableSourceStage.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/ExternalCommittableSourceStage.cs @@ -3,6 +3,7 @@ using Akka.Actor; using Akka.Event; using Akka.Streams.Kafka.Dsl; +using Akka.Streams.Kafka.Helpers; using Akka.Streams.Kafka.Messages; using Akka.Streams.Kafka.Settings; using Akka.Streams.Kafka.Stages.Consumers.Abstract; @@ -48,9 +49,11 @@ public ExternalCommittableSourceStage(IActorRef consumer, string groupId, TimeSp } /// - protected override GraphStageLogic Logic(SourceShape> shape, TaskCompletionSource completion, Attributes inheritedAttributes) + protected override (GraphStageLogic, IControl) Logic(SourceShape> shape, Attributes inheritedAttributes) { - return new ExternalSingleSourceLogic>(shape, Consumer, Subscription, completion, inheritedAttributes, GetMessageBuilder); + var logic = new ExternalSingleSourceLogic>(shape, Consumer, Subscription, inheritedAttributes, GetMessageBuilder); + + return (logic, logic.Control); } private CommittableSourceMessageBuilder GetMessageBuilder(BaseSingleSourceLogic> logic) diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/ExternalPlainSourceStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/ExternalPlainSourceStage.cs index 73cc5db4..036dad04 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/ExternalPlainSourceStage.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/ExternalPlainSourceStage.cs @@ -1,5 +1,6 @@ using System.Threading.Tasks; using Akka.Actor; +using Akka.Streams.Kafka.Helpers; using Akka.Streams.Kafka.Settings; using Akka.Streams.Kafka.Stages.Consumers.Abstract; using Akka.Streams.Kafka.Stages.Consumers.Actors; @@ -37,10 +38,12 @@ public ExternalPlainSourceStage(IActorRef consumer, IManualSubscription subscrip } /// - protected override GraphStageLogic Logic(SourceShape> shape, TaskCompletionSource completion, Attributes inheritedAttributes) + protected override (GraphStageLogic, IControl) Logic(SourceShape> shape, Attributes inheritedAttributes) { - return new ExternalSingleSourceLogic>(shape, Consumer, Subscription, completion, - inheritedAttributes, _ => new PlainMessageBuilder()); + var logic = new ExternalSingleSourceLogic>(shape, Consumer, Subscription, + inheritedAttributes, _ => new PlainMessageBuilder()); + + return (logic, logic.Control); } } } \ No newline at end of file diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSourceStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSourceStage.cs index d0a9ab0a..7e15086a 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSourceStage.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSourceStage.cs @@ -1,5 +1,6 @@ using System.Threading.Tasks; using Akka.Streams.Kafka.Dsl; +using Akka.Streams.Kafka.Helpers; using Akka.Streams.Kafka.Settings; using Akka.Streams.Kafka.Stages.Consumers.Abstract; using Akka.Streams.Stage; @@ -39,13 +40,13 @@ public PlainSourceStage(ConsumerSettings settings, ISubscription subscript /// Provides actual stage logic /// /// Shape of the stage - /// Used to specify stage task completion /// Stage attributes /// Stage logic - protected override GraphStageLogic Logic(SourceShape> shape, TaskCompletionSource completion, Attributes inheritedAttributes) + protected override (GraphStageLogic, IControl) Logic(SourceShape> shape, Attributes inheritedAttributes) { - return new SingleSourceStageLogic>(shape, Settings, Subscription, inheritedAttributes, - completion, _ => new PlainMessageBuilder()); + var logic = new SingleSourceStageLogic>(shape, Settings, Subscription, inheritedAttributes, + _ => new PlainMessageBuilder()); + return (logic, logic.Control); } } } diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/SourceWithOffsetContextStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/SourceWithOffsetContextStage.cs index 591ab881..dfe124eb 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/SourceWithOffsetContextStage.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/SourceWithOffsetContextStage.cs @@ -1,6 +1,7 @@ using System; using System.Threading.Tasks; using Akka.Streams.Kafka.Dsl; +using Akka.Streams.Kafka.Helpers; using Akka.Streams.Kafka.Messages; using Akka.Streams.Kafka.Settings; using Akka.Streams.Kafka.Stages.Consumers.Abstract; @@ -46,14 +47,14 @@ public SourceWithOffsetContextStage(ConsumerSettings settings, ISubscripti } /// - protected override GraphStageLogic Logic( + protected override (GraphStageLogic, IControl) Logic( SourceShape<(ConsumeResult, ICommittableOffset)> shape, - TaskCompletionSource completion, Attributes inheritedAttributes) { - return new SingleSourceStageLogic, ICommittableOffset)>(shape, Settings, Subscription, - inheritedAttributes, completion, - GetMessageBuilder); + var logic = new SingleSourceStageLogic, ICommittableOffset)>(shape, Settings, Subscription, + inheritedAttributes, + GetMessageBuilder); + return (logic, logic.Control); } private OffsetContextBuilder GetMessageBuilder(BaseSingleSourceLogic, ICommittableOffset)> logic) From 8c5ad54c61ae4039087bf1de9fb520da43b2c0dd Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Sat, 5 Oct 2019 15:48:48 +0300 Subject: [PATCH 10/28] Fixed PerformShutdown overriding --- src/Akka.Streams.Kafka/Helpers/Control.cs | 6 +++--- .../Helpers/PromiseControl.cs | 12 +++++++++--- .../Abstract/BaseSingleSourceLogic.cs | 18 ++++++++++++++++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/Akka.Streams.Kafka/Helpers/Control.cs b/src/Akka.Streams.Kafka/Helpers/Control.cs index 63efe553..c9978931 100644 --- a/src/Akka.Streams.Kafka/Helpers/Control.cs +++ b/src/Akka.Streams.Kafka/Helpers/Control.cs @@ -29,7 +29,7 @@ public interface IControl /// and the underlying has been closed. Shutdown can be triggered /// from downstream cancellation, errors, or /// - Task IsShutdown(); + Task IsShutdown { get; } /// /// Stop producing messages from the `Source`, wait for stream completion @@ -68,7 +68,7 @@ private DrainingControl(IControl control, Task streamCompletion) public Task Shutdown() => Control.Shutdown(); /// - public Task IsShutdown() => Control.IsShutdown(); + public Task IsShutdown => Control.IsShutdown; /// public Task DrainAndShutdown(Task streamCompletion) => Control.DrainAndShutdown(streamCompletion); @@ -102,7 +102,7 @@ public class NoopControl : IControl public Task Shutdown() => Task.FromException(Exception); /// - public Task IsShutdown() => Task.FromException(Exception); + public Task IsShutdown => Task.FromException(Exception); /// public Task DrainAndShutdown(Task streamCompletion) => this.DrainAndShutdownDefault(streamCompletion); diff --git a/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs b/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs index 069bacb3..5fbb2be3 100644 --- a/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs +++ b/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs @@ -1,6 +1,7 @@ using System; using System.Threading.Tasks; using Akka.Streams.Kafka.Extensions; +using Akka.Streams.Util; namespace Akka.Streams.Kafka.Helpers { @@ -13,18 +14,21 @@ class PromiseControl : IControl private readonly SourceShape _shape; private readonly Action> _completeStageOutlet; private readonly Action _setStageKeepGoing; - + private readonly Option _performShutdown; + private readonly TaskCompletionSource _shutdownTaskSource = new TaskCompletionSource(); private readonly TaskCompletionSource _stopTaskSource = new TaskCompletionSource(); private readonly Action _stopCallback; private readonly Action _shutdownCallback; public PromiseControl(SourceShape shape, Action> completeStageOutlet, - Action setStageKeepGoing, Func asyncCallbackFactory) + Action setStageKeepGoing, Func asyncCallbackFactory, + Option performShutdown) { _shape = shape; _completeStageOutlet = completeStageOutlet; _setStageKeepGoing = setStageKeepGoing; + _performShutdown = performShutdown; _stopCallback = asyncCallbackFactory(PerformStop); _shutdownCallback = asyncCallbackFactory(PerformShutdown); @@ -45,7 +49,7 @@ public Task Shutdown() } /// - public Task IsShutdown() => _shutdownTaskSource.Task; + public Task IsShutdown => _shutdownTaskSource.Task; /// public Task DrainAndShutdown(Task streamCompletion) => this.DrainAndShutdownDefault(streamCompletion); @@ -65,6 +69,8 @@ public virtual void PerformStop() /// public virtual void PerformShutdown() { + if (_performShutdown.HasValue) + _performShutdown.Value.Invoke(); } /// diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs index ab24201e..a4c55518 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs @@ -12,6 +12,7 @@ using Akka.Streams.Kafka.Stages.Consumers.Exceptions; using Akka.Streams.Stage; using Akka.Streams.Supervision; +using Akka.Streams.Util; using Confluent.Kafka; using Decider = Akka.Streams.Supervision.Decider; using Directive = Akka.Streams.Supervision.Directive; @@ -54,7 +55,7 @@ protected BaseSingleSourceLogic(SourceShape shape, Attributes attribut { _shape = shape; _messageBuilder = messageBuilderFactory(this); - InternalControl = new PromiseControl(_shape, Complete, SetKeepGoing, GetAsyncCallback); + InternalControl = new PromiseControl(_shape, Complete, SetKeepGoing, GetAsyncCallback, new Option(PerformShutdown)); var supervisionStrategy = attributes.GetAttribute(null); _decider = supervisionStrategy != null ? supervisionStrategy.Decider : Deciders.ResumingDecider; @@ -126,7 +127,20 @@ private void MessageHandling(Tuple args) break; case Status.Failure failure: - FailStage(failure.Cause); + var exception = failure.Cause; + switch (_decider(failure.Cause)) + { + case Directive.Stop: + // Throw + FailStage(exception); + break; + case Directive.Resume: + // keep going + break; + case Directive.Restart: + // TODO: Need to do something here: https://github.com/akkadotnet/Akka.Streams.Kafka/issues/33 + break; + } break; case Terminated terminated: From 2f68bd7b8554af4b9061b3cd3af233e62179f437 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Sat, 5 Oct 2019 15:48:53 +0300 Subject: [PATCH 11/28] Updated tests --- .../AtMostOnceSourceIntegrationTests.cs | 4 ++-- ...ommitWithMetadataSourceIntegrationTests.cs | 4 ++-- .../CommittableSourceIntegrationTests.cs | 2 +- .../ExternalPlainSourceIntegrationTests.cs | 20 +++++++++---------- .../PlainSourceIntegrationTests.cs | 20 +++++++++++-------- ...SourceWithOffsetContextIntegrationTests.cs | 4 ++-- .../KafkaIntegrationTests.cs | 3 ++- .../TestsConfiguration.cs | 2 +- 8 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs index c34a58ad..001ca6d8 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs @@ -26,13 +26,13 @@ public async Task AtMostOnceSource_Should_stop_consuming_actor_when_used_with_Ta await ProduceStrings(topic, Enumerable.Range(1, 10), ProducerSettings); - var (task, result) = KafkaConsumer.AtMostOnceSource(CreateConsumerSettings(group), Subscriptions.Topics(topic)) + var (control, result) = KafkaConsumer.AtMostOnceSource(CreateConsumerSettings(group), Subscriptions.Topics(topic)) .Select(m => m.Value) .Take(5) .ToMaterialized(Sink.Seq(), Keep.Both) .Run(Materializer); - AwaitCondition(() => task.IsCompletedSuccessfully, TimeSpan.FromSeconds(10)); + AwaitCondition(() => control.IsShutdown.IsCompletedSuccessfully, TimeSpan.FromSeconds(10)); result.Result.Should().BeEquivalentTo(Enumerable.Range(1, 5).Select(i => i.ToString())); } diff --git a/src/Akka.Streams.Kafka.Tests/Integration/CommitWithMetadataSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/CommitWithMetadataSourceIntegrationTests.cs index 4b0ec3a8..fb8d3b45 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/CommitWithMetadataSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/CommitWithMetadataSourceIntegrationTests.cs @@ -37,7 +37,7 @@ public async Task CommitWithMetadataSource_Commit_metadata_in_message_Should_wor await ProduceStrings(topic, Enumerable.Range(1, 10), ProducerSettings); - var (task, probe) = KafkaConsumer.CommitWithMetadataSource(CreateConsumerSettings(group), Subscriptions.Topics(topic), MetadataFromMessage) + var (control, probe) = KafkaConsumer.CommitWithMetadataSource(CreateConsumerSettings(group), Subscriptions.Topics(topic), MetadataFromMessage) .ToMaterialized(this.SinkProbe>(), Keep.Both) .Run(Materializer); @@ -52,7 +52,7 @@ public async Task CommitWithMetadataSource_Commit_metadata_in_message_Should_wor probe.Cancel(); - AwaitCondition(() => task.IsCompletedSuccessfully, TimeSpan.FromSeconds(10)); + AwaitCondition(() => control.IsShutdown.IsCompletedSuccessfully, TimeSpan.FromSeconds(10)); } } } \ No newline at end of file diff --git a/src/Akka.Streams.Kafka.Tests/Integration/CommittableSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/CommittableSourceIntegrationTests.cs index 779edc81..0ab47e91 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/CommittableSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/CommittableSourceIntegrationTests.cs @@ -91,7 +91,7 @@ await Source probe1.Cancel(); - AwaitCondition(() => task.IsCompletedSuccessfully); + AwaitCondition(() => task.IsShutdown.IsCompletedSuccessfully); var probe2 = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.Assignment(new TopicPartition(topic1, 0))) .Select(_ => _.Record.Value) diff --git a/src/Akka.Streams.Kafka.Tests/Integration/ExternalPlainSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/ExternalPlainSourceIntegrationTests.cs index 4989ad29..5b560210 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/ExternalPlainSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/ExternalPlainSourceIntegrationTests.cs @@ -37,8 +37,8 @@ public async Task ExternalPlainSource_with_external_consumer_Should_work() var consumer = Sys.ActorOf(KafkaConsumerActorMetadata.GetProps(CreateConsumerSettings(group))); //Manually assign topic partition to it - var (partitionTask1, probe1) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 0))); - var (partitionTask2, probe2) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 1))); + var (control1, probe1) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 0))); + var (control2, probe2) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 1))); // Produce messages to partitions await ProduceStrings(new TopicPartition(topic, new Partition(0)), Enumerable.Range(1, elementsCount), ProducerSettings); @@ -55,7 +55,7 @@ public async Task ExternalPlainSource_with_external_consumer_Should_work() probe2.Cancel(); // Make sure stages are stopped gracefully - AwaitCondition(() => partitionTask1.IsCompletedSuccessfully && partitionTask2.IsCompletedSuccessfully); + AwaitCondition(() => control1.IsShutdown.IsCompletedSuccessfully && control2.IsShutdown.IsCompletedSuccessfully); // Cleanup consumer.Tell(new KafkaConsumerActorMetadata.Internal.Stop(), ActorRefs.NoSender); @@ -72,9 +72,9 @@ public async Task ExternalPlainSource_should_be_stopped_on_serialization_error_o var consumer = Sys.ActorOf(KafkaConsumerActorMetadata.GetProps(settings)); // Subscribe to partitions - var (partitionTask1, probe1) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 0))); - var (partitionTask2, probe2) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 1))); - var (partitionTask3, probe3) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 2))); + var (control1, probe1) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 0))); + var (control2, probe2) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 1))); + var (control3, probe3) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 2))); // request from 2 streams probe1.Request(1); @@ -91,7 +91,7 @@ public async Task ExternalPlainSource_should_be_stopped_on_serialization_error_o probe3.Cancel(); // Make sure source tasks finish accordingly - AwaitCondition(() => partitionTask1.IsFaulted && partitionTask2.IsFaulted && partitionTask3.IsCompletedSuccessfully); + AwaitCondition(() => control1.IsShutdown.IsCompleted && control2.IsShutdown.IsCompleted && control3.IsShutdown.IsCompletedSuccessfully); // Cleanup consumer.Tell(new KafkaConsumerActorMetadata.Internal.Stop(), ActorRefs.NoSender); @@ -111,8 +111,8 @@ public async Task ExternalPlainSource_verify_consuming_actor_pause_resume_partit await ProduceStrings(new TopicPartition(topic, 1), Enumerable.Range(1, 100), ProducerSettings); // Subscribe to partitions - var (partitionTask1, probe1) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 0))); - var (partitionTask2, probe2) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 1))); + var (control1, probe1) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 0))); + var (control2, probe2) = CreateExternalPlainSourceProbe(consumer, Subscriptions.Assignment(new TopicPartition(topic, 1))); var probes = new[] { probe1, probe2 }; @@ -140,7 +140,7 @@ public async Task ExternalPlainSource_verify_consuming_actor_pause_resume_partit // Stop and check gracefull shutdown probes.ForEach(p => p.Cancel()); - AwaitCondition(() => partitionTask1.IsCompletedSuccessfully && partitionTask2.IsCompletedSuccessfully); + AwaitCondition(() => control1.IsShutdown.IsCompletedSuccessfully && control2.IsShutdown.IsCompletedSuccessfully); // Cleanup consumer.Tell(new KafkaConsumerActorMetadata.Internal.Stop(), ActorRefs.NoSender); diff --git a/src/Akka.Streams.Kafka.Tests/Integration/PlainSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/PlainSourceIntegrationTests.cs index 565bb681..ace678d9 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/PlainSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/PlainSourceIntegrationTests.cs @@ -7,6 +7,7 @@ using Akka.Configuration; using Akka.Streams.Dsl; using Akka.Streams.Kafka.Dsl; +using Akka.Streams.Kafka.Helpers; using Akka.Streams.Kafka.Messages; using Akka.Streams.Kafka.Settings; using Akka.Streams.Kafka.Tests.Logging; @@ -28,13 +29,14 @@ public PlainSourceIntegrationTests(ITestOutputHelper output, KafkaFixture fixtur } - private TestSubscriber.Probe CreateProbe(ConsumerSettings consumerSettings, ISubscription sub) + private Tuple> CreateProbe(ConsumerSettings consumerSettings, ISubscription sub) { return KafkaConsumer .PlainSource(consumerSettings, sub) .Where(c => !c.Value.Equals(InitialMsg)) .Select(c => c.Value) - .RunWith(this.SinkProbe(), Materializer); + .ToMaterialized(this.SinkProbe(), Keep.Both) + .Run(Materializer); } [Fact] @@ -51,7 +53,7 @@ public async Task PlainSource_consumes_messages_from_KafkaProducer_with_topicPar var consumerSettings = CreateConsumerSettings(group1); - var probe = CreateProbe(consumerSettings, Subscriptions.Assignment(topicPartition1)); + var (_, probe) = CreateProbe(consumerSettings, Subscriptions.Assignment(topicPartition1)); probe.Request(elementsCount); foreach (var i in Enumerable.Range(1, elementsCount).Select(c => c.ToString())) @@ -75,7 +77,7 @@ public async Task PlainSource_consumes_messages_from_KafkaProducer_with_topicPar var consumerSettings = CreateConsumerSettings(group1); - var probe = CreateProbe(consumerSettings, Subscriptions.AssignmentWithOffset(new TopicPartitionOffset(topicPartition1, new Offset(offset)))); + var (_, probe) = CreateProbe(consumerSettings, Subscriptions.AssignmentWithOffset(new TopicPartitionOffset(topicPartition1, new Offset(offset)))); probe.Request(elementsCount); foreach (var i in Enumerable.Range(offset, elementsCount - offset).Select(c => c.ToString())) @@ -98,13 +100,14 @@ public async Task PlainSource_consumes_messages_from_KafkaProducer_with_subscrib var consumerSettings = CreateConsumerSettings(group1); - var probe = CreateProbe(consumerSettings, Subscriptions.Topics(topic1)); + var (control, probe) = CreateProbe(consumerSettings, Subscriptions.Topics(topic1)); probe.Request(elementsCount); foreach (var i in Enumerable.Range(1, elementsCount).Select(c => c.ToString())) probe.ExpectNext(i, TimeSpan.FromSeconds(10)); - probe.Cancel(); + var shutdown = control.Shutdown(); + AwaitCondition(() => shutdown.IsCompleted); } [Fact] @@ -120,8 +123,9 @@ public async Task PlainSource_should_fail_stage_if_broker_unavailable() .WithBootstrapServers("localhost:10092") .WithGroupId(group1); - var probe = CreateProbe(config, Subscriptions.Assignment(topicPartition1)); - probe.Request(1).ExpectError().Should().BeOfType(); + var (control, probe) = CreateProbe(config, Subscriptions.Assignment(topicPartition1)); + probe.Request(1); + AwaitCondition(() => control.IsShutdown.IsCompleted, TimeSpan.FromSeconds(10)); } [Fact] diff --git a/src/Akka.Streams.Kafka.Tests/Integration/SourceWithOffsetContextIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/SourceWithOffsetContextIntegrationTests.cs index 95e54864..308fab9f 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/SourceWithOffsetContextIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/SourceWithOffsetContextIntegrationTests.cs @@ -33,7 +33,7 @@ public async Task SourceWithOffsetContext_at_least_once_consuming_should_work() var committerSettings = CommitterSettings.WithMaxBatch(batchSize); - var (task, probe) = KafkaConsumer.SourceWithOffsetContext(settings, Subscriptions.Topics(topic)) + var (control, probe) = KafkaConsumer.SourceWithOffsetContext(settings, Subscriptions.Topics(topic)) .SelectAsync(10, message => Task.FromResult(Done.Instance)) .Via(Committer.FlowWithOffsetContext(committerSettings)) .AsSource() @@ -45,7 +45,7 @@ public async Task SourceWithOffsetContext_at_least_once_consuming_should_work() probe.Cancel(); - AwaitCondition(() => task.IsCompletedSuccessfully, TimeSpan.FromSeconds(10)); + AwaitCondition(() => control.IsShutdown.IsCompletedSuccessfully, TimeSpan.FromSeconds(10)); committedBatches.Select(r => r.Item2).Sum(batch => batch.BatchSize).Should().Be(10); } diff --git a/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs index 7583710e..412f34ea 100644 --- a/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs @@ -5,6 +5,7 @@ using Akka.Configuration; using Akka.Streams.Dsl; using Akka.Streams.Kafka.Dsl; +using Akka.Streams.Kafka.Helpers; using Akka.Streams.Kafka.Messages; using Akka.Streams.Kafka.Settings; using Akka.Streams.TestKit; @@ -90,7 +91,7 @@ protected async Task GivenInitializedTopic(TopicPartition topicPartition) } } - protected Tuple> CreateExternalPlainSourceProbe(IActorRef consumer, IManualSubscription sub) + protected Tuple> CreateExternalPlainSourceProbe(IActorRef consumer, IManualSubscription sub) { return KafkaConsumer .PlainExternalSource(consumer, sub) diff --git a/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs b/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs index fbc71855..e09bb894 100644 --- a/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs +++ b/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs @@ -18,6 +18,6 @@ public static class TestsConfiguration /// /// When this option is enabled, use docker-compose to start kafka manually (see docker-compose.yml file in the root folder) /// - public static readonly bool UseExistingDockerContainer = Environment.GetEnvironmentVariable("AKKA_STREAMS_KAFKA_TEST_CONTAINER_REUSE") != null; + public static readonly bool UseExistingDockerContainer = false && Environment.GetEnvironmentVariable("AKKA_STREAMS_KAFKA_TEST_CONTAINER_REUSE") != null; } } \ No newline at end of file From e9f5752d677348482eaad4d36507f3c64d943ac1 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Sat, 5 Oct 2019 15:55:20 +0300 Subject: [PATCH 12/28] Removed temporary debugging flag --- src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs b/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs index e09bb894..fbc71855 100644 --- a/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs +++ b/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs @@ -18,6 +18,6 @@ public static class TestsConfiguration /// /// When this option is enabled, use docker-compose to start kafka manually (see docker-compose.yml file in the root folder) /// - public static readonly bool UseExistingDockerContainer = false && Environment.GetEnvironmentVariable("AKKA_STREAMS_KAFKA_TEST_CONTAINER_REUSE") != null; + public static readonly bool UseExistingDockerContainer = Environment.GetEnvironmentVariable("AKKA_STREAMS_KAFKA_TEST_CONTAINER_REUSE") != null; } } \ No newline at end of file From 77c5c4ee6a1e226cdc1a284cb45c8b15433b64ca Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Sat, 5 Oct 2019 16:42:37 +0300 Subject: [PATCH 13/28] Changed the way to override `PromiseControl` virtual methods --- .../Helpers/PromiseControl.cs | 7 +------ .../Abstract/BaseSingleSourceLogic.cs | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs b/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs index 5fbb2be3..d672c6f7 100644 --- a/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs +++ b/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs @@ -14,7 +14,6 @@ class PromiseControl : IControl private readonly SourceShape _shape; private readonly Action> _completeStageOutlet; private readonly Action _setStageKeepGoing; - private readonly Option _performShutdown; private readonly TaskCompletionSource _shutdownTaskSource = new TaskCompletionSource(); private readonly TaskCompletionSource _stopTaskSource = new TaskCompletionSource(); @@ -22,13 +21,11 @@ class PromiseControl : IControl private readonly Action _shutdownCallback; public PromiseControl(SourceShape shape, Action> completeStageOutlet, - Action setStageKeepGoing, Func asyncCallbackFactory, - Option performShutdown) + Action setStageKeepGoing, Func asyncCallbackFactory) { _shape = shape; _completeStageOutlet = completeStageOutlet; _setStageKeepGoing = setStageKeepGoing; - _performShutdown = performShutdown; _stopCallback = asyncCallbackFactory(PerformStop); _shutdownCallback = asyncCallbackFactory(PerformShutdown); @@ -69,8 +66,6 @@ public virtual void PerformStop() /// public virtual void PerformShutdown() { - if (_performShutdown.HasValue) - _performShutdown.Value.Invoke(); } /// diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs index a4c55518..c79b2ce5 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs @@ -20,7 +20,7 @@ namespace Akka.Streams.Kafka.Stages.Consumers.Abstract { /// - /// Shared GraphStageLogic for and + /// Shared GraphStageLogic for and /// /// Key type /// Value type @@ -55,7 +55,7 @@ protected BaseSingleSourceLogic(SourceShape shape, Attributes attribut { _shape = shape; _messageBuilder = messageBuilderFactory(this); - InternalControl = new PromiseControl(_shape, Complete, SetKeepGoing, GetAsyncCallback, new Option(PerformShutdown)); + InternalControl = new BaseSingleSourceControl(_shape, Complete, SetKeepGoing, GetAsyncCallback, PerformShutdown); var supervisionStrategy = attributes.GetAttribute(null); _decider = supervisionStrategy != null ? supervisionStrategy.Decider : Deciders.ResumingDecider; @@ -174,5 +174,19 @@ protected void RequestMessages() } protected abstract void PerformShutdown(); + + protected class BaseSingleSourceControl : PromiseControl + { + private readonly Action _performShutdown; + + public BaseSingleSourceControl(SourceShape shape, Action> completeStageOutlet, Action setStageKeepGoing, + Func asyncCallbackFactory, Action performShutdown) + : base(shape, completeStageOutlet, setStageKeepGoing, asyncCallbackFactory) + { + _performShutdown = performShutdown; + } + + public override void PerformShutdown() => _performShutdown(); + } } } \ No newline at end of file From 7c7cc2960a583569691a63379c59b88ce009911b Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Sat, 5 Oct 2019 17:33:56 +0300 Subject: [PATCH 14/28] Updated implementation to use IControl --- .../PlainPartitionedSourceIntegrationTests.cs | 27 ++-- .../TestsConfiguration.cs | 2 +- src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs | 6 +- src/Akka.Streams.Kafka/Helpers/Control.cs | 2 +- .../Consumers/Abstract/SubSourceLogic.cs | 127 +++++++++++++----- .../Consumers/Concrete/PlainSubSourceStage.cs | 10 +- 6 files changed, 119 insertions(+), 55 deletions(-) diff --git a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs index ca8419a9..0135d32b 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using Akka.Streams.Dsl; using Akka.Streams.Kafka.Dsl; +using Akka.Streams.Kafka.Helpers; using Akka.Streams.Kafka.Messages; using Akka.Streams.Kafka.Settings; using Akka.Streams.TestKit; @@ -21,7 +22,7 @@ public PlainPartitionedSourceIntegrationTests(ITestOutputHelper output, KafkaFix } [Fact] - public async Task PlainPartitionedSource_Should_not_lose_any_messages_when_Kafka_node_dies() + public async Task PlainPartitionedSource_should_work() { var topic = CreateTopic(1); var group = CreateGroup(1); @@ -29,9 +30,7 @@ public async Task PlainPartitionedSource_Should_not_lose_any_messages_when_Kafka var consumerSettings = CreateConsumerSettings(group); - await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); - - var (consumeTask, probe) = KafkaConsumer.PlainPartitionedSource(consumerSettings, Subscriptions.Topics(topic)) + var control = KafkaConsumer.PlainPartitionedSource(consumerSettings, Subscriptions.Topics(topic)) .GroupBy(3, tuple => tuple.Item1) .SelectAsync(8, async tuple => { @@ -46,22 +45,20 @@ public async Task PlainPartitionedSource_Should_not_lose_any_messages_when_Kafka return sourceMessages; }) .MergeSubstreams() - .As>() + .As>() .Scan(0L, (i, subValue) => i + subValue) - .ToMaterialized(this.SinkProbe(), Keep.Both) + .ToMaterialized(Sink.Last(), Keep.Both) + .MapMaterializedValue(tuple => DrainingControl.Create(tuple.Item1, tuple.Item2)) .Run(Materializer); - AwaitCondition(() => - { - Log.Debug("Expecting next number..."); - var next = probe.RequestNext(TimeSpan.FromSeconds(10)); - Log.Debug("Got requested number: " + next); - return next == totalMessages; - }, TimeSpan.FromSeconds(20)); + await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); - probe.Cancel(); + // Give it some time to consume all messages + await Task.Delay(5000); - AwaitCondition(() => consumeTask.IsCompletedSuccessfully); + var shutdown = control.DrainAndShutdown(); + AwaitCondition(() => shutdown.IsCompleted, TimeSpan.FromSeconds(10)); + shutdown.Result.Should().Be(totalMessages); } private int LogSentMessages(int counter) diff --git a/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs b/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs index fbc71855..e09bb894 100644 --- a/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs +++ b/src/Akka.Streams.Kafka.Tests/TestsConfiguration.cs @@ -18,6 +18,6 @@ public static class TestsConfiguration /// /// When this option is enabled, use docker-compose to start kafka manually (see docker-compose.yml file in the root folder) /// - public static readonly bool UseExistingDockerContainer = Environment.GetEnvironmentVariable("AKKA_STREAMS_KAFKA_TEST_CONTAINER_REUSE") != null; + public static readonly bool UseExistingDockerContainer = false && Environment.GetEnvironmentVariable("AKKA_STREAMS_KAFKA_TEST_CONTAINER_REUSE") != null; } } \ No newline at end of file diff --git a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs index 5ee28ad9..9b3fc0cc 100644 --- a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs +++ b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs @@ -21,7 +21,7 @@ namespace Akka.Streams.Kafka.Dsl public static class KafkaConsumer { /// - /// The emits elements (as received from the underlying + /// The emits elements (as received from the underlying /// ). It has no support for committing offsets to Kafka. It can be used when the /// offset is stored externally or with auto-commit (note that auto-commit is by default disabled). /// The consumer application doesn't need to use Kafka's built-in offset storage and can store offsets in a store of its own @@ -63,8 +63,8 @@ public static Source, IControl> CommittableSource /// source of `ConsumerRecord`s. /// When a topic-partition is revoked, the corresponding source completes. /// - public static Source<(TopicPartition, Source, NotUsed>), Task> PlainPartitionedSource(ConsumerSettings settings, - IAutoSubscription subscription) + public static Source<(TopicPartition, Source, NotUsed>), IControl> PlainPartitionedSource(ConsumerSettings settings, + IAutoSubscription subscription) { return Source.FromGraph(new PlainSubSourceStage(settings, subscription, Option, Task>>>.None, diff --git a/src/Akka.Streams.Kafka/Helpers/Control.cs b/src/Akka.Streams.Kafka/Helpers/Control.cs index c9978931..d2d235d9 100644 --- a/src/Akka.Streams.Kafka/Helpers/Control.cs +++ b/src/Akka.Streams.Kafka/Helpers/Control.cs @@ -78,7 +78,7 @@ private DrainingControl(IControl control, Task streamCompletion) /// and shut down the consumer `Source` so that all consumed messages /// reach the end of the stream. /// - public Task DrainAndShutdown(Task streamCompletion) => Control.DrainAndShutdownDefault(StreamCompletion); + public Task DrainAndShutdown() => Control.DrainAndShutdown(StreamCompletion); /// /// Combine control and a stream completion signal materialized values into diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs index b6a9a993..8608c7d2 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs @@ -5,6 +5,7 @@ using System.Runtime.InteropServices; using System.Threading.Tasks; using Akka.Actor; +using Akka.Event; using Akka.Streams.Dsl; using Akka.Streams.Kafka.Extensions; using Akka.Streams.Kafka.Helpers; @@ -31,15 +32,14 @@ private class CloseRevokedPartitions { } private readonly IMessageBuilder _messageBuilder; private readonly Option, Task>>> _getOffsetsOnAssign; private readonly Action> _onRevoke; - private readonly TaskCompletionSource _completion; private readonly int _actorNumber = KafkaConsumerActorMetadata.NextNumber(); - private Action> _partitionAssignedCallback; + private readonly Action> _partitionAssignedCallback; private Action> _updatePendingPartitionsAndEmitSubSourcesCallback; - private Action> _partitionRevokedCallback; - private Action<(TopicPartition, Option>)> _subsourceCancelledCallback; - private Action<(TopicPartition, TaskCompletionSource)> _subsourceStartedCallback; - private Action _stageFailCallback; + private readonly Action> _partitionRevokedCallback; + private readonly Action<(TopicPartition, Option>)> _subsourceCancelledCallback; + private readonly Action<(TopicPartition, IControl)> _subsourceStartedCallback; + private readonly Action _stageFailCallback; /// /// Kafka has notified us that we have these partitions assigned, but we have not created a source for them yet. @@ -50,7 +50,7 @@ private class CloseRevokedPartitions { } /// We have created a source for these partitions, but it has not started up and is not in subSources yet. /// private IImmutableSet _partitionsInStartup = ImmutableHashSet.Empty; - private IImmutableDictionary> _subSources = ImmutableDictionary>.Empty; + private IImmutableDictionary _subSources = ImmutableDictionary.Empty; /// /// Kafka has signalled these partitions are revoked, but some may be re-assigned just after revoking. @@ -60,6 +60,10 @@ private class CloseRevokedPartitions { } protected StageActor SourceActor { get; private set; } protected IActorRef ConsumerActor { get; private set; } + + protected SubSourcePromiseControl InternalControl { get; } + + public IControl Control => InternalControl; /// /// SubSourceLogic @@ -67,7 +71,7 @@ private class CloseRevokedPartitions { } public SubSourceLogic(SourceShape<(TopicPartition, Source)> shape, ConsumerSettings settings, IAutoSubscription subscription, Func, IMessageBuilder> messageBuilderFactory, Option, Task>>> getOffsetsOnAssign, - Action> onRevoke, TaskCompletionSource completion) + Action> onRevoke) : base(shape) { _shape = shape; @@ -76,14 +80,15 @@ public SubSourceLogic(SourceShape<(TopicPartition, Source)> s _messageBuilder = messageBuilderFactory(this); _getOffsetsOnAssign = getOffsetsOnAssign; _onRevoke = onRevoke; - _completion = completion; + + InternalControl = new SubSourcePromiseControl(_shape, Complete, SetKeepGoing, GetAsyncCallback, PerformStop, PerformShutdown); _updatePendingPartitionsAndEmitSubSourcesCallback = GetAsyncCallback>(UpdatePendingPartitionsAndEmitSubSources); _partitionAssignedCallback = GetAsyncCallback>(HandlePartitionsAssigned); _partitionRevokedCallback = GetAsyncCallback>(HandlePartitionsRevoked); _stageFailCallback = GetAsyncCallback(FailStage); _subsourceCancelledCallback = GetAsyncCallback<(TopicPartition, Option>)>(HandleSubsourceCancelled); - _subsourceStartedCallback = GetAsyncCallback<(TopicPartition, TaskCompletionSource)>(HandleSubsourceStarted); + _subsourceStartedCallback = GetAsyncCallback<(TopicPartition, IControl)>(HandleSubsourceStarted); SetHandler(shape.Outlet, onPull: EmitSubSourcesForPendingPartitions, onDownstreamFinish: PerformShutdown); } @@ -133,7 +138,7 @@ public override void PostStop() { ConsumerActor.Tell(new KafkaConsumerActorMetadata.Internal.Stop(), SourceActor.Ref); - OnShutdown(); + InternalControl.OnShutdown(); base.PostStop(); } @@ -147,7 +152,7 @@ protected override void OnTimer(object timerKey) _onRevoke(_partitionsToRevoke); _pendingPartitions = _pendingPartitions.Except(_partitionsToRevoke); _partitionsInStartup = _partitionsInStartup.Except(_partitionsToRevoke); - _partitionsToRevoke.ForEach(tp => _subSources[tp].SetResult(Done.Instance)); + _partitionsToRevoke.ForEach(tp => _subSources[tp].Shutdown()); _subSources = _subSources.RemoveRange(_partitionsToRevoke); _partitionsToRevoke = _partitionsToRevoke.Clear(); } @@ -190,7 +195,7 @@ private async void SeekAndEmitSubSources(IImmutableSet formerlyU { await ConsumerActor.Ask(new KafkaConsumerActorMetadata.Internal.Seek(offsets), TimeSpan.FromSeconds(10)); - UpdatePendingPartitionsAndEmitSubSources(formerlyUnknown); + _updatePendingPartitionsAndEmitSubSourcesCallback(formerlyUnknown); } catch (AskTimeoutException ex) { @@ -226,18 +231,18 @@ private void HandleSubsourceCancelled((TopicPartition, Option) obj) + private void HandleSubsourceStarted((TopicPartition, IControl) obj) { - var (topicPartition, taskCompletionSource) = obj; + var (topicPartition, control) = obj; if (!_partitionsInStartup.Contains(topicPartition)) { // Partition was revoked while starting up. Kill! - taskCompletionSource.SetResult(Done.Instance); + control.Shutdown(); } else { - _subSources.SetItem(topicPartition, taskCompletionSource); + _subSources.SetItem(topicPartition, control); _partitionsInStartup.Remove(topicPartition); } } @@ -267,13 +272,16 @@ private void EmitSubSourcesForPendingPartitions() EmitSubSourcesForPendingPartitions(); } } - - /// - /// Makes this logic task finished - /// - protected void OnShutdown() + + private void PerformStop() { - _completion.TrySetResult(NotUsed.Instance); + SetKeepGoing(true); + + _subSources.Values.ForEach(control => control.Stop()); + + Complete(_shape.Outlet); + + InternalControl.OnStop(); } private void PerformShutdown() @@ -281,7 +289,7 @@ private void PerformShutdown() SetKeepGoing(true); // TODO from alpakka: we should wait for subsources to be shutdown and next shutdown main stage - _subSources.Values.ForEach(task => task.SetResult(Done.Instance)); + _subSources.Values.ForEach(control => control.Shutdown()); if (!IsClosed(_shape.Outlet)) Complete(_shape.Outlet); @@ -291,7 +299,7 @@ private void PerformShutdown() var (actor, message) = args; if (message is Terminated terminated && terminated.ActorRef.Equals(ConsumerActor)) { - OnShutdown(); + InternalControl.OnShutdown(); CompleteStage(); } }); @@ -299,11 +307,36 @@ private void PerformShutdown() Materializer.ScheduleOnce(_settings.StopTimeout, () => ConsumerActor.Tell(new KafkaConsumerActorMetadata.Internal.Stop())); } + /// + /// Overrides some method of base + /// + protected class SubSourcePromiseControl : PromiseControl<(TopicPartition, Source)> + { + private readonly Action _performStop; + private readonly Action _performShutdown; + + public SubSourcePromiseControl(SourceShape<(TopicPartition, Source)> shape, + Action)>> completeStageOutlet, + Action setStageKeepGoing, Func asyncCallbackFactory, + Action performStop, Action performShutdown) + : base(shape, completeStageOutlet, setStageKeepGoing, asyncCallbackFactory) + { + _performStop = performStop; + _performShutdown = performShutdown; + } + + /// + public override void PerformStop() => _performStop(); + + /// + public override void PerformShutdown() => _performShutdown(); + } + private class SubSourceStreamStage : GraphStage> { private readonly TopicPartition _topicPartition; private readonly IActorRef _consumerActor; - private readonly Action<(TopicPartition, TaskCompletionSource)> _subSourceStartedCallback; + private readonly Action<(TopicPartition, IControl)> _subSourceStartedCallback; private readonly Action<(TopicPartition, Option>)> _subSourceCancelledCallback; private readonly IMessageBuilder _messageBuilder; private readonly int _actorNumber; @@ -312,7 +345,7 @@ private class SubSourceStreamStage : GraphStage> public override SourceShape Shape { get; } public SubSourceStreamStage(TopicPartition topicPartition, IActorRef consumerActor, - Action<(TopicPartition, TaskCompletionSource)> subSourceStartedCallback, + Action<(TopicPartition, IControl)> subSourceStartedCallback, Action<(TopicPartition, Option>)> subSourceCancelledCallback, IMessageBuilder messageBuilder, int actorNumber) @@ -341,15 +374,18 @@ private class SubSourceStreamStageLogic : GraphStageLogic private readonly IActorRef _consumerActor; private readonly int _actorNumber; private readonly IMessageBuilder _messageBuilder; - private readonly Action<(TopicPartition, TaskCompletionSource)> _subSourceStartedCallback; + private readonly Action<(TopicPartition, IControl)> _subSourceStartedCallback; private KafkaConsumerActorMetadata.Internal.RequestMessages _requestMessages; private bool _requested = false; private StageActor _subSourceActor; private Queue> _buffer = new Queue>(); + + private readonly SubSourceStreamPromiseControl _internalControl; + public IControl Control => _internalControl; public SubSourceStreamStageLogic(SourceShape shape, TopicPartition topicPartition, IActorRef consumerActor, int actorNumber, IMessageBuilder messageBuilder, - Action<(TopicPartition, TaskCompletionSource)> subSourceStartedCallback, + Action<(TopicPartition, IControl)> subSourceStartedCallback, Action<(TopicPartition, Option>)> subSourceCancelledCallback) : base(shape) { @@ -361,6 +397,8 @@ public SubSourceStreamStageLogic(SourceShape shape, TopicPartition topicPa _subSourceStartedCallback = subSourceStartedCallback; _requestMessages = new KafkaConsumerActorMetadata.Internal.RequestMessages(0, ImmutableHashSet.Create(topicPartition)); + _internalControl = new SubSourceStreamPromiseControl(shape, Complete, SetKeepGoing, GetAsyncCallback, Log.Debug, actorNumber, topicPartition, CompleteStage); + SetHandler(shape.Outlet, onPull: Pump, onDownstreamFinish: () => { var firstUnconsumed = _buffer.Count > 0 ? new Option>(_buffer.Dequeue()) : Option>.None; @@ -374,7 +412,7 @@ public override void PreStart() base.PreStart(); - _subSourceStartedCallback((_topicPartition, new TaskCompletionSource())); + _subSourceStartedCallback((_topicPartition, Control)); _subSourceActor = GetStageActor(args => { var (actor, message) = args; @@ -403,7 +441,7 @@ public override void PreStart() public override void PostStop() { - CompleteStage(); + _internalControl.OnShutdown(); base.PostStop(); } @@ -425,6 +463,33 @@ private void Pump() } } } + + private class SubSourceStreamPromiseControl : PromiseControl + { + private readonly ILoggingAdapter _log; + private readonly Action _debugLog; + private readonly int _actorNumber; + private readonly TopicPartition _topicPartition; + private readonly Action _completeStage; + + public SubSourceStreamPromiseControl(SourceShape shape, Action> completeStageOutlet, + Action setStageKeepGoing, Func asyncCallbackFactory, + Action debugLog, int actorNumber, + TopicPartition topicPartition, Action completeStage) + : base(shape, completeStageOutlet, setStageKeepGoing, asyncCallbackFactory) + { + _debugLog = debugLog; + _actorNumber = actorNumber; + _topicPartition = topicPartition; + _completeStage = completeStage; + } + + public override void PerformShutdown() + { + _debugLog("#{0} Completing SubSource for partition {1}", new object[] { _actorNumber, _topicPartition }); + _completeStage(); + } + } } } } diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs index 0c844918..dbb5ca3a 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs @@ -4,6 +4,7 @@ using Akka.Event; using Akka.Streams.Dsl; using Akka.Streams.Kafka.Dsl; +using Akka.Streams.Kafka.Helpers; using Akka.Streams.Kafka.Settings; using Akka.Streams.Kafka.Stages.Consumers.Abstract; using Akka.Streams.Stage; @@ -50,11 +51,12 @@ public PlainSubSourceStage(ConsumerSettings settings, IAutoSubscription su OnRevoke = onRevoke; } - protected override GraphStageLogic Logic(SourceShape<(TopicPartition, Source, NotUsed>)> shape, - TaskCompletionSource completion, Attributes inheritedAttributes) + protected override (GraphStageLogic, IControl) Logic(SourceShape<(TopicPartition, Source, NotUsed>)> shape, + Attributes inheritedAttributes) { - return new SubSourceLogic>(shape, Settings, Subscription, _ => new PlainMessageBuilder(), - GetOffsetsOnAssign, OnRevoke, completion); + var logic = new SubSourceLogic>(shape, Settings, Subscription, _ => new PlainMessageBuilder(), + GetOffsetsOnAssign, OnRevoke); + return (logic, logic.Control); } } } \ No newline at end of file From 9c9e026a6258d15de45520cdd1d15987ff41dbc5 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Sat, 5 Oct 2019 17:41:21 +0300 Subject: [PATCH 15/28] Applied minor fixes and refactoring --- src/Akka.Streams.Kafka/Helpers/PromiseControl.cs | 2 +- .../Consumers/Abstract/BaseSingleSourceLogic.cs | 11 +++-------- .../Consumers/Abstract/SingleSourceStageLogic.cs | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs b/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs index d672c6f7..a478f2f1 100644 --- a/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs +++ b/src/Akka.Streams.Kafka/Helpers/PromiseControl.cs @@ -9,7 +9,7 @@ namespace Akka.Streams.Kafka.Helpers /// Used in source logic classes to provide implementation. /// /// - class PromiseControl : IControl + internal class PromiseControl : IControl { private readonly SourceShape _shape; private readonly Action> _completeStageOutlet; diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs index c79b2ce5..c46de7ec 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/BaseSingleSourceLogic.cs @@ -36,18 +36,13 @@ internal abstract class BaseSingleSourceLogic : GraphStageLogic private readonly ConcurrentQueue> _buffer = new ConcurrentQueue>(); protected IImmutableSet TopicPartitions { get; set; } = ImmutableHashSet.Create(); - /// - /// Implements for logic control - /// - protected readonly PromiseControl InternalControl; - protected StageActor SourceActor { get; private set; } internal IActorRef ConsumerActor { get; private set; } /// /// Implements to provide control over executed source /// - public virtual IControl Control => InternalControl; + public virtual PromiseControl Control { get; } protected BaseSingleSourceLogic(SourceShape shape, Attributes attributes, Func, IMessageBuilder> messageBuilderFactory) @@ -55,7 +50,7 @@ protected BaseSingleSourceLogic(SourceShape shape, Attributes attribut { _shape = shape; _messageBuilder = messageBuilderFactory(this); - InternalControl = new BaseSingleSourceControl(_shape, Complete, SetKeepGoing, GetAsyncCallback, PerformShutdown); + Control = new BaseSingleSourceControl(_shape, Complete, SetKeepGoing, GetAsyncCallback, PerformShutdown); var supervisionStrategy = attributes.GetAttribute(null); _decider = supervisionStrategy != null ? supervisionStrategy.Decider : Deciders.ResumingDecider; @@ -76,7 +71,7 @@ public override void PreStart() public override void PostStop() { - InternalControl.OnShutdown(); + Control.OnShutdown(); base.PostStop(); } diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SingleSourceStageLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SingleSourceStageLogic.cs index 3d610114..4a55e8fe 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SingleSourceStageLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SingleSourceStageLogic.cs @@ -104,7 +104,7 @@ private void ShuttingDownReceive(Tuple args) switch (args.Item2) { case Terminated terminated when terminated.ActorRef.Equals(ConsumerActor): - InternalControl.OnShutdown(); + Control.OnShutdown(); CompleteStage(); break; default: From 5856178e17441a9784eb4a3ebf771e36e038b19f Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Sat, 5 Oct 2019 17:44:42 +0300 Subject: [PATCH 16/28] Minor refactoring --- .../Consumers/Abstract/SubSourceLogic.cs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs index 8608c7d2..19f579af 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs @@ -61,9 +61,7 @@ private class CloseRevokedPartitions { } protected StageActor SourceActor { get; private set; } protected IActorRef ConsumerActor { get; private set; } - protected SubSourcePromiseControl InternalControl { get; } - - public IControl Control => InternalControl; + public PromiseControl<(TopicPartition, Source)> Control { get; } /// /// SubSourceLogic @@ -81,7 +79,7 @@ public SubSourceLogic(SourceShape<(TopicPartition, Source)> s _getOffsetsOnAssign = getOffsetsOnAssign; _onRevoke = onRevoke; - InternalControl = new SubSourcePromiseControl(_shape, Complete, SetKeepGoing, GetAsyncCallback, PerformStop, PerformShutdown); + Control = new SubSourcePromiseControl(_shape, Complete, SetKeepGoing, GetAsyncCallback, PerformStop, PerformShutdown); _updatePendingPartitionsAndEmitSubSourcesCallback = GetAsyncCallback>(UpdatePendingPartitionsAndEmitSubSources); _partitionAssignedCallback = GetAsyncCallback>(HandlePartitionsAssigned); @@ -138,7 +136,7 @@ public override void PostStop() { ConsumerActor.Tell(new KafkaConsumerActorMetadata.Internal.Stop(), SourceActor.Ref); - InternalControl.OnShutdown(); + Control.OnShutdown(); base.PostStop(); } @@ -281,7 +279,7 @@ private void PerformStop() Complete(_shape.Outlet); - InternalControl.OnStop(); + Control.OnStop(); } private void PerformShutdown() @@ -299,7 +297,7 @@ private void PerformShutdown() var (actor, message) = args; if (message is Terminated terminated && terminated.ActorRef.Equals(ConsumerActor)) { - InternalControl.OnShutdown(); + Control.OnShutdown(); CompleteStage(); } }); @@ -380,8 +378,7 @@ private class SubSourceStreamStageLogic : GraphStageLogic private StageActor _subSourceActor; private Queue> _buffer = new Queue>(); - private readonly SubSourceStreamPromiseControl _internalControl; - public IControl Control => _internalControl; + public PromiseControl Control { get; } public SubSourceStreamStageLogic(SourceShape shape, TopicPartition topicPartition, IActorRef consumerActor, int actorNumber, IMessageBuilder messageBuilder, @@ -397,7 +394,7 @@ public SubSourceStreamStageLogic(SourceShape shape, TopicPartition topicPa _subSourceStartedCallback = subSourceStartedCallback; _requestMessages = new KafkaConsumerActorMetadata.Internal.RequestMessages(0, ImmutableHashSet.Create(topicPartition)); - _internalControl = new SubSourceStreamPromiseControl(shape, Complete, SetKeepGoing, GetAsyncCallback, Log.Debug, actorNumber, topicPartition, CompleteStage); + Control = new SubSourceStreamPromiseControl(shape, Complete, SetKeepGoing, GetAsyncCallback, Log.Debug, actorNumber, topicPartition, CompleteStage); SetHandler(shape.Outlet, onPull: Pump, onDownstreamFinish: () => { @@ -441,7 +438,7 @@ public override void PreStart() public override void PostStop() { - _internalControl.OnShutdown(); + Control.OnShutdown(); base.PostStop(); } From bcb4c113c1a4980a232f4fc1c41691c563fb7033 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Mon, 7 Oct 2019 17:41:11 +0300 Subject: [PATCH 17/28] Fixed immutable collections usage --- .../PlainPartitionedSourceIntegrationTests.cs | 2 +- src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs | 2 +- .../Stages/Consumers/Abstract/SubSourceLogic.cs | 11 ++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs index 0135d32b..521ca452 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs @@ -54,7 +54,7 @@ public async Task PlainPartitionedSource_should_work() await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); // Give it some time to consume all messages - await Task.Delay(5000); + await Task.Delay(10000); var shutdown = control.DrainAndShutdown(); AwaitCondition(() => shutdown.IsCompleted, TimeSpan.FromSeconds(10)); diff --git a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs index 9b3fc0cc..35e94a4c 100644 --- a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs +++ b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs @@ -108,7 +108,7 @@ public static SourceWithContext, IContro public static Source, IControl> CommittableExternalSource(IActorRef consumer, IManualSubscription subscription, string groupId, TimeSpan commitTimeout) { - return Source.FromGraph, IControl>(new ExternalCommittableSourceStage(consumer, groupId, commitTimeout, subscription)); + return Source.FromGraph(new ExternalCommittableSourceStage(consumer, groupId, commitTimeout, subscription)); } /// diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs index 19f579af..6a8d190f 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs @@ -240,8 +240,8 @@ private void HandleSubsourceStarted((TopicPartition, IControl) obj) } else { - _subSources.SetItem(topicPartition, control); - _partitionsInStartup.Remove(topicPartition); + _subSources = _subSources.SetItem(topicPartition, control); + _partitionsInStartup = _partitionsInStartup.Remove(topicPartition); } } @@ -373,10 +373,10 @@ private class SubSourceStreamStageLogic : GraphStageLogic private readonly int _actorNumber; private readonly IMessageBuilder _messageBuilder; private readonly Action<(TopicPartition, IControl)> _subSourceStartedCallback; - private KafkaConsumerActorMetadata.Internal.RequestMessages _requestMessages; + private readonly KafkaConsumerActorMetadata.Internal.RequestMessages _requestMessages; private bool _requested = false; private StageActor _subSourceActor; - private Queue> _buffer = new Queue>(); + private readonly Queue> _buffer = new Queue>(); public PromiseControl Control { get; } @@ -394,7 +394,8 @@ public SubSourceStreamStageLogic(SourceShape shape, TopicPartition topicPa _subSourceStartedCallback = subSourceStartedCallback; _requestMessages = new KafkaConsumerActorMetadata.Internal.RequestMessages(0, ImmutableHashSet.Create(topicPartition)); - Control = new SubSourceStreamPromiseControl(shape, Complete, SetKeepGoing, GetAsyncCallback, Log.Debug, actorNumber, topicPartition, CompleteStage); + Control = new SubSourceStreamPromiseControl(shape, Complete, SetKeepGoing, GetAsyncCallback, (message, args) => Log.Debug(message, args), + actorNumber, topicPartition, CompleteStage); SetHandler(shape.Outlet, onPull: Pump, onDownstreamFinish: () => { From 71f0776fbb7d25d12826c306a7d267fafe5f1810 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Mon, 7 Oct 2019 22:38:35 +0300 Subject: [PATCH 18/28] Fixed test and added one more --- .../PlainPartitionedSourceIntegrationTests.cs | 60 ++++++++++++++++++- .../KafkaIntegrationTests.cs | 2 +- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs index 521ca452..05b744a5 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs @@ -26,7 +26,7 @@ public async Task PlainPartitionedSource_should_work() { var topic = CreateTopic(1); var group = CreateGroup(1); - var totalMessages = 1000 * 10; + var totalMessages = 100; var consumerSettings = CreateConsumerSettings(group); @@ -54,13 +54,69 @@ public async Task PlainPartitionedSource_should_work() await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); // Give it some time to consume all messages - await Task.Delay(10000); + await Task.Delay(5000); var shutdown = control.DrainAndShutdown(); AwaitCondition(() => shutdown.IsCompleted, TimeSpan.FromSeconds(10)); shutdown.Result.Should().Be(totalMessages); } + [Fact] + public async Task PlainPartitionedSource_Should_split_messages_by_partitions() + { + var topic = CreateTopic(1); + var group = CreateGroup(1); + var totalMessages = 100; + + var consumerSettings = CreateConsumerSettings(group); + + var control = KafkaConsumer.PlainPartitionedSource(consumerSettings, Subscriptions.Topics(topic)) + .SelectAsync(6, async tuple => + { + var (topicPartition, source) = tuple; + Log.Info($"Sub-source for {topicPartition}"); + var consumedPartitions = await source + .Select(m => m.TopicPartition.Partition) + .RunWith(Sink.Seq(), Materializer); + + // Return flag that all messages in child source are from the same, expected partition + return consumedPartitions.All(partition => partition == topicPartition.Partition); + }) + .As>() + .ToMaterialized(Sink.Aggregate(true, (result, childSourceIsValid) => result && childSourceIsValid), Keep.Both) + .MapMaterializedValue(tuple => DrainingControl.Create(tuple.Item1, tuple.Item2)) + .Run(Materializer); + + await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); + + // Give it some time to consume all messages + await Task.Delay(5000); + + var shutdown = control.DrainAndShutdown(); + AwaitCondition(() => shutdown.IsCompleted, TimeSpan.FromSeconds(10)); + shutdown.Result.Should().BeTrue(); + } + + /* Needs to be finished */ + /* + [Fact] + public async Task PlainPartitionedSource_should_stop_partition_sources_when_stopped() + { + var topic = CreateTopic(1); + var group = CreateGroup(1); + var totalMessages = 100; + + await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); + + var consumerSettings = CreateConsumerSettings(group).WithStopTimeout(TimeSpan.FromMilliseconds(10)); + var (control, probe) = KafkaConsumer.PlainPartitionedSource(consumerSettings, Subscriptions.Topics(topic)) + .MergeMany<(TopicPartition, Source, NotUsed>), ConsumeResult, IControl>(1, tuple => tuple.Item2) + .Select(message => message.Value) + .ToMaterialized(this.SinkProbe(), Keep.Both) + .Run(Materializer); + } + */ + private int LogSentMessages(int counter) { if (counter % 1000 == 0) diff --git a/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs index 95a3ac68..b464e232 100644 --- a/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs @@ -62,7 +62,7 @@ protected async Task ProduceStrings(string topic, IEnumerable range, Produc { await Source .From(range) - .Select(elem => new MessageAndMeta { TopicPartition = new TopicPartition(topic, 0), Message = new Message { Value = elem.ToString() } }) + .Select(elem => new MessageAndMeta { Topic = topic, Message = new Message { Value = elem.ToString() } }) .RunWith(KafkaProducer.PlainSink(producerSettings), Materializer); } From 389a26c08b6767ca0f166ea16711bc27d0460521 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Tue, 8 Oct 2019 14:29:53 +0300 Subject: [PATCH 19/28] Added more tests --- .../PlainPartitionedSourceIntegrationTests.cs | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs index 05b744a5..e850f615 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs @@ -97,8 +97,6 @@ public async Task PlainPartitionedSource_Should_split_messages_by_partitions() shutdown.Result.Should().BeTrue(); } - /* Needs to be finished */ - /* [Fact] public async Task PlainPartitionedSource_should_stop_partition_sources_when_stopped() { @@ -110,19 +108,36 @@ public async Task PlainPartitionedSource_should_stop_partition_sources_when_stop var consumerSettings = CreateConsumerSettings(group).WithStopTimeout(TimeSpan.FromMilliseconds(10)); var (control, probe) = KafkaConsumer.PlainPartitionedSource(consumerSettings, Subscriptions.Topics(topic)) - .MergeMany<(TopicPartition, Source, NotUsed>), ConsumeResult, IControl>(1, tuple => tuple.Item2) - .Select(message => message.Value) + .MergeMany(3, tuple => tuple.Item2.MapMaterializedValue(notUsed => new NoopControl())) + .Select(message => + { + Log.Debug($"Consumed partition {message.Partition.Value}"); + return message.Value; + }) .ToMaterialized(this.SinkProbe(), Keep.Both) .Run(Materializer); + + probe.Request(totalMessages).Within(TimeSpan.FromSeconds(10), () => probe.ExpectNextN(totalMessages)); + + var stopped = control.Stop(); + probe.ExpectComplete(); + + AwaitCondition(() => stopped.IsCompleted, TimeSpan.FromSeconds(10)); + + await control.Shutdown(); + probe.Cancel(); } - */ - private int LogSentMessages(int counter) + [Fact] + public async Task PlainPartitionedSource_should_be_signalled_the_stream_by_partitioned_sources() { - if (counter % 1000 == 0) - Log.Info($"Sent {counter} messages so far"); - - return counter; + var settings = CreateConsumerSettings(CreateGroup(1)) + .WithBootstrapServers("localhost:1111"); // Bad address + + var result = KafkaConsumer.PlainPartitionedSource(settings, Subscriptions.Topics("topic")) + .RunWith(Sink.First<(TopicPartition, Source, NotUsed>)>(), Materializer); + + result.Invoking(r => r.Wait()).Should().Throw(); } private long LogReceivedMessages(TopicPartition tp, int counter) From d6f83706c647f6231a2136ccf7134c0259bc34bb Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Tue, 8 Oct 2019 15:05:58 +0300 Subject: [PATCH 20/28] Added serialization failure test --- .../PlainPartitionedSourceIntegrationTests.cs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs index e850f615..e1aa84d7 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs @@ -1,12 +1,15 @@ using System; using System.Linq; +using System.Runtime.Serialization; using System.Threading.Tasks; using Akka.Streams.Dsl; using Akka.Streams.Kafka.Dsl; using Akka.Streams.Kafka.Helpers; using Akka.Streams.Kafka.Messages; using Akka.Streams.Kafka.Settings; +using Akka.Streams.Supervision; using Akka.Streams.TestKit; +using Akka.Util.Internal; using Confluent.Kafka; using FluentAssertions; using Xunit; @@ -139,6 +142,34 @@ public async Task PlainPartitionedSource_should_be_signalled_the_stream_by_parti result.Invoking(r => r.Wait()).Should().Throw(); } + + [Fact] + public async Task PlainPartitionedSource_should_be_signalled_about_serialization_errors() + { + var topic = CreateTopic(1); + var group = CreateGroup(1); + + var settings = CreateConsumerSettings(group).WithValueDeserializer(Deserializers.Int32); + + var (control1, partitionedProbe) = KafkaConsumer.PlainPartitionedSource(settings, Subscriptions.Topics(topic)) + .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.StoppingDecider)) + .ToMaterialized(this.SinkProbe<(TopicPartition, Source, NotUsed>)>(), Keep.Both) + .Run(Materializer); + + partitionedProbe.Request(3); + + var subsources = partitionedProbe.Within(TimeSpan.FromSeconds(10), () => partitionedProbe.ExpectNextN(3).Select(t => t.Item2).ToList()); + var substream = subsources.Aggregate((s1, s2) => s1.Merge(s2)).RunWith(this.SinkProbe>(), Materializer); + + substream.Request(1); + + await ProduceStrings(topic, new int[] { 0 }, ProducerSettings); // Produce "0" string + + Within(TimeSpan.FromSeconds(10), () => substream.ExpectError().Should().BeOfType()); + + var shutdown = control1.Shutdown(); + AwaitCondition(() => shutdown.IsCompleted, TimeSpan.FromSeconds(10)); + } private long LogReceivedMessages(TopicPartition tp, int counter) { From c0c16e9120861fd08dde687fe30432bf46b94c55 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Tue, 8 Oct 2019 16:44:50 +0300 Subject: [PATCH 21/28] Added failover tests --- .../PlainPartitionedSourceIntegrationTests.cs | 87 +++++++++++++++++++ .../Consumers/Abstract/SubSourceLogic.cs | 62 ++++++++++--- .../Consumers/Concrete/PlainSubSourceStage.cs | 2 +- 3 files changed, 138 insertions(+), 13 deletions(-) diff --git a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs index e1aa84d7..ee28e9f8 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/PlainPartitionedSourceIntegrationTests.cs @@ -170,6 +170,93 @@ public async Task PlainPartitionedSource_should_be_signalled_about_serialization var shutdown = control1.Shutdown(); AwaitCondition(() => shutdown.IsCompleted, TimeSpan.FromSeconds(10)); } + + [Fact] + public async Task PlainPartitionedSource_should_not_leave_gaps_when_subsource_is_cancelled() + { + var topic = CreateTopic(1); + var group = CreateGroup(1); + var totalMessages = 100; + + await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); + + var consumedMessagesTask = KafkaConsumer.PlainPartitionedSource(CreateConsumerSettings(group), Subscriptions.Topics(topic)) + .Log(topic, m => $"Consuming topic partition {m.Item1}") + .MergeMany(3, tuple => + { + var (topicPartition, source) = tuple; + return source + .MapMaterializedValue(notUsed => new NoopControl()) + .Log(topicPartition.ToString(), m => $"Consumed offset {m.Offset} (value: {m.Value})") + .Take(10); + }) + .Select(m => int.Parse(m.Value)) + .Log("Merged stream", m => m) + .Scan(0, (c, _) => c + 1) + .TakeWhile(m => m < totalMessages, inclusive: true) + .RunWith(Sink.Last(), Materializer); + + AwaitCondition(() => consumedMessagesTask.IsCompleted, TimeSpan.FromSeconds(10)); + + consumedMessagesTask.Result.Should().Be(totalMessages); + } + + [Fact] + public async Task PlainPartitionedSource_should_not_leave_gaps_when_subsource_failes() + { + var topic = CreateTopic(1); + var group = CreateGroup(1); + var totalMessages = 105; + + await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); + + var (queue, accumulatorTask) = Source.Queue(8, OverflowStrategy.Backpressure) + .Scan(0, (c, _) => c + 1) + .TakeWhile(val => val < totalMessages) + .ToMaterialized(Sink.Aggregate(0, (c, _) => c + 1), Keep.Both) + .Run(Materializer); + + var (killSwitch, consumerCompletion) = KafkaConsumer.PlainPartitionedSource(CreateConsumerSettings(group), Subscriptions.Topics(topic)) + .Log(topic, m => $"Consuming topic partition {m.Item1}") + .ViaMaterialized(KillSwitches.Single<(TopicPartition, Source, NotUsed>)>(), Keep.Both) + .ToMaterialized(Sink.ForEach<(TopicPartition, Source, NotUsed>)>(tuple => + { + var (topicPartition, source) = tuple; + source + .Log(topicPartition.ToString(), m => $"Consumed offset {m.Offset} (value: {m.Value})") + .SelectAsync(1, async message => + { + await queue.OfferAsync(message.Offset); + var value = int.Parse(message.Value); + + if (value % 10 == 0) + { + Log.Debug("Reached message to fail: {0}", value); + throw new Exception("Stopping subsource"); + } + + return value; + }) + .Select(value => + { + if (value % 10 == 0) + { + Log.Debug("Reached message to fail: {0}", value); + throw new Exception("Stopping subsource"); + } + + return value; + }) + .RunWith(Sink.Ignore(), Materializer); + }), Keep.Both) + .Run(Materializer); + + AwaitCondition(() => accumulatorTask.IsCompleted, TimeSpan.FromSeconds(10)); + accumulatorTask.Result.Should().Be(totalMessages); + + killSwitch.Item2.Shutdown(); + AwaitCondition(() => consumerCompletion.IsCompleted, TimeSpan.FromSeconds(10)); + } private long LogReceivedMessages(TopicPartition tp, int counter) { diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs index 6a8d190f..ae0a0d2c 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs @@ -1,8 +1,8 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; -using System.Runtime.InteropServices; using System.Threading.Tasks; using Akka.Actor; using Akka.Event; @@ -13,9 +13,12 @@ using Akka.Streams.Kafka.Stages.Consumers.Actors; using Akka.Streams.Kafka.Stages.Consumers.Exceptions; using Akka.Streams.Stage; +using Akka.Streams.Supervision; using Akka.Streams.Util; using Akka.Util.Internal; using Confluent.Kafka; +using Decider = Akka.Streams.Supervision.Decider; +using Directive = Akka.Streams.Supervision.Directive; namespace Akka.Streams.Kafka.Stages.Consumers.Abstract { @@ -40,6 +43,7 @@ private class CloseRevokedPartitions { } private readonly Action<(TopicPartition, Option>)> _subsourceCancelledCallback; private readonly Action<(TopicPartition, IControl)> _subsourceStartedCallback; private readonly Action _stageFailCallback; + private readonly Decider _decider; /// /// Kafka has notified us that we have these partitions assigned, but we have not created a source for them yet. @@ -69,7 +73,7 @@ private class CloseRevokedPartitions { } public SubSourceLogic(SourceShape<(TopicPartition, Source)> shape, ConsumerSettings settings, IAutoSubscription subscription, Func, IMessageBuilder> messageBuilderFactory, Option, Task>>> getOffsetsOnAssign, - Action> onRevoke) + Action> onRevoke, Attributes attributes) : base(shape) { _shape = shape; @@ -79,6 +83,9 @@ public SubSourceLogic(SourceShape<(TopicPartition, Source)> s _getOffsetsOnAssign = getOffsetsOnAssign; _onRevoke = onRevoke; + var supervisionStrategy = attributes.GetAttribute(null); + _decider = supervisionStrategy != null ? supervisionStrategy.Decider : Deciders.StoppingDecider; + Control = new SubSourcePromiseControl(_shape, Complete, SetKeepGoing, GetAsyncCallback, PerformStop, PerformShutdown); _updatePendingPartitionsAndEmitSubSourcesCallback = GetAsyncCallback>(UpdatePendingPartitionsAndEmitSubSources); @@ -100,7 +107,20 @@ public override void PreStart() switch (args.Item2) { case Status.Failure failure: - FailStage(failure.Cause); + var exception = failure.Cause; + switch (_decider(failure.Cause)) + { + case Directive.Stop: + // Throw + FailStage(exception); + break; + case Directive.Resume: + // keep going + break; + case Directive.Restart: + // TODO: Need to do something here: https://github.com/akkadotnet/Akka.Streams.Kafka/issues/33 + break; + } break; case Terminated terminated when terminated.ActorRef.Equals(ConsumerActor): @@ -258,11 +278,11 @@ private void EmitSubSourcesForPendingPartitions() { var topicPartition = _pendingPartitions.First(); - _pendingPartitions = _pendingPartitions.Skip(1).ToImmutableHashSet(); + _pendingPartitions = _pendingPartitions.Remove(topicPartition); _partitionsInStartup = _partitionsInStartup.Add(topicPartition); var subSourceStage = new SubSourceStreamStage(topicPartition, ConsumerActor, _subsourceStartedCallback, - _subsourceCancelledCallback, _messageBuilder, _actorNumber); + _subsourceCancelledCallback, _messageBuilder, _decider, _actorNumber); var subsource = Source.FromGraph(subSourceStage); Push(_shape.Outlet, (topicPartition, subsource)); @@ -338,6 +358,7 @@ private class SubSourceStreamStage : GraphStage> private readonly Action<(TopicPartition, Option>)> _subSourceCancelledCallback; private readonly IMessageBuilder _messageBuilder; private readonly int _actorNumber; + private readonly Decider _decider; public Outlet Out { get; } public override SourceShape Shape { get; } @@ -346,6 +367,7 @@ public SubSourceStreamStage(TopicPartition topicPartition, IActorRef consumerAct Action<(TopicPartition, IControl)> subSourceStartedCallback, Action<(TopicPartition, Option>)> subSourceCancelledCallback, IMessageBuilder messageBuilder, + Decider decider, int actorNumber) { _topicPartition = topicPartition; @@ -353,6 +375,7 @@ public SubSourceStreamStage(TopicPartition topicPartition, IActorRef consumerAct _subSourceStartedCallback = subSourceStartedCallback; _subSourceCancelledCallback = subSourceCancelledCallback; _messageBuilder = messageBuilder; + _decider = decider; _actorNumber = actorNumber; Out = new Outlet("out"); @@ -361,7 +384,7 @@ public SubSourceStreamStage(TopicPartition topicPartition, IActorRef consumerAct protected override GraphStageLogic CreateLogic(Attributes inheritedAttributes) { - return new SubSourceStreamStageLogic(Shape, _topicPartition, _consumerActor, _actorNumber, _messageBuilder, + return new SubSourceStreamStageLogic(Shape, _topicPartition, _consumerActor, _actorNumber, _messageBuilder, _decider, _subSourceStartedCallback, _subSourceCancelledCallback); } @@ -376,12 +399,13 @@ private class SubSourceStreamStageLogic : GraphStageLogic private readonly KafkaConsumerActorMetadata.Internal.RequestMessages _requestMessages; private bool _requested = false; private StageActor _subSourceActor; - private readonly Queue> _buffer = new Queue>(); + private readonly Decider _decider; + private readonly ConcurrentQueue> _buffer = new ConcurrentQueue>(); public PromiseControl Control { get; } public SubSourceStreamStageLogic(SourceShape shape, TopicPartition topicPartition, IActorRef consumerActor, - int actorNumber, IMessageBuilder messageBuilder, + int actorNumber, IMessageBuilder messageBuilder, Decider decider, Action<(TopicPartition, IControl)> subSourceStartedCallback, Action<(TopicPartition, Option>)> subSourceCancelledCallback) : base(shape) @@ -391,6 +415,7 @@ public SubSourceStreamStageLogic(SourceShape shape, TopicPartition topicPa _consumerActor = consumerActor; _actorNumber = actorNumber; _messageBuilder = messageBuilder; + _decider = decider; _subSourceStartedCallback = subSourceStartedCallback; _requestMessages = new KafkaConsumerActorMetadata.Internal.RequestMessages(0, ImmutableHashSet.Create(topicPartition)); @@ -399,8 +424,9 @@ public SubSourceStreamStageLogic(SourceShape shape, TopicPartition topicPa SetHandler(shape.Outlet, onPull: Pump, onDownstreamFinish: () => { - var firstUnconsumed = _buffer.Count > 0 ? new Option>(_buffer.Dequeue()) : Option>.None; + var firstUnconsumed = _buffer.TryDequeue(out var message) ? new Option>(message) : Option>.None; subSourceCancelledCallback((topicPartition, firstUnconsumed)); + CompleteStage(); }); } @@ -426,7 +452,20 @@ public override void PreStart() Pump(); break; case Status.Failure failure: - FailStage(failure.Cause); + var exception = failure.Cause; + switch (_decider(failure.Cause)) + { + case Directive.Stop: + // Throw + FailStage(exception); + break; + case Directive.Resume: + // keep going + break; + case Directive.Restart: + // TODO: Need to do something here: https://github.com/akkadotnet/Akka.Streams.Kafka/issues/33 + break; + } break; case Terminated terminated when terminated.ActorRef.Equals(_consumerActor): FailStage(new ConsumerFailed()); @@ -448,9 +487,8 @@ private void Pump() { if (IsAvailable(_shape.Outlet)) { - if (_buffer.Count > 0) + if (_buffer.TryDequeue(out var message)) { - var message = _buffer.Dequeue(); Push(_shape.Outlet, _messageBuilder.CreateMessage(message)); Pump(); } diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs index dbb5ca3a..f4e31ddc 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs @@ -55,7 +55,7 @@ protected override (GraphStageLogic, IControl) Logic(SourceShape<(TopicPartition Attributes inheritedAttributes) { var logic = new SubSourceLogic>(shape, Settings, Subscription, _ => new PlainMessageBuilder(), - GetOffsetsOnAssign, OnRevoke); + GetOffsetsOnAssign, OnRevoke, inheritedAttributes); return (logic, logic.Control); } } From 75aaa2fba25934a550658eb99ee985b667460d7d Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Tue, 8 Oct 2019 19:04:43 +0300 Subject: [PATCH 22/28] Skipping AtMostOnce test --- .../Integration/AtMostOnceSourceIntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs index 001ca6d8..160e06cf 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs @@ -18,7 +18,7 @@ public AtMostOnceSourceIntegrationTests(ITestOutputHelper output, KafkaFixture f { } - [Fact] + [Fact(Skip = "Issue https://github.com/akkadotnet/Akka.Streams.Kafka/issues/74")] public async Task AtMostOnceSource_Should_stop_consuming_actor_when_used_with_Take() { var topic = CreateTopic(1); From b68e2520cfd577524bbb6e9221f65d7444efd834 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Tue, 8 Oct 2019 19:07:44 +0300 Subject: [PATCH 23/28] Fixed AtMostOnceSource test messages ordering --- .../Integration/AtMostOnceSourceIntegrationTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs index 160e06cf..273bee58 100644 --- a/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/Integration/AtMostOnceSourceIntegrationTests.cs @@ -5,6 +5,7 @@ using Akka.Streams.Kafka.Dsl; using Akka.Streams.Kafka.Settings; using Akka.Streams.TestKit; +using Confluent.Kafka; using FluentAssertions; using Xunit; using Xunit.Abstractions; @@ -18,15 +19,15 @@ public AtMostOnceSourceIntegrationTests(ITestOutputHelper output, KafkaFixture f { } - [Fact(Skip = "Issue https://github.com/akkadotnet/Akka.Streams.Kafka/issues/74")] + [Fact] public async Task AtMostOnceSource_Should_stop_consuming_actor_when_used_with_Take() { var topic = CreateTopic(1); var group = CreateGroup(1); - await ProduceStrings(topic, Enumerable.Range(1, 10), ProducerSettings); + await ProduceStrings(new TopicPartition(topic, 0), Enumerable.Range(1, 10), ProducerSettings); - var (control, result) = KafkaConsumer.AtMostOnceSource(CreateConsumerSettings(group), Subscriptions.Topics(topic)) + var (control, result) = KafkaConsumer.AtMostOnceSource(CreateConsumerSettings(group), Subscriptions.Assignment(new TopicPartition(topic, 0))) .Select(m => m.Value) .Take(5) .ToMaterialized(Sink.Seq(), Keep.Both) From f2159b84c591f26010f6396714dc75a413158164 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Thu, 10 Oct 2019 16:56:37 +0300 Subject: [PATCH 24/28] Fixed typo --- src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs index b464e232..0f4757ea 100644 --- a/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs @@ -78,7 +78,7 @@ await Source /// Asserts that task will finish successfully until specified timeout. /// Throws task exception if task failes /// - protected async Task AssertCompletesSuccessfullyWithing(TimeSpan timeout, Task task) + protected async Task AssertCompletesSuccessfullyWithin(TimeSpan timeout, Task task) { var timeoutTask = Task.Delay(timeout); From d6abcc9d0066eb7595485f5631188e0d7c825711 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Thu, 10 Oct 2019 17:00:19 +0300 Subject: [PATCH 25/28] Removed nightly feed --- NuGet.Config | 1 - 1 file changed, 1 deletion(-) diff --git a/NuGet.Config b/NuGet.Config index 5916b6f4..c1584dbb 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -6,6 +6,5 @@ - \ No newline at end of file From c0d1e5ee3842d9dc5abd7dcbbb0d532b43088536 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Fri, 11 Oct 2019 16:21:00 +0300 Subject: [PATCH 26/28] Added CommittablePartitionedSource implementation --- src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs | 11 +++- .../Consumers/Abstract/SubSourceLogic.cs | 2 +- .../Concrete/CommittableSubSourceStage.cs | 55 +++++++++++++++++++ .../Consumers/Concrete/PlainSubSourceStage.cs | 8 ++- 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 src/Akka.Streams.Kafka/Stages/Consumers/Concrete/CommittableSubSourceStage.cs diff --git a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs index 35e94a4c..b2f73a82 100644 --- a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs +++ b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs @@ -106,11 +106,20 @@ public static SourceWithContext, IContro /// The same as but for offset commit support /// public static Source, IControl> CommittableExternalSource(IActorRef consumer, IManualSubscription subscription, - string groupId, TimeSpan commitTimeout) + string groupId, TimeSpan commitTimeout) { return Source.FromGraph(new ExternalCommittableSourceStage(consumer, groupId, commitTimeout, subscription)); } + /// + /// The same as but with offset commit support. + /// + public static Source<(TopicPartition, Source, NotUsed>), IControl> CommittablePartitionedSource( + ConsumerSettings settings, IAutoSubscription subscription) + { + return Source.FromGraph(new CommittableSubSourceStage(settings, subscription)); + } + /// /// Convenience for "at-most once delivery" semantics. /// The offset of each message is committed to Kafka before being emitted downstream. diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs index ae0a0d2c..be6b9e52 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Abstract/SubSourceLogic.cs @@ -63,7 +63,7 @@ private class CloseRevokedPartitions { } protected StageActor SourceActor { get; private set; } - protected IActorRef ConsumerActor { get; private set; } + public IActorRef ConsumerActor { get; private set; } public PromiseControl<(TopicPartition, Source)> Control { get; } diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/CommittableSubSourceStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/CommittableSubSourceStage.cs new file mode 100644 index 00000000..e1d93e64 --- /dev/null +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/CommittableSubSourceStage.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Immutable; +using System.Threading.Tasks; +using Akka.Streams.Dsl; +using Akka.Streams.Kafka.Helpers; +using Akka.Streams.Kafka.Messages; +using Akka.Streams.Kafka.Settings; +using Akka.Streams.Kafka.Stages.Consumers.Abstract; +using Akka.Streams.Stage; +using Akka.Streams.Util; +using Confluent.Kafka; + +namespace Akka.Streams.Kafka.Stages.Consumers.Concrete +{ + public class CommittableSubSourceStage : KafkaSourceStage, NotUsed>)> + { + private readonly Func, string> _metadataFromRecord; + + /// + /// Consumer settings + /// + public ConsumerSettings Settings { get; } + /// + /// Subscription + /// + public IAutoSubscription Subscription { get; } + + public CommittableSubSourceStage(ConsumerSettings settings, IAutoSubscription subscription, Func, string> metadataFromRecord = null) + : base("CommittableSubSourceStage") + { + Settings = settings; + Subscription = subscription; + _metadataFromRecord = metadataFromRecord ?? (_ => string.Empty); + } + + protected override (GraphStageLogic, IControl) Logic(SourceShape<(TopicPartition, Source, NotUsed>)> shape, Attributes inheritedAttributes) + { + var logic = new SubSourceLogic>(shape, Settings, Subscription, + messageBuilderFactory: GetMessageBuilder, + getOffsetsOnAssign: Option, Task>>>.None, + onRevoke: _ => { }, + attributes: inheritedAttributes); + return (logic, logic.Control); + } + + /// + /// Creates message builder for sub-source logic + /// + private CommittableSourceMessageBuilder GetMessageBuilder(SubSourceLogic> logic) + { + var committer = new KafkaAsyncConsumerCommitter(() => logic.ConsumerActor, Settings.CommitTimeout); + return new CommittableSourceMessageBuilder(committer, Settings.GroupId, _metadataFromRecord); + } + } +} \ No newline at end of file diff --git a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs index f4e31ddc..1c90e278 100644 --- a/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs +++ b/src/Akka.Streams.Kafka/Stages/Consumers/Concrete/PlainSubSourceStage.cs @@ -51,11 +51,15 @@ public PlainSubSourceStage(ConsumerSettings settings, IAutoSubscription su OnRevoke = onRevoke; } + /// protected override (GraphStageLogic, IControl) Logic(SourceShape<(TopicPartition, Source, NotUsed>)> shape, Attributes inheritedAttributes) { - var logic = new SubSourceLogic>(shape, Settings, Subscription, _ => new PlainMessageBuilder(), - GetOffsetsOnAssign, OnRevoke, inheritedAttributes); + var logic = new SubSourceLogic>(shape, Settings, Subscription, + messageBuilderFactory: _ => new PlainMessageBuilder(), + getOffsetsOnAssign: GetOffsetsOnAssign, + onRevoke: OnRevoke, + attributes: inheritedAttributes); return (logic, logic.Control); } } From 295449cded4976b0743e8f7ef0b3a1b2a36a1754 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Fri, 11 Oct 2019 21:14:47 +0300 Subject: [PATCH 27/28] Added tests --- ...ttablePartitionedSourceIntegrationTests.cs | 106 ++++++++++++++++++ .../KafkaIntegrationTests.cs | 8 ++ 2 files changed, 114 insertions(+) create mode 100644 src/Akka.Streams.Kafka.Tests/Integration/CommittablePartitionedSourceIntegrationTests.cs diff --git a/src/Akka.Streams.Kafka.Tests/Integration/CommittablePartitionedSourceIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/Integration/CommittablePartitionedSourceIntegrationTests.cs new file mode 100644 index 00000000..a1665e24 --- /dev/null +++ b/src/Akka.Streams.Kafka.Tests/Integration/CommittablePartitionedSourceIntegrationTests.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Akka.Streams.Dsl; +using Akka.Streams.Kafka.Dsl; +using Akka.Streams.Kafka.Helpers; +using Akka.Streams.Kafka.Settings; +using Akka.Util; +using Confluent.Kafka; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +namespace Akka.Streams.Kafka.Tests.Integration +{ + public class CommittablePartitionedSourceIntegrationTests : KafkaIntegrationTests + { + public CommittablePartitionedSourceIntegrationTests(ITestOutputHelper output, KafkaFixture fixture) + : base(nameof(CommittablePartitionedSourceIntegrationTests), output, fixture) + { + } + + [Fact] + public async Task CommittablePartitionedSource_Should_handle_exceptions_in_stream_without_commit_failures() + { + var partitionsCount = 3; + var topic = CreateTopic(1); + var group = CreateGroup(1); + var totalMessages = 100; + var exceptionTriggered = new AtomicBoolean(false); + var allTopicPartitions = Enumerable.Range(0, partitionsCount).Select(i => new TopicPartition(topic, i)).ToList(); + + var consumerSettings = CreateConsumerSettings(group).WithStopTimeout(TimeSpan.FromSeconds(2)); + + var createdSubSources = new ConcurrentSet(); + var commitFailures = new ConcurrentSet<(TopicPartition, Exception)>(); + + var control = KafkaConsumer.CommittablePartitionedSource(consumerSettings, Subscriptions.Topics(topic)) + .GroupBy(partitionsCount, tuple => tuple.Item1) + .SelectAsync(6, tuple => + { + var (topicPartition, source) = tuple; + createdSubSources.TryAdd(topicPartition); + return source + .Log($"Subsource for partition #{topicPartition.Partition.Value}", m => m.Record.Value) + .SelectAsync(3, async message => + { + // fail on first partition; otherwise delay slightly and emit + if (topicPartition.Partition.Value == 0) + { + Log.Debug($"Failing {topicPartition} source"); + exceptionTriggered.GetAndSet(true); + throw new Exception("FAIL"); + } + else + { + await Task.Delay(50); + } + + return message; + }) + .Log($"Subsource {topicPartition} pre commit") + .SelectAsync(1, async message => + { + try + { + await message.CommitableOffset.Commit(); + } + catch (Exception ex) + { + Log.Error("Commit failure: " + ex); + commitFailures.TryAdd((topicPartition, ex)); + } + + return message; + }) + .Scan(0, (c, _) => c + 1) + .RunWith(Sink.Last(), Materializer) + .ContinueWith(t => + { + Log.Info($"sub-source for {topicPartition} completed: Received {t.Result} messages in total."); + return t.Result; + }); + }) + .MergeSubstreams().As>() + .Scan(0, (c, n) => c + n) + .ToMaterialized(Sink.Last(), Keep.Both) + .MapMaterializedValue(tuple => DrainingControl.Create(tuple.Item1, tuple.Item2)) + .Run(Materializer); + + await ProduceStrings(i => new TopicPartition(topic, i % partitionsCount), Enumerable.Range(1, totalMessages), ProducerSettings); + + AwaitCondition(() => exceptionTriggered.Value, TimeSpan.FromSeconds(10)); + + var shutdown = control.DrainAndShutdown(); + AwaitCondition(() => shutdown.IsCompleted); + createdSubSources.Should().Contain(allTopicPartitions); + shutdown.Exception.GetBaseException().Message.Should().Be("FAIL"); + + // commits will fail if we shut down the consumer too early + commitFailures.Should().BeEmpty(); + + } + } +} \ No newline at end of file diff --git a/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs b/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs index 0f4757ea..b15c71ab 100644 --- a/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs +++ b/src/Akka.Streams.Kafka.Tests/KafkaIntegrationTests.cs @@ -66,6 +66,14 @@ await Source .RunWith(KafkaProducer.PlainSink(producerSettings), Materializer); } + protected async Task ProduceStrings(Func partitionSelector, IEnumerable range, ProducerSettings producerSettings) + { + await Source + .From(range) + .Select(elem => new MessageAndMeta { TopicPartition = partitionSelector(elem), Message = new Message { Value = elem.ToString() } }) + .RunWith(KafkaProducer.PlainSink(producerSettings), Materializer); + } + protected async Task ProduceStrings(TopicPartition topicPartition, IEnumerable range, ProducerSettings producerSettings) { await Source From 1f9353e08eda2624a2ae7557067239b8ee193f99 Mon Sep 17 00:00:00 2001 From: IgorFedchenko Date: Fri, 11 Oct 2019 22:16:03 +0300 Subject: [PATCH 28/28] Nothing --- src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs index b2f73a82..d2c2f3e7 100644 --- a/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs +++ b/src/Akka.Streams.Kafka/Dsl/KafkaConsumer.cs @@ -121,7 +121,7 @@ public static Source, IControl> CommittableExternalSour } /// - /// Convenience for "at-most once delivery" semantics. + /// Convenience for "at-most once delivery" semantics. /// The offset of each message is committed to Kafka before being emitted downstream. /// public static Source, IControl> AtMostOnceSource(ConsumerSettings settings, ISubscription subscription)