From 4014879bf2642f462e332430f42cb8da88016ffd Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Wed, 26 Mar 2025 15:12:33 +0100 Subject: [PATCH 01/30] #1326 Pulsar support for native retry and DLQ queues - beware of ugly code --- .../DocumentationSamples.cs | 8 + .../Wolverine.Pulsar/DeadLetterTopic.cs | 124 +++++++++++++ .../Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs | 28 +++ .../Pulsar/Wolverine.Pulsar/PulsarListener.cs | 175 +++++++++++++++++- .../Wolverine.Pulsar/PulsarTransport.cs | 23 +++ .../PulsarTransportExtensions.cs | 95 ++++++++++ .../Wolverine.Pulsar/Wolverine.Pulsar.csproj | 7 +- src/Wolverine/Transports/IChannelCallback.cs | 10 + 8 files changed, 463 insertions(+), 7 deletions(-) create mode 100644 src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs index 6895b36c2..8e1aef7ec 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs @@ -37,6 +37,14 @@ public static async Task configure() // And all the normal Wolverine options... .Sequential(); + + + // Listen for incoming messages from a Pulsar topic with a shared subscription and using RETRY and DLQ queues + opts.ListenToPulsarTopic("persistent://public/default/three") + .WithSharedSubscriptionType() + .DeadLetterQueueing(new DeadLetterTopic("name", DeadLetterTopicMode.Native)) + .RetryLetterQueueing(new RetryTopic("retry", [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)])) + .Sequential(); }); #endregion diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs new file mode 100644 index 000000000..795301ccf --- /dev/null +++ b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs @@ -0,0 +1,124 @@ + + +namespace Wolverine.Pulsar; + +public enum DeadLetterTopicMode +{ + /// + /// Opt into using Pulsar's native dead letter topic approach. This is the default and recommended + /// + Native, + + /// + /// Completely ignore Pulsar native dead letter topic in favor of Wolverine persistent dead letter queueing + /// + WolverineStorage +} + +public class DeadLetterTopic +{ + private string? _topicName; + + public DeadLetterTopicMode Mode { get; set; } = DeadLetterTopicMode.Native; + + public DeadLetterTopic(string topicName) + { + _topicName = topicName; + } + + public DeadLetterTopic(string topicName, DeadLetterTopicMode mode) + { + _topicName = topicName; + Mode = mode; + } + + public string TopicName + { + get => _topicName; + set => _topicName = value?? throw new ArgumentNullException(nameof(TopicName)); + } + + protected bool Equals(DeadLetterTopic other) + { + return _topicName == other._topicName; + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) + { + return false; + } + + if (ReferenceEquals(this, obj)) + { + return true; + } + + if (obj.GetType() != this.GetType()) + { + return false; + } + + return Equals((DeadLetterTopic)obj); + } + + public override int GetHashCode() + { + return _topicName.GetHashCode(); + } +} + + +/// +/// TODO: how to handle retries internally in Wolverine? +/// +public class RetryTopic +{ + private string? _topicName; + private readonly List _retries; + + public RetryTopic(string topicName, List retries) + { + _topicName = topicName; + _retries = retries; + } + + public string TopicName + { + get => _topicName; + set => _topicName = value ?? throw new ArgumentNullException(nameof(TopicName)); + } + + public List Retry => _retries.ToList(); + + protected bool Equals(RetryTopic other) + { + return _topicName == other._topicName; + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) + { + return false; + } + + if (ReferenceEquals(this, obj)) + { + return true; + } + + if (obj.GetType() != this.GetType()) + { + return false; + } + + return Equals((RetryTopic)obj); + } + + public override int GetHashCode() + { + return _topicName.GetHashCode(); + } +} \ No newline at end of file diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs index d1c0cadce..99e797f28 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs @@ -1,5 +1,6 @@ using DotPulsar; using JasperFx.Core; +using Microsoft.Extensions.Logging; using Wolverine.Configuration; using Wolverine.Runtime; using Wolverine.Transports; @@ -28,6 +29,18 @@ public PulsarEndpoint(Uri uri, PulsarTransport parent) : base(uri, EndpointRole. public string SubscriptionName { get; internal set; } = "Wolverine"; public SubscriptionType SubscriptionType { get; internal set; } = SubscriptionType.Exclusive; + /// + /// Use to override the dead letter topic for this endpoint + /// + public DeadLetterTopic? DeadLetterTopic { get; set; } + + /// + /// Use to override the retry letter topic for this endpoint + /// + public RetryTopic? RetryLetterTopic { get; set; } + + public bool IsPersistent => Persistence.Equals(Persistent); + public static Uri UriFor(bool persistent, string tenant, string @namespace, string topicName) { var scheme = persistent ? "persistent" : "non-persistent"; @@ -94,4 +107,19 @@ protected override ISender CreateSender(IWolverineRuntime runtime) { return new PulsarSender(runtime, this, _parent, runtime.Cancellation); } + + public override ValueTask InitializeAsync(ILogger logger) + { + return base.InitializeAsync(logger); + } + + public override bool TryBuildDeadLetterSender(IWolverineRuntime runtime, out ISender? deadLetterSender) + { + //return base.TryBuildDeadLetterSender(runtime, out deadLetterSender); + + var queueName = this.DeadLetterTopic?.TopicName ?? _parent.DeadLetterTopic.TopicName; + var dlq = _parent[UriFor(queueName)]; + deadLetterSender = dlq.CreateSender(runtime); + return true; + } } \ No newline at end of file diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index 9ba45d69a..cb6c6c335 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -1,4 +1,5 @@ using System.Buffers; +using DotPulsar; using DotPulsar.Abstractions; using DotPulsar.Extensions; using Wolverine.Runtime; @@ -6,13 +7,16 @@ namespace Wolverine.Pulsar; -internal class PulsarListener : IListener +internal class PulsarListener : IListener, ISupportDeadLetterQueue, ISupportRetryLetterQueue { private readonly CancellationToken _cancellation; private readonly IConsumer>? _consumer; + private readonly IConsumer>? _retryConsumer; private readonly CancellationTokenSource _localCancellation; private readonly Task? _receivingLoop; + private readonly Task? _receivingRetryLoop; private readonly PulsarSender _sender; + private DeadLetterPolicy? _dlqClient; public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IReceiver receiver, PulsarTransport transport, @@ -40,20 +44,151 @@ public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IRecei .Topic(endpoint.PulsarTopic()) .Create(); + // TODO: check + var endpointRetryLetterTopicSettings = endpoint.RetryLetterTopic; + NativeDeadLetterQueueEnabled = transport.DeadLetterTopic != null && + transport.DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage || + endpoint.DeadLetterTopic != null && endpoint.DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage; + + NativeRetryLetterQueueEnabled = endpointRetryLetterTopicSettings != null; + + trySetupNativeResiliency(endpoint, transport); + _receivingLoop = Task.Run(async () => { + await foreach (var message in _consumer.Messages(combined.Token)) { var envelope = new PulsarEnvelope(message) { Data = message.Data.ToArray() }; + try + { + mapper.MapIncomingToEnvelope(envelope, message); - mapper.MapIncomingToEnvelope(envelope, message); - - await receiver.ReceivedAsync(this, envelope); + await receiver.ReceivedAsync(this, envelope); + } + catch (TaskCanceledException) + { + throw; + } + catch (Exception) + { + if (_dlqClient != null) + { + await _dlqClient.ReconsumeLater(message); + await receiver.ReceivedAsync(this, envelope); + //await _retryConsumer.Acknowledge(message); // TODO: check: original message should be acked and copy is sent to retry topic + } + } } + + }, combined.Token); + + + if (_dlqClient != null) + { + _retryConsumer = createRetryConsumer(endpoint, transport); + _receivingRetryLoop = Task.Run(async () => + { + await foreach (var message in _retryConsumer.Messages(combined.Token)) + { + var envelope = new PulsarEnvelope(message) + { + Data = message.Data.ToArray() + }; + try + { + mapper.MapIncomingToEnvelope(envelope, message); + + await receiver.ReceivedAsync(this, envelope); + } + catch (TaskCanceledException) + { + throw; + } + catch (Exception) + { + if (_dlqClient != null) + { + // TODO: can use to manage retries + //var retryCount = int.Parse(message.Properties["RECONSUMETIMES"]); + //await _dlqClient.ReconsumeLater(message, delayTime: endpointRetryLetterTopicSettings.Retry[retryCount]); + await _dlqClient.ReconsumeLater(message); + await receiver.ReceivedAsync(this, envelope); + //await _retryConsumer.Acknowledge(message); // TODO: check: original message should be acked and copy is sent to retry/DLQ + } + } + } + + }, combined.Token); + } + } + + private void trySetupNativeResiliency(PulsarEndpoint endpoint, PulsarTransport transport) + { + if (NativeRetryLetterQueueEnabled && NativeDeadLetterQueueEnabled) + { + var topicDql = getDeadLetteredTopicUri(endpoint); + var topicRetry = getRetryLetterTopicUri(endpoint); + + _dlqClient = new DeadLetterPolicy( + transport.Client!.NewProducer().Topic(topicDql.ToString()), + transport.Client!.NewProducer().Topic(topicRetry!.ToString()), + endpoint.RetryLetterTopic!.Retry.Count + ); + + } + else if (NativeRetryLetterQueueEnabled) + { + var topicRetry = getRetryLetterTopicUri(endpoint); + + _dlqClient = new DeadLetterPolicy( + null, + transport.Client!.NewProducer().Topic(topicRetry!.ToString()), + endpoint.RetryLetterTopic!.Retry.Count + ); + + } + else if (NativeDeadLetterQueueEnabled) + { + var topicDql = getDeadLetteredTopicUri(endpoint); + + _dlqClient = new DeadLetterPolicy( + transport.Client!.NewProducer().Topic(topicDql.ToString()), + null, + 0 + ); + } + } + + private IConsumer> createRetryConsumer(PulsarEndpoint endpoint, PulsarTransport transport) + { + var topicRetry = getRetryLetterTopicUri(endpoint); + + return transport.Client!.NewConsumer() + .SubscriptionName(endpoint.SubscriptionName) + .SubscriptionType(endpoint.SubscriptionType) + .Topic(topicRetry!.ToString()) + .Create(); + } + + private Uri? getRetryLetterTopicUri(PulsarEndpoint endpoint) + { + return NativeDeadLetterQueueEnabled + ? PulsarEndpoint.UriFor(endpoint.IsPersistent, endpoint.Tenant, endpoint.Namespace, + endpoint.RetryLetterTopic?.TopicName ?? $"{endpoint.TopicName}-RETRY") + : null; + } + + private Uri getDeadLetteredTopicUri(PulsarEndpoint endpoint) + { + var topicDql = PulsarEndpoint.UriFor(endpoint.IsPersistent, endpoint.Tenant, endpoint.Namespace, + endpoint.DeadLetterTopic?.TopicName ?? $"{endpoint.TopicName}-DLQ"); + + return topicDql; } public ValueTask CompleteAsync(Envelope envelope) @@ -87,9 +222,20 @@ public async ValueTask DisposeAsync() await _consumer.DisposeAsync(); } + if (_retryConsumer != null) + { + await _retryConsumer.DisposeAsync(); + } + + if (_dlqClient != null) + { + await _dlqClient.DisposeAsync(); + } + await _sender.DisposeAsync(); _receivingLoop!.Dispose(); + _receivingRetryLoop?.Dispose(); } public Uri Address { get; } @@ -103,6 +249,13 @@ public async ValueTask StopAsync() await _consumer.Unsubscribe(_cancellation); await _consumer.RedeliverUnacknowledgedMessages(_cancellation); + + + if (_retryConsumer != null) + { + await _retryConsumer.Unsubscribe(_cancellation); + await _retryConsumer.RedeliverUnacknowledgedMessages(_cancellation); + } } public async Task TryRequeueAsync(Envelope envelope) @@ -115,4 +268,18 @@ public async Task TryRequeueAsync(Envelope envelope) return false; } + + public bool NativeDeadLetterQueueEnabled { get; } + public Task MoveToErrorsAsync(Envelope envelope, Exception exception) + { + // TODO: pass through? + return Task.CompletedTask; + } + + public bool NativeRetryLetterQueueEnabled { get; } + public Task MoveToRetryQueueAsync(Envelope envelope, Exception exception) + { + // TODO: how to handle retries internally? + throw new NotImplementedException(); + } } \ No newline at end of file diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs index d5fdf4dd9..f9f9fe5aa 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs @@ -1,6 +1,7 @@ using DotPulsar; using DotPulsar.Abstractions; using JasperFx.Core; +using Wolverine.Configuration; using Wolverine.Runtime; using Wolverine.Transports; @@ -9,6 +10,7 @@ namespace Wolverine.Pulsar; public class PulsarTransport : TransportBase, IAsyncDisposable { public const string ProtocolName = "pulsar"; + public const string DeadLetterTopicName = "wolverine-dead-letter-topic"; private readonly LightweightCache _endpoints; @@ -25,6 +27,25 @@ public PulsarTransport() : base(ProtocolName, "Pulsar") public IPulsarClientBuilder Builder { get; } internal IPulsarClient? Client { get; private set; } + public DeadLetterTopic DeadLetterTopic { get; } = new(DeadLetterTopicName); + + + private IEnumerable enabledDeadLetterTopics() + { + if (DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage) + { + yield return DeadLetterTopic; + } + + foreach (var queue in endpoints()) + { + if (queue.IsPersistent && queue.Role == EndpointRole.Application && queue.DeadLetterTopic != null && + queue.DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage) + { + yield return queue.DeadLetterTopic; + } + } + } public ValueTask DisposeAsync() { @@ -52,6 +73,8 @@ public override ValueTask InitializeAsync(IWolverineRuntime runtime) return ValueTask.CompletedTask; } + + public PulsarEndpoint EndpointFor(string topicPath) { var uri = PulsarEndpoint.UriFor(topicPath); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs index bbcf640c9..38030d420 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs @@ -110,6 +110,32 @@ public PulsarListenerConfiguration SubscriptionType(SubscriptionType subscriptio return this; } + + /// + /// Override the Pulsar subscription type for just this topic + /// + /// + /// + public PulsarSharedListenerConfiguration WithSharedSubscriptionType() + { + add(e => { e.SubscriptionType = DotPulsar.SubscriptionType.Shared; }); + + return new PulsarSharedListenerConfiguration(this._endpoint); + } + + + /// + /// Override the Pulsar subscription type for just this topic + /// + /// + /// + public PulsarSharedListenerConfiguration WithKeySharedSubscriptionType() + { + add(e => { e.SubscriptionType = DotPulsar.SubscriptionType.KeyShared; }); + + return new PulsarSharedListenerConfiguration(this._endpoint); + } + /// /// Add circuit breaker exception handling to this listener /// @@ -143,6 +169,75 @@ public PulsarListenerConfiguration CircuitBreaker(Action? // } } + + +public class PulsarSharedListenerConfiguration : ListenerConfiguration +{ + public PulsarSharedListenerConfiguration(PulsarEndpoint endpoint) : base(endpoint) + { + } + + + /// + /// Customize the dead letter queueing for this specific endpoint + /// + /// Optional configuration + /// + public PulsarSharedListenerConfiguration DeadLetterQueueing(DeadLetterTopic dlq) + { + add(e => + { + e.DeadLetterTopic = dlq; + }); + + return this; + } + + /// + /// Remove all dead letter queueing declarations from this queue + /// + /// + public PulsarSharedListenerConfiguration DisableDeadLetterQueueing() + { + add(e => + { + e.DeadLetterTopic = null; + }); + + return this; + } + + /// + /// Customize the Retry letter queueing for this specific endpoint + /// + /// Optional configuration + /// + public PulsarSharedListenerConfiguration RetryLetterQueueing(RetryTopic rt) + { + add(e => + { + e.RetryLetterTopic = rt; + }); + + return this; + } + + /// + /// Remove all Retry letter queueing declarations from this queue + /// + /// + public PulsarSharedListenerConfiguration DisableRetryLetterQueueing() + { + add(e => + { + e.RetryLetterTopic = null; + }); + + return this; + } + +} + public class PulsarSubscriberConfiguration : SubscriberConfiguration { public PulsarSubscriberConfiguration(PulsarEndpoint endpoint) : base(endpoint) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/Wolverine.Pulsar.csproj b/src/Transports/Pulsar/Wolverine.Pulsar/Wolverine.Pulsar.csproj index 28c81e20c..a2178e27f 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/Wolverine.Pulsar.csproj +++ b/src/Transports/Pulsar/Wolverine.Pulsar/Wolverine.Pulsar.csproj @@ -9,12 +9,13 @@ false - + - + + - + diff --git a/src/Wolverine/Transports/IChannelCallback.cs b/src/Wolverine/Transports/IChannelCallback.cs index 10960df3d..85acd7d71 100644 --- a/src/Wolverine/Transports/IChannelCallback.cs +++ b/src/Wolverine/Transports/IChannelCallback.cs @@ -10,6 +10,16 @@ public interface ISupportDeadLetterQueue Task MoveToErrorsAsync(Envelope envelope, Exception exception); } +/// +/// Marks an IChannelCallback as supporting a native retry letter queue +/// functionality +/// +public interface ISupportRetryLetterQueue +{ + bool NativeRetryLetterQueueEnabled { get; } + Task MoveToRetryQueueAsync(Envelope envelope, Exception exception); +} + /// /// Marks an IChannelCallback as supporting native scheduled send /// From e846db90eaa40b652a3b96d30c46199458ee5578 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Wed, 26 Mar 2025 15:12:49 +0100 Subject: [PATCH 02/30] #1326 Pulsar support for native retry and DLQ queues - beware of ugly code --- .../Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs index 99e797f28..a198afe88 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs @@ -115,11 +115,13 @@ public override ValueTask InitializeAsync(ILogger logger) public override bool TryBuildDeadLetterSender(IWolverineRuntime runtime, out ISender? deadLetterSender) { - //return base.TryBuildDeadLetterSender(runtime, out deadLetterSender); + return base.TryBuildDeadLetterSender(runtime, out deadLetterSender); - var queueName = this.DeadLetterTopic?.TopicName ?? _parent.DeadLetterTopic.TopicName; - var dlq = _parent[UriFor(queueName)]; - deadLetterSender = dlq.CreateSender(runtime); - return true; + + // TODO: ? + //var queueName = this.DeadLetterTopic?.TopicName ?? _parent.DeadLetterTopic.TopicName; + //var dlq = _parent[UriFor(queueName)]; + //deadLetterSender = dlq.CreateSender(runtime); + //return true; } } \ No newline at end of file From 0d14f9bf301b336890c8b6630d39e70aa99265a8 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Wed, 26 Mar 2025 15:28:30 +0100 Subject: [PATCH 03/30] #1326 Pulsar support for native retry and DLQ queues - beware of ugly code --- src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs index f9f9fe5aa..88365a025 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs @@ -27,7 +27,7 @@ public PulsarTransport() : base(ProtocolName, "Pulsar") public IPulsarClientBuilder Builder { get; } internal IPulsarClient? Client { get; private set; } - public DeadLetterTopic DeadLetterTopic { get; } = new(DeadLetterTopicName); + public DeadLetterTopic DeadLetterTopic { get; } = new(DeadLetterTopicName); // TODO: should we even have a default or just per endpoint based private IEnumerable enabledDeadLetterTopics() From 9caf90d5d8fdf02f4f25ef071bbb7d806387aae9 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Thu, 27 Mar 2025 10:40:05 +0100 Subject: [PATCH 04/30] #1326 Additional constructors for DeadLetterTopic and RetryLetterTopic --- .../Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs | 10 +++++++--- .../Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs | 2 +- .../Wolverine.Pulsar/PulsarTransportExtensions.cs | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs index 795301ccf..75d89b64b 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs @@ -19,11 +19,11 @@ public class DeadLetterTopic { private string? _topicName; - public DeadLetterTopicMode Mode { get; set; } = DeadLetterTopicMode.Native; + public DeadLetterTopicMode Mode { get; set; } - public DeadLetterTopic(string topicName) + public DeadLetterTopic(DeadLetterTopicMode mode) { - _topicName = topicName; + Mode = mode; } public DeadLetterTopic(string topicName, DeadLetterTopicMode mode) @@ -78,6 +78,10 @@ public class RetryTopic private string? _topicName; private readonly List _retries; + public RetryTopic(List retries) + { + _retries = retries; + } public RetryTopic(string topicName, List retries) { _topicName = topicName; diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs index a198afe88..3eb6736b5 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs @@ -37,7 +37,7 @@ public PulsarEndpoint(Uri uri, PulsarTransport parent) : base(uri, EndpointRole. /// /// Use to override the retry letter topic for this endpoint /// - public RetryTopic? RetryLetterTopic { get; set; } + public RetryLetterTopic? RetryLetterTopic { get; set; } public bool IsPersistent => Persistence.Equals(Persistent); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs index 38030d420..06635bd99 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs @@ -212,7 +212,7 @@ public PulsarSharedListenerConfiguration DisableDeadLetterQueueing() /// /// Optional configuration /// - public PulsarSharedListenerConfiguration RetryLetterQueueing(RetryTopic rt) + public PulsarSharedListenerConfiguration RetryLetterQueueing(RetryLetterTopic rt) { add(e => { From 8dd878676b9ebc1e5dbc5f026d1f9ee417b2919c Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Thu, 27 Mar 2025 10:40:31 +0100 Subject: [PATCH 05/30] #1326 Additional constructors for DeadLetterTopic and RetryLetterTopic --- .../Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs index 75d89b64b..400c418b3 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs @@ -73,16 +73,16 @@ public override int GetHashCode() /// /// TODO: how to handle retries internally in Wolverine? /// -public class RetryTopic +public class RetryLetterTopic { private string? _topicName; private readonly List _retries; - public RetryTopic(List retries) + public RetryLetterTopic(List retries) { _retries = retries; } - public RetryTopic(string topicName, List retries) + public RetryLetterTopic(string topicName, List retries) { _topicName = topicName; _retries = retries; @@ -96,7 +96,7 @@ public string TopicName public List Retry => _retries.ToList(); - protected bool Equals(RetryTopic other) + protected bool Equals(RetryLetterTopic other) { return _topicName == other._topicName; } @@ -118,7 +118,7 @@ public override bool Equals(object? obj) return false; } - return Equals((RetryTopic)obj); + return Equals((RetryLetterTopic)obj); } public override int GetHashCode() From 42890d6002b56054f2d381259cd9fca63e53860d Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Thu, 27 Mar 2025 10:42:03 +0100 Subject: [PATCH 06/30] #1326 Additional constructors for DeadLetterTopic and RetryLetterTopic --- .../Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs | 2 +- src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs index 8e1aef7ec..575cd1605 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs @@ -43,7 +43,7 @@ public static async Task configure() opts.ListenToPulsarTopic("persistent://public/default/three") .WithSharedSubscriptionType() .DeadLetterQueueing(new DeadLetterTopic("name", DeadLetterTopicMode.Native)) - .RetryLetterQueueing(new RetryTopic("retry", [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)])) + .RetryLetterQueueing(new RetryLetterTopic("retry", [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)])) .Sequential(); }); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs index 400c418b3..39655ffe6 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs @@ -88,7 +88,7 @@ public RetryLetterTopic(string topicName, List retries) _retries = retries; } - public string TopicName + public string? TopicName { get => _topicName; set => _topicName = value ?? throw new ArgumentNullException(nameof(TopicName)); From 90ad042a95e017f301950a05e3e4912c9b0a4dfc Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Thu, 27 Mar 2025 11:10:03 +0100 Subject: [PATCH 07/30] #1326 refactoring --- .../Wolverine.Pulsar/DeadLetterTopic.cs | 18 +++++++ .../Pulsar/Wolverine.Pulsar/PulsarListener.cs | 54 +++++++------------ .../Wolverine.Pulsar/PulsarTransport.cs | 5 +- 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs index 39655ffe6..728943543 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs @@ -1,5 +1,7 @@ +using DotPulsar; + namespace Wolverine.Pulsar; public enum DeadLetterTopicMode @@ -17,6 +19,9 @@ public enum DeadLetterTopicMode public class DeadLetterTopic { + + public static DeadLetterTopic DefaultNative => new(DeadLetterTopicMode.Native); + private string? _topicName; public DeadLetterTopicMode Mode { get; set; } @@ -75,6 +80,19 @@ public override int GetHashCode() /// public class RetryLetterTopic { + public static RetryLetterTopic DefaultNative => new([ + TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(5), TimeSpan.FromMinutes(2) + ]); + + + /// + /// Message delaying does not work with Pulsar if the subscription type is not shared or key shared + /// + public static IReadOnlySet SupportedSubscriptionTypes = new HashSet() + { + SubscriptionType.Shared, SubscriptionType.KeyShared + }; + private string? _topicName; private readonly List _retries; diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index cb6c6c335..2f498cf5c 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -45,12 +45,11 @@ public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IRecei .Create(); // TODO: check - var endpointRetryLetterTopicSettings = endpoint.RetryLetterTopic; - NativeDeadLetterQueueEnabled = transport.DeadLetterTopic != null && + NativeDeadLetterQueueEnabled = transport.DeadLetterTopic is not null && transport.DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage || - endpoint.DeadLetterTopic != null && endpoint.DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage; + endpoint.DeadLetterTopic is not null && endpoint.DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage; - NativeRetryLetterQueueEnabled = endpointRetryLetterTopicSettings != null; + NativeRetryLetterQueueEnabled = endpoint.RetryLetterTopic is not null && RetryLetterTopic.SupportedSubscriptionTypes.Contains(endpoint.SubscriptionType); trySetupNativeResiliency(endpoint, transport); @@ -113,9 +112,9 @@ public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IRecei { if (_dlqClient != null) { - // TODO: can use to manage retries - //var retryCount = int.Parse(message.Properties["RECONSUMETIMES"]); - //await _dlqClient.ReconsumeLater(message, delayTime: endpointRetryLetterTopicSettings.Retry[retryCount]); + // TODO: used to manage retries - refactor + var retryCount = int.Parse(message.Properties["RECONSUMETIMES"]); + await _dlqClient.ReconsumeLater(message, delayTime: endpoint.RetryLetterTopic!.Retry[retryCount]); await _dlqClient.ReconsumeLater(message); await receiver.ReceivedAsync(this, envelope); //await _retryConsumer.Acknowledge(message); // TODO: check: original message should be acked and copy is sent to retry/DLQ @@ -129,41 +128,24 @@ public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IRecei private void trySetupNativeResiliency(PulsarEndpoint endpoint, PulsarTransport transport) { - if (NativeRetryLetterQueueEnabled && NativeDeadLetterQueueEnabled) + if (!NativeRetryLetterQueueEnabled && !NativeDeadLetterQueueEnabled) { - var topicDql = getDeadLetteredTopicUri(endpoint); - var topicRetry = getRetryLetterTopicUri(endpoint); - - _dlqClient = new DeadLetterPolicy( - transport.Client!.NewProducer().Topic(topicDql.ToString()), - transport.Client!.NewProducer().Topic(topicRetry!.ToString()), - endpoint.RetryLetterTopic!.Retry.Count - ); - + return; } - else if (NativeRetryLetterQueueEnabled) - { - var topicRetry = getRetryLetterTopicUri(endpoint); - - _dlqClient = new DeadLetterPolicy( - null, - transport.Client!.NewProducer().Topic(topicRetry!.ToString()), - endpoint.RetryLetterTopic!.Retry.Count - ); - } - else if (NativeDeadLetterQueueEnabled) - { - var topicDql = getDeadLetteredTopicUri(endpoint); + var topicDql = NativeDeadLetterQueueEnabled ? getDeadLetteredTopicUri(endpoint) : null; + var topicRetry = NativeRetryLetterQueueEnabled ? getRetryLetterTopicUri(endpoint) : null; + var retryCount = NativeRetryLetterQueueEnabled ? endpoint.RetryLetterTopic!.Retry.Count : 0; - _dlqClient = new DeadLetterPolicy( - transport.Client!.NewProducer().Topic(topicDql.ToString()), - null, - 0 - ); - } + _dlqClient = new DeadLetterPolicy( + topicDql != null ? transport.Client!.NewProducer().Topic(topicDql.ToString()) : null, + topicRetry != null ? transport.Client!.NewProducer().Topic(topicRetry.ToString()) : null, + retryCount + ); } + + private IConsumer> createRetryConsumer(PulsarEndpoint endpoint, PulsarTransport transport) { var topicRetry = getRetryLetterTopicUri(endpoint); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs index 88365a025..b35243f22 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs @@ -10,7 +10,7 @@ namespace Wolverine.Pulsar; public class PulsarTransport : TransportBase, IAsyncDisposable { public const string ProtocolName = "pulsar"; - public const string DeadLetterTopicName = "wolverine-dead-letter-topic"; + private readonly LightweightCache _endpoints; @@ -27,7 +27,8 @@ public PulsarTransport() : base(ProtocolName, "Pulsar") public IPulsarClientBuilder Builder { get; } internal IPulsarClient? Client { get; private set; } - public DeadLetterTopic DeadLetterTopic { get; } = new(DeadLetterTopicName); // TODO: should we even have a default or just per endpoint based + public DeadLetterTopic? DeadLetterTopic { get; internal set; } // TODO: should we even have a default or just per endpoint based? + public RetryLetterTopic? RetryLetterTopic { get; internal set; } // TODO: should we even have a default or just per endpoint based? private IEnumerable enabledDeadLetterTopics() From ab74d4f5b91a9d7fb72bfb9b7c2b6b800566db4a Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Thu, 27 Mar 2025 11:16:29 +0100 Subject: [PATCH 08/30] #1326 refactoring --- src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index 2f498cf5c..bbee1c56d 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -115,7 +115,6 @@ public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IRecei // TODO: used to manage retries - refactor var retryCount = int.Parse(message.Properties["RECONSUMETIMES"]); await _dlqClient.ReconsumeLater(message, delayTime: endpoint.RetryLetterTopic!.Retry[retryCount]); - await _dlqClient.ReconsumeLater(message); await receiver.ReceivedAsync(this, envelope); //await _retryConsumer.Acknowledge(message); // TODO: check: original message should be acked and copy is sent to retry/DLQ } From d0d1d8c086f3fade43af76ff0888bdbc9be0e937 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Thu, 27 Mar 2025 11:53:22 +0100 Subject: [PATCH 09/30] #1326 fix --- .../Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs | 4 ++-- .../Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs index 575cd1605..f90cb08f1 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/DocumentationSamples.cs @@ -42,8 +42,8 @@ public static async Task configure() // Listen for incoming messages from a Pulsar topic with a shared subscription and using RETRY and DLQ queues opts.ListenToPulsarTopic("persistent://public/default/three") .WithSharedSubscriptionType() - .DeadLetterQueueing(new DeadLetterTopic("name", DeadLetterTopicMode.Native)) - .RetryLetterQueueing(new RetryLetterTopic("retry", [TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)])) + .DeadLetterQueueing(new DeadLetterTopic(DeadLetterTopicMode.Native)) + .RetryLetterQueueing(new RetryLetterTopic([TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)])) .Sequential(); }); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs index 06635bd99..5065a7c05 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs @@ -171,7 +171,7 @@ public PulsarListenerConfiguration CircuitBreaker(Action? -public class PulsarSharedListenerConfiguration : ListenerConfiguration +public class PulsarSharedListenerConfiguration : ListenerConfiguration { public PulsarSharedListenerConfiguration(PulsarEndpoint endpoint) : base(endpoint) { From 656daf0be36b9fa753e24de4792aeaae20cdff05 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Thu, 27 Mar 2025 13:23:49 +0100 Subject: [PATCH 10/30] #1326 refactoring for MoveToErrorsAsync --- .../Pulsar/Wolverine.Pulsar/PulsarListener.cs | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index bbee1c56d..1b01091ac 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -2,6 +2,7 @@ using DotPulsar; using DotPulsar.Abstractions; using DotPulsar.Extensions; +using DotPulsar.Internal; using Wolverine.Runtime; using Wolverine.Transports; @@ -17,16 +18,15 @@ internal class PulsarListener : IListener, ISupportDeadLetterQueue, ISupportRetr private readonly Task? _receivingRetryLoop; private readonly PulsarSender _sender; private DeadLetterPolicy? _dlqClient; + private IReceiver _receiver; + private PulsarEndpoint _endpoint; public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IReceiver receiver, PulsarTransport transport, CancellationToken cancellation) { - if (receiver == null) - { - throw new ArgumentNullException(nameof(receiver)); - } - + _endpoint = endpoint; + _receiver = receiver ?? throw new ArgumentNullException(nameof(receiver)); _cancellation = cancellation; Address = endpoint.Uri; @@ -251,10 +251,34 @@ public async Task TryRequeueAsync(Envelope envelope) } public bool NativeDeadLetterQueueEnabled { get; } - public Task MoveToErrorsAsync(Envelope envelope, Exception exception) + public async Task MoveToErrorsAsync(Envelope envelope, Exception exception) { - // TODO: pass through? - return Task.CompletedTask; + // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) + + if (envelope is PulsarEnvelope e) + { + if (_dlqClient != null) + { + var message = e.MessageData; + // TODO: used to manage retries - refactor + if(message.Properties.TryGetValue("RECONSUMETIMES", out var reconsumeTimesValue)) + { + var retryCount = int.Parse(reconsumeTimesValue); + await _retryConsumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ + //await _retryConsumer.Acknowledge(message); // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection + await _dlqClient.ReconsumeLater(message, delayTime: _endpoint.RetryLetterTopic!.Retry[retryCount], cancellationToken: _cancellation); + } + else + { + // first time failure or no retry letter topic configured + await _consumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ + //await _retryConsumer.Acknowledge(message); // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? + await _dlqClient.ReconsumeLater(message, delayTime: _endpoint.RetryLetterTopic!.Retry.First(), cancellationToken: _cancellation); + } + } + + } + } public bool NativeRetryLetterQueueEnabled { get; } From f5f844e41835a9dcbf3121a6e72f5b345685b530 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Thu, 27 Mar 2025 13:42:21 +0100 Subject: [PATCH 11/30] #1326 fix --- src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index 1b01091ac..3b1bcc264 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -266,7 +266,8 @@ public async Task MoveToErrorsAsync(Envelope envelope, Exception exception) var retryCount = int.Parse(reconsumeTimesValue); await _retryConsumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ //await _retryConsumer.Acknowledge(message); // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection - await _dlqClient.ReconsumeLater(message, delayTime: _endpoint.RetryLetterTopic!.Retry[retryCount], cancellationToken: _cancellation); + //TODO: e.Attempts / attempts header value is out of sync with Pulsar's RECONSUMETIMES header! + await _dlqClient.ReconsumeLater(message, delayTime: _endpoint.RetryLetterTopic!.Retry[retryCount - 1], cancellationToken: _cancellation); } else { From c163b41abc048416b2db3d34c9d51f7361f807dc Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Thu, 27 Mar 2025 13:49:07 +0100 Subject: [PATCH 12/30] #1326 refactoring --- src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index 3b1bcc264..f90b1dbb1 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -265,7 +265,7 @@ public async Task MoveToErrorsAsync(Envelope envelope, Exception exception) { var retryCount = int.Parse(reconsumeTimesValue); await _retryConsumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ - //await _retryConsumer.Acknowledge(message); // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection + //await _retryConsumer.Acknowledge(message); // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection. If I understand correctly, Wolverine is/should handle original Wolverine message and its copies across Pulsar's topics as same identity? //TODO: e.Attempts / attempts header value is out of sync with Pulsar's RECONSUMETIMES header! await _dlqClient.ReconsumeLater(message, delayTime: _endpoint.RetryLetterTopic!.Retry[retryCount - 1], cancellationToken: _cancellation); } From 30def929bfb139ba32ed8f6dfb562036c9526b38 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Fri, 28 Mar 2025 14:36:42 +0100 Subject: [PATCH 13/30] #1326 added support for ISupportRetryLetterQueue --- .../Pulsar/Wolverine.Pulsar/PulsarListener.cs | 98 ++++++++++++++--- .../PulsarTransportExtensions.cs | 14 +++ .../ErrorHandling/MoveToRetryQueueSource.cs | 104 ++++++++++++++++++ src/Wolverine/IEnvelopeLifecycle.cs | 2 + src/Wolverine/Logging/IMessageLogger.cs | 8 ++ src/Wolverine/Runtime/MessageContext.cs | 23 ++++ .../Runtime/WolverineRuntime.Tracking.cs | 10 ++ src/Wolverine/Runtime/WolverineTracing.cs | 8 +- src/Wolverine/Tracking/MessageEventType.cs | 3 +- src/Wolverine/Transports/IChannelCallback.cs | 1 + 10 files changed, 252 insertions(+), 19 deletions(-) create mode 100644 src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index f90b1dbb1..1932e277d 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -254,38 +254,102 @@ public async Task TryRequeueAsync(Envelope envelope) public async Task MoveToErrorsAsync(Envelope envelope, Exception exception) { // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) + await moveToQueueAsync(envelope, exception, isRetry: false); + } + + public bool NativeRetryLetterQueueEnabled { get; } + public bool RetryLimitReached(Envelope envelope) + { + if (NativeRetryLetterQueueEnabled && envelope is PulsarEnvelope e) + { + if (e.MessageData.Properties.TryGetValue("RECONSUMETIMES", out var reconsumeTimesValue)) + { + var currentRetryCount = int.Parse(reconsumeTimesValue); + + return currentRetryCount >= _endpoint.RetryLetterTopic!.Retry.Count; + } + // first time failure + return false; + } + + return true; + } + + public async Task MoveToRetryQueueAsync(Envelope envelope, Exception exception) + { + // TODO: how to handle retries internally? + // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) + await moveToQueueAsync(envelope, exception, isRetry: false); + } + + private async Task moveToQueueAsync(Envelope envelope, Exception exception, bool isRetry) + { if (envelope is PulsarEnvelope e) { if (_dlqClient != null) { var message = e.MessageData; - // TODO: used to manage retries - refactor - if(message.Properties.TryGetValue("RECONSUMETIMES", out var reconsumeTimesValue)) + IConsumer>? associatedConsumer; + TimeSpan? delayTime = null; + + if (message.TryGetMessageProperty("RECONSUMETIMES", out var reconsumeTimesValue)) { + associatedConsumer = _retryConsumer; var retryCount = int.Parse(reconsumeTimesValue); - await _retryConsumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ - //await _retryConsumer.Acknowledge(message); // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection. If I understand correctly, Wolverine is/should handle original Wolverine message and its copies across Pulsar's topics as same identity? - //TODO: e.Attempts / attempts header value is out of sync with Pulsar's RECONSUMETIMES header! - await _dlqClient.ReconsumeLater(message, delayTime: _endpoint.RetryLetterTopic!.Retry[retryCount - 1], cancellationToken: _cancellation); + delayTime = _endpoint.RetryLetterTopic!.Retry[retryCount - 1]; } else { - // first time failure or no retry letter topic configured - await _consumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ - //await _retryConsumer.Acknowledge(message); // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? - await _dlqClient.ReconsumeLater(message, delayTime: _endpoint.RetryLetterTopic!.Retry.First(), cancellationToken: _cancellation); + associatedConsumer = _consumer; } - } + await associatedConsumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ + // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection. If I understand correctly, Wolverine is/should handle original Wolverine message and its copies across Pulsar's topics as same identity? + // TODO: e.Attempts / attempts header value is out of sync with Pulsar's RECONSUMETIMES header! + await _dlqClient.ReconsumeLater(message, delayTime: delayTime, cancellationToken: _cancellation); + } } - } - public bool NativeRetryLetterQueueEnabled { get; } - public Task MoveToRetryQueueAsync(Envelope envelope, Exception exception) + //public async Task MoveToRetryQueueAsync(Envelope envelope, Exception exception) + //{ + // // TODO: how to handle retries internally? + // // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) + + // if (envelope is PulsarEnvelope e) + // { + // if (_dlqClient != null) + // { + // var message = e.MessageData; + // // TODO: used to manage retries - refactor + // if (message.Properties.TryGetValue("RECONSUMETIMES", out var reconsumeTimesValue)) + // { + // var retryCount = int.Parse(reconsumeTimesValue); + // await _retryConsumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ + // //await _retryConsumer.Acknowledge(message); // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection. If I understand correctly, Wolverine is/should handle original Wolverine message and its copies across Pulsar's topics as same identity? + // //TODO: e.Attempts / attempts header value is out of sync with Pulsar's RECONSUMETIMES header! + // await _dlqClient.ReconsumeLater(message, delayTime: _endpoint.RetryLetterTopic!.Retry[retryCount - 1], cancellationToken: _cancellation); + // } + // else + // { + // // first time failure or no retry letter topic configured + // await _consumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ + // //await _retryConsumer.Acknowledge(message); // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? + // await _dlqClient.ReconsumeLater(message, delayTime: _endpoint.RetryLetterTopic!.Retry.First(), cancellationToken: _cancellation); + // } + // } + + // } + //} +} + + +public static class MessageExtensions +{ + public static bool TryGetMessageProperty(this DotPulsar.Abstractions.IMessage message, string key, out string val) { - // TODO: how to handle retries internally? - throw new NotImplementedException(); + return message.Properties.TryGetValue(key , out val); } -} \ No newline at end of file +} + diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs index 5065a7c05..e2d45b7e8 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs @@ -3,6 +3,7 @@ using JasperFx.Core.Reflection; using Wolverine.Configuration; using Wolverine.ErrorHandling; +using Wolverine.ErrorHandling.Matches; namespace Wolverine.Pulsar; @@ -216,7 +217,19 @@ public PulsarSharedListenerConfiguration RetryLetterQueueing(RetryLetterTopic rt { add(e => { + + // TODO: This is a bit of a hack to get the retry letter topic in place e.RetryLetterTopic = rt; + + var exceptionMatch = new AlwaysMatches(); // currently can't determine if endpoint listener needs it just based on exception + var failureRule = new FailureRule(exceptionMatch); + + foreach (var _ in rt.Retry) + { + failureRule.AddSlot(new MoveToRetryQueueSource()); + } + + e.Runtime.Options.Policies.Failures.Add(failureRule); }); return this; @@ -231,6 +244,7 @@ public PulsarSharedListenerConfiguration DisableRetryLetterQueueing() add(e => { e.RetryLetterTopic = null; + //e.Runtime.Options.Policies.Failures // TODO: remove the failure rule }); return this; diff --git a/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs b/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs new file mode 100644 index 000000000..e949f6902 --- /dev/null +++ b/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs @@ -0,0 +1,104 @@ +using System.Diagnostics; +using Wolverine.Runtime; +using Wolverine.Transports; +using Wolverine.Transports.Local; +using Wolverine.Util; + +namespace Wolverine.ErrorHandling; + +internal class MoveToRetryQueueSource : IContinuationSource +{ + public string Description => "Move to retry queue"; + + public IContinuation Build(Exception ex, Envelope envelope) + { + return new MoveToRetryQueue(ex); + } +} + +internal class MoveToRetryQueue : IContinuation +{ + public MoveToRetryQueue(Exception exception) + { + Exception = exception ?? throw new ArgumentNullException(nameof(exception)); + } + + public Exception Exception { get; } + + public async ValueTask ExecuteAsync(IEnvelopeLifecycle lifecycle, + IWolverineRuntime runtime, + DateTimeOffset now, Activity? activity) + { + //var scheme = lifecycle.Envelope.Destination.Scheme; + //if (runtime.Options.EnableAutomaticFailureAcks && scheme != TransportConstants.Local && scheme != "external-table") + //{ + // // TODO: remove or rather ack? + // await lifecycle.SendFailureAcknowledgementAsync( + // $"Moved message {lifecycle.Envelope!.Id} to the Retry Queue.\n{Exception}"); + //} + + if (lifecycle.Envelope.Listener is ISupportRetryLetterQueue retryListener && + !retryListener.RetryLimitReached(lifecycle.Envelope)) + { + if (lifecycle.Envelope.Message != null) + { + lifecycle.Envelope.MessageType = lifecycle.Envelope.Message.GetType().ToMessageTypeName(); + } + + await lifecycle.MoveToRetryLetterQueueAsync(Exception); + + activity?.AddEvent(new ActivityEvent(WolverineTracing.MovedToRetryQueue)); + + runtime.MessageTracking.Requeued(lifecycle.Envelope); // TODO: new method + runtime.MessageTracking.MovedToRetryQueue(lifecycle.Envelope, Exception); + + } + else + { + await buildFallbackContinuation(lifecycle) + .ExecuteAsync(lifecycle, runtime, now, activity); + } + } + + private IContinuation buildFallbackContinuation(IEnvelopeLifecycle lifecycle) + { + var cs = new MoveToErrorQueueSource(); + var continuation = cs.Build(Exception, lifecycle.Envelope); + return continuation; + } + + public override string ToString() + { + return "Move to Retry Queue"; + } + + protected bool Equals(MoveToRetryQueue other) + { + return Equals(Exception, other.Exception); + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) + { + return false; + } + + if (ReferenceEquals(this, obj)) + { + return true; + } + + if (obj.GetType() != GetType()) + { + return false; + } + + return Equals((MoveToRetryQueue)obj); + } + + public override int GetHashCode() + { + return Exception.GetHashCode(); + } +} \ No newline at end of file diff --git a/src/Wolverine/IEnvelopeLifecycle.cs b/src/Wolverine/IEnvelopeLifecycle.cs index 706f7d33a..ed42c1774 100644 --- a/src/Wolverine/IEnvelopeLifecycle.cs +++ b/src/Wolverine/IEnvelopeLifecycle.cs @@ -24,6 +24,8 @@ public interface IEnvelopeLifecycle : IMessageBus Task ReScheduleAsync(DateTimeOffset scheduledTime); + Task MoveToRetryLetterQueueAsync(Exception exception); + Task MoveToDeadLetterQueueAsync(Exception exception); /// diff --git a/src/Wolverine/Logging/IMessageLogger.cs b/src/Wolverine/Logging/IMessageLogger.cs index 408a8f807..e35f4a82a 100644 --- a/src/Wolverine/Logging/IMessageLogger.cs +++ b/src/Wolverine/Logging/IMessageLogger.cs @@ -72,6 +72,14 @@ public interface IMessageTracker /// void NoRoutesFor(Envelope envelope); + + /// + /// Called when Wolverine moves an envelope into the retry letter queue + /// + /// + /// + void MovedToRetryQueue(Envelope envelope, Exception ex); + /// /// Called when Wolverine moves an envelope into the dead letter queue /// diff --git a/src/Wolverine/Runtime/MessageContext.cs b/src/Wolverine/Runtime/MessageContext.cs index fcea4c9c9..35c5f591e 100644 --- a/src/Wolverine/Runtime/MessageContext.cs +++ b/src/Wolverine/Runtime/MessageContext.cs @@ -135,6 +135,29 @@ public async Task ReScheduleAsync(DateTimeOffset scheduledTime) } } + public async Task MoveToRetryLetterQueueAsync(Exception exception) + { + if (_channel == null || Envelope == null) + { + throw new InvalidOperationException("No Envelope is active for this context"); + } + + if (_channel is ISupportRetryLetterQueue c && c.NativeRetryLetterQueueEnabled) + { + if (Envelope.Batch != null) + { + foreach (var envelope in Envelope.Batch) + { + await c.MoveToRetryQueueAsync(envelope, exception); + } + } + else + { + await c.MoveToRetryQueueAsync(Envelope, exception); + } + } + } + public async Task MoveToDeadLetterQueueAsync(Exception exception) { // Don't bother with agent commands diff --git a/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs b/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs index f56d4edb6..c606509cc 100644 --- a/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs +++ b/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs @@ -15,6 +15,7 @@ public sealed partial class WolverineRuntime : IMessageTracker public const int UndeliverableEventId = 108; private static readonly Action _movedToErrorQueue; + private static readonly Action _movedToRetryQueue; private static readonly Action _noHandler; private static readonly Action _noRoutes; private static readonly Action _received; @@ -41,6 +42,9 @@ static WolverineRuntime() _noRoutes = LoggerMessage.Define(LogLevel.Information, NoRoutesEventId, "No routes can be determined for {envelope}"); + _movedToRetryQueue = LoggerMessage.Define(LogLevel.Error, MovedToErrorQueueId, + "Envelope {envelope} was moved to the retry queue"); + _movedToErrorQueue = LoggerMessage.Define(LogLevel.Error, MovedToErrorQueueId, "Envelope {envelope} was moved to the error queue"); @@ -128,6 +132,12 @@ public void NoRoutesFor(Envelope envelope) _noRoutes(Logger, envelope, null); } + public void MovedToRetryQueue(Envelope envelope, Exception ex) + { + ActiveSession?.Record(MessageEventType.MovedToRetryQueue, envelope, _serviceName, _uniqueNodeId); + _movedToRetryQueue(Logger, envelope, ex); + } + public void MovedToErrorQueue(Envelope envelope, Exception ex) { ActiveSession?.Record(MessageEventType.MovedToErrorQueue, envelope, _serviceName, _uniqueNodeId); diff --git a/src/Wolverine/Runtime/WolverineTracing.cs b/src/Wolverine/Runtime/WolverineTracing.cs index 0f2e03f8a..08e5e4fc0 100644 --- a/src/Wolverine/Runtime/WolverineTracing.cs +++ b/src/Wolverine/Runtime/WolverineTracing.cs @@ -30,7 +30,13 @@ public const string /// ActivityEvent marking when an incoming envelope is discarded /// public const string EnvelopeDiscarded = "wolverine.envelope.discarded"; - + + + /// + /// ActivityEvent marking when an incoming envelope is being moved to the retry queue + /// + public const string MovedToRetryQueue = "wolverine.error.retry.queued"; + /// /// ActivityEvent marking when an incoming envelope is being moved to the error queue /// diff --git a/src/Wolverine/Tracking/MessageEventType.cs b/src/Wolverine/Tracking/MessageEventType.cs index d71a2a55a..ac9cd0c58 100644 --- a/src/Wolverine/Tracking/MessageEventType.cs +++ b/src/Wolverine/Tracking/MessageEventType.cs @@ -12,6 +12,7 @@ public enum MessageEventType NoHandlers, NoRoutes, MovedToErrorQueue, - Requeued + Requeued, + MovedToRetryQueue, } #endregion \ No newline at end of file diff --git a/src/Wolverine/Transports/IChannelCallback.cs b/src/Wolverine/Transports/IChannelCallback.cs index 85acd7d71..bed88f60e 100644 --- a/src/Wolverine/Transports/IChannelCallback.cs +++ b/src/Wolverine/Transports/IChannelCallback.cs @@ -17,6 +17,7 @@ public interface ISupportDeadLetterQueue public interface ISupportRetryLetterQueue { bool NativeRetryLetterQueueEnabled { get; } + bool RetryLimitReached(Envelope envelope); Task MoveToRetryQueueAsync(Envelope envelope, Exception exception); } From 93baf89aa1a075ffd8949fe2c20991b048f3c055 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Fri, 28 Mar 2025 14:42:37 +0100 Subject: [PATCH 14/30] #1326 refactoring --- .../Wolverine.Pulsar/DeadLetterTopic.cs | 73 ------------------- .../Wolverine.Pulsar/RetryLetterTopic.cs | 73 +++++++++++++++++++ 2 files changed, 73 insertions(+), 73 deletions(-) create mode 100644 src/Transports/Pulsar/Wolverine.Pulsar/RetryLetterTopic.cs diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs index 728943543..1805d473e 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/DeadLetterTopic.cs @@ -1,7 +1,5 @@ -using DotPulsar; - namespace Wolverine.Pulsar; public enum DeadLetterTopicMode @@ -68,77 +66,6 @@ public override bool Equals(object? obj) return Equals((DeadLetterTopic)obj); } - public override int GetHashCode() - { - return _topicName.GetHashCode(); - } -} - - -/// -/// TODO: how to handle retries internally in Wolverine? -/// -public class RetryLetterTopic -{ - public static RetryLetterTopic DefaultNative => new([ - TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(5), TimeSpan.FromMinutes(2) - ]); - - - /// - /// Message delaying does not work with Pulsar if the subscription type is not shared or key shared - /// - public static IReadOnlySet SupportedSubscriptionTypes = new HashSet() - { - SubscriptionType.Shared, SubscriptionType.KeyShared - }; - - private string? _topicName; - private readonly List _retries; - - public RetryLetterTopic(List retries) - { - _retries = retries; - } - public RetryLetterTopic(string topicName, List retries) - { - _topicName = topicName; - _retries = retries; - } - - public string? TopicName - { - get => _topicName; - set => _topicName = value ?? throw new ArgumentNullException(nameof(TopicName)); - } - - public List Retry => _retries.ToList(); - - protected bool Equals(RetryLetterTopic other) - { - return _topicName == other._topicName; - } - - public override bool Equals(object? obj) - { - if (ReferenceEquals(null, obj)) - { - return false; - } - - if (ReferenceEquals(this, obj)) - { - return true; - } - - if (obj.GetType() != this.GetType()) - { - return false; - } - - return Equals((RetryLetterTopic)obj); - } - public override int GetHashCode() { return _topicName.GetHashCode(); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/RetryLetterTopic.cs b/src/Transports/Pulsar/Wolverine.Pulsar/RetryLetterTopic.cs new file mode 100644 index 000000000..c3000ea7f --- /dev/null +++ b/src/Transports/Pulsar/Wolverine.Pulsar/RetryLetterTopic.cs @@ -0,0 +1,73 @@ +using DotPulsar; + +namespace Wolverine.Pulsar; + +/// +/// TODO: how to handle retries internally in Wolverine? +/// +public class RetryLetterTopic +{ + public static RetryLetterTopic DefaultNative => new([ + TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(5), TimeSpan.FromMinutes(2) + ]); + + + /// + /// Message delaying does not work with Pulsar if the subscription type is not shared or key shared + /// + public static IReadOnlySet SupportedSubscriptionTypes = new HashSet() + { + SubscriptionType.Shared, SubscriptionType.KeyShared + }; + + private string? _topicName; + private readonly List _retries; + + public RetryLetterTopic(List retries) + { + _retries = retries; + } + public RetryLetterTopic(string topicName, List retries) + { + _topicName = topicName; + _retries = retries; + } + + public string? TopicName + { + get => _topicName; + set => _topicName = value ?? throw new ArgumentNullException(nameof(TopicName)); + } + + public List Retry => _retries.ToList(); + + protected bool Equals(RetryLetterTopic other) + { + return _topicName == other._topicName; + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) + { + return false; + } + + if (ReferenceEquals(this, obj)) + { + return true; + } + + if (obj.GetType() != this.GetType()) + { + return false; + } + + return Equals((RetryLetterTopic)obj); + } + + public override int GetHashCode() + { + return _topicName.GetHashCode(); + } +} \ No newline at end of file From 5619c31cc51474d241e7475fc3ed44f01ef1f9b9 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Sat, 29 Mar 2025 12:58:37 +0100 Subject: [PATCH 15/30] #1326 refactoring and some dirty fixes for envelope's Attempts sync with Pulsar RECONSUMETIMES header --- .../PulsarEnvelopeConstants.cs | 6 ++ .../Wolverine.Pulsar/PulsarEnvelopeMapper.cs | 20 +++++- .../Pulsar/Wolverine.Pulsar/PulsarListener.cs | 62 +++++-------------- .../ErrorHandling/MoveToErrorQueue.cs | 17 ++++- .../ErrorHandling/MoveToRetryQueueSource.cs | 10 +-- 5 files changed, 58 insertions(+), 57 deletions(-) create mode 100644 src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs new file mode 100644 index 000000000..6ca785822 --- /dev/null +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs @@ -0,0 +1,6 @@ +namespace Wolverine.Pulsar; + +public static class PulsarEnvelopeConstants +{ + public const string ReconsumeTimes = "RECONSUMETIMES"; +} \ No newline at end of file diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs index 13539ff4f..b27349750 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs @@ -21,11 +21,29 @@ protected override void writeOutgoingHeader(MessageMetadata outgoing, string key protected override bool tryReadIncomingHeader(IMessage> incoming, string key, out string? value) { + if (key == EnvelopeConstants.AttemptsKey && incoming.Properties.TryGetValue(PulsarEnvelopeConstants.ReconsumeTimes, out value)) + { + // dirty hack, handler increments Attempt field + int val = int.Parse(value); + val--; + value = val.ToString(); + return true; + } return incoming.Properties.TryGetValue(key, out value); } protected override void writeIncomingHeaders(IMessage> incoming, Envelope envelope) { - foreach (var pair in incoming.Properties) envelope.Headers[pair.Key] = pair.Value; + foreach (var pair in incoming.Properties) + { + envelope.Headers[pair.Key] = pair.Value; + + // doesn't work, it gets overwritten in next step - fix in tryReadIncomingHeader + //if (pair.Key == PulsarEnvelopeConstants.ReconsumeTimes) + //{ + // envelope.Headers[EnvelopeConstants.AttemptsKey] = pair.Value; + // envelope.Attempts = int.Parse(pair.Value); + //} + } } } \ No newline at end of file diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index 1932e277d..c2e162cad 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -47,9 +47,11 @@ public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IRecei // TODO: check NativeDeadLetterQueueEnabled = transport.DeadLetterTopic is not null && transport.DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage || - endpoint.DeadLetterTopic is not null && endpoint.DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage; + endpoint.DeadLetterTopic is not null && endpoint.DeadLetterTopic.Mode != + DeadLetterTopicMode.WolverineStorage; - NativeRetryLetterQueueEnabled = endpoint.RetryLetterTopic is not null && RetryLetterTopic.SupportedSubscriptionTypes.Contains(endpoint.SubscriptionType); + NativeRetryLetterQueueEnabled = endpoint.RetryLetterTopic is not null && + RetryLetterTopic.SupportedSubscriptionTypes.Contains(endpoint.SubscriptionType); trySetupNativeResiliency(endpoint, transport); @@ -62,25 +64,10 @@ public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IRecei { Data = message.Data.ToArray() }; - try - { - mapper.MapIncomingToEnvelope(envelope, message); - await receiver.ReceivedAsync(this, envelope); - } - catch (TaskCanceledException) - { - throw; - } - catch (Exception) - { - if (_dlqClient != null) - { - await _dlqClient.ReconsumeLater(message); - await receiver.ReceivedAsync(this, envelope); - //await _retryConsumer.Acknowledge(message); // TODO: check: original message should be acked and copy is sent to retry topic - } - } + mapper.MapIncomingToEnvelope(envelope, message); + + await receiver.ReceivedAsync(this, envelope); } @@ -98,27 +85,12 @@ public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IRecei { Data = message.Data.ToArray() }; - try - { - mapper.MapIncomingToEnvelope(envelope, message); - await receiver.ReceivedAsync(this, envelope); - } - catch (TaskCanceledException) - { - throw; - } - catch (Exception) - { - if (_dlqClient != null) - { - // TODO: used to manage retries - refactor - var retryCount = int.Parse(message.Properties["RECONSUMETIMES"]); - await _dlqClient.ReconsumeLater(message, delayTime: endpoint.RetryLetterTopic!.Retry[retryCount]); - await receiver.ReceivedAsync(this, envelope); - //await _retryConsumer.Acknowledge(message); // TODO: check: original message should be acked and copy is sent to retry/DLQ - } - } + mapper.MapIncomingToEnvelope(envelope, message); + + await receiver.ReceivedAsync(this, envelope); + + } }, combined.Token); @@ -254,7 +226,7 @@ public async Task TryRequeueAsync(Envelope envelope) public async Task MoveToErrorsAsync(Envelope envelope, Exception exception) { // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) - await moveToQueueAsync(envelope, exception, isRetry: false); + await moveToQueueAsync(envelope, exception); } public bool NativeRetryLetterQueueEnabled { get; } @@ -262,7 +234,7 @@ public bool RetryLimitReached(Envelope envelope) { if (NativeRetryLetterQueueEnabled && envelope is PulsarEnvelope e) { - if (e.MessageData.Properties.TryGetValue("RECONSUMETIMES", out var reconsumeTimesValue)) + if (e.MessageData.Properties.TryGetValue(PulsarEnvelopeConstants.ReconsumeTimes, out var reconsumeTimesValue)) { var currentRetryCount = int.Parse(reconsumeTimesValue); @@ -279,11 +251,11 @@ public async Task MoveToRetryQueueAsync(Envelope envelope, Exception exception) { // TODO: how to handle retries internally? // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) - await moveToQueueAsync(envelope, exception, isRetry: false); + await moveToQueueAsync(envelope, exception); } - private async Task moveToQueueAsync(Envelope envelope, Exception exception, bool isRetry) + private async Task moveToQueueAsync(Envelope envelope, Exception exception) { if (envelope is PulsarEnvelope e) { @@ -293,7 +265,7 @@ private async Task moveToQueueAsync(Envelope envelope, Exception exception, bool IConsumer>? associatedConsumer; TimeSpan? delayTime = null; - if (message.TryGetMessageProperty("RECONSUMETIMES", out var reconsumeTimesValue)) + if (message.TryGetMessageProperty(PulsarEnvelopeConstants.ReconsumeTimes, out var reconsumeTimesValue)) { associatedConsumer = _retryConsumer; var retryCount = int.Parse(reconsumeTimesValue); diff --git a/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs b/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs index a864de071..5da20a492 100644 --- a/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs +++ b/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs @@ -16,10 +16,23 @@ public IContinuation Build(Exception ex, Envelope envelope) } } +internal class MoveToErrorQueueWithoutFailureAckSource : IContinuationSource +{ + public string Description => "Move to error queue without failure ack"; + + public IContinuation Build(Exception ex, Envelope envelope) + { + return new MoveToErrorQueue(ex, true); + } +} + internal class MoveToErrorQueue : IContinuation { - public MoveToErrorQueue(Exception exception) + private readonly bool _forceSkipSendFailureAcknowledgment; + + public MoveToErrorQueue(Exception exception, bool forceSkipSendFailureAcknowledgment = false) { + _forceSkipSendFailureAcknowledgment = forceSkipSendFailureAcknowledgment; Exception = exception ?? throw new ArgumentNullException(nameof(exception)); } @@ -31,7 +44,7 @@ public async ValueTask ExecuteAsync(IEnvelopeLifecycle lifecycle, { // TODO -- at some point, we need a more systematic way of doing this var scheme = lifecycle.Envelope.Destination.Scheme; - if (runtime.Options.EnableAutomaticFailureAcks && scheme != TransportConstants.Local && scheme != "external-table") + if (!_forceSkipSendFailureAcknowledgment && runtime.Options.EnableAutomaticFailureAcks && scheme != TransportConstants.Local && scheme != "external-table") { await lifecycle.SendFailureAcknowledgementAsync( $"Moved message {lifecycle.Envelope!.Id} to the Error Queue.\n{Exception}"); diff --git a/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs b/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs index e949f6902..64aea74a3 100644 --- a/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs +++ b/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs @@ -29,14 +29,6 @@ public async ValueTask ExecuteAsync(IEnvelopeLifecycle lifecycle, IWolverineRuntime runtime, DateTimeOffset now, Activity? activity) { - //var scheme = lifecycle.Envelope.Destination.Scheme; - //if (runtime.Options.EnableAutomaticFailureAcks && scheme != TransportConstants.Local && scheme != "external-table") - //{ - // // TODO: remove or rather ack? - // await lifecycle.SendFailureAcknowledgementAsync( - // $"Moved message {lifecycle.Envelope!.Id} to the Retry Queue.\n{Exception}"); - //} - if (lifecycle.Envelope.Listener is ISupportRetryLetterQueue retryListener && !retryListener.RetryLimitReached(lifecycle.Envelope)) { @@ -62,7 +54,7 @@ await buildFallbackContinuation(lifecycle) private IContinuation buildFallbackContinuation(IEnvelopeLifecycle lifecycle) { - var cs = new MoveToErrorQueueSource(); + var cs = new MoveToErrorQueueWithoutFailureAckSource(); var continuation = cs.Build(Exception, lifecycle.Envelope); return continuation; } From 3d3382840bace2f85517c20c83152f5df535fb05 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Sun, 30 Mar 2025 12:37:48 +0200 Subject: [PATCH 16/30] #1326 refactoring --- .../Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs | 3 +-- .../Wolverine.Pulsar/PulsarTransportExtensions.cs | 13 ++++++++++--- src/Wolverine/ErrorHandling/FailureRule.cs | 11 +++++++++++ .../ErrorHandling/FailureRuleCollection.cs | 5 +++++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs index b27349750..7034ea24b 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs @@ -25,8 +25,7 @@ protected override bool tryReadIncomingHeader(IMessage> i { // dirty hack, handler increments Attempt field int val = int.Parse(value); - val--; - value = val.ToString(); + value = (--val).ToString(); return true; } return incoming.Properties.TryGetValue(key, out value); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs index e2d45b7e8..07faccab8 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs @@ -109,6 +109,9 @@ public PulsarListenerConfiguration SubscriptionType(SubscriptionType subscriptio e.SubscriptionType = subscriptionType; }); + if (subscriptionType is DotPulsar.SubscriptionType.Shared or DotPulsar.SubscriptionType.KeyShared) + new PulsarSharedListenerConfiguration(this._endpoint); + return this; } @@ -203,6 +206,8 @@ public PulsarSharedListenerConfiguration DisableDeadLetterQueueing() add(e => { e.DeadLetterTopic = null; + if (e.RetryLetterTopic is null && e.DeadLetterTopic is null) + e.Runtime.Options.Policies.Failures.Remove(rule => rule.Id == "PulsarNativeResiliency"); }); return this; @@ -221,8 +226,8 @@ public PulsarSharedListenerConfiguration RetryLetterQueueing(RetryLetterTopic rt // TODO: This is a bit of a hack to get the retry letter topic in place e.RetryLetterTopic = rt; - var exceptionMatch = new AlwaysMatches(); // currently can't determine if endpoint listener needs it just based on exception - var failureRule = new FailureRule(exceptionMatch); + var exceptionMatch = new AlwaysMatches(); // currently can't determine if endpoint listener needs it just based on exception, should handler that supports native resiliency, wrap the thrown exception into a new dedicated one? + var failureRule = new FailureRule(exceptionMatch, "PulsarNativeResiliency"); foreach (var _ in rt.Retry) { @@ -230,6 +235,7 @@ public PulsarSharedListenerConfiguration RetryLetterQueueing(RetryLetterTopic rt } e.Runtime.Options.Policies.Failures.Add(failureRule); + }); return this; @@ -244,7 +250,8 @@ public PulsarSharedListenerConfiguration DisableRetryLetterQueueing() add(e => { e.RetryLetterTopic = null; - //e.Runtime.Options.Policies.Failures // TODO: remove the failure rule + if (e.RetryLetterTopic is null && e.DeadLetterTopic is null) + e.Runtime.Options.Policies.Failures.Remove(rule => rule.Id == "PulsarNativeResiliency"); }); return this; diff --git a/src/Wolverine/ErrorHandling/FailureRule.cs b/src/Wolverine/ErrorHandling/FailureRule.cs index 2e7b01022..51b94a303 100644 --- a/src/Wolverine/ErrorHandling/FailureRule.cs +++ b/src/Wolverine/ErrorHandling/FailureRule.cs @@ -8,14 +8,21 @@ public class FailureRule : IEnumerable { private readonly List _slots = new(); + public FailureRule(IExceptionMatch match) { Match = match; } + public FailureRule(IExceptionMatch match, string id) + { + Match = match; + Id = id; + } public FailureSlot this[int attempt] => _slots[attempt - 1]; public IExceptionMatch Match { get; } + public string? Id { get; } internal IContinuationSource? InfiniteSource { get; set; } public IEnumerator GetEnumerator() @@ -55,4 +62,8 @@ public FailureSlot AddSlot(IContinuationSource source) return slot; } + + + + } \ No newline at end of file diff --git a/src/Wolverine/ErrorHandling/FailureRuleCollection.cs b/src/Wolverine/ErrorHandling/FailureRuleCollection.cs index 6c7019d92..54be76904 100644 --- a/src/Wolverine/ErrorHandling/FailureRuleCollection.cs +++ b/src/Wolverine/ErrorHandling/FailureRuleCollection.cs @@ -95,4 +95,9 @@ internal void Add(FailureRule rule) { _rules.Add(rule); } + + internal void Remove(Func rule) + { + _rules.RemoveAll(new Predicate(rule)); + } } \ No newline at end of file From 971d51e6462ac4376ccca45319083b408f85e0dc Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Sun, 30 Mar 2025 23:27:58 +0200 Subject: [PATCH 17/30] #1326 Pulsar support for native retry and DLQ queues in buffered endpoints, added first test for it --- .../PulsarNativeReliabilityTests.cs | 102 ++++++++++++++++++ src/Wolverine/Runtime/MessageContext.cs | 24 ++++- src/Wolverine/Tracking/EnvelopeHistory.cs | 1 + 3 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs new file mode 100644 index 000000000..ad3a07d30 --- /dev/null +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs @@ -0,0 +1,102 @@ +using JasperFx.Core; +using Microsoft.Extensions.Hosting; +using Oakton; +using Shouldly; +using Wolverine.ComplianceTests; +using Wolverine.ComplianceTests.Compliance; +using Wolverine.ComplianceTests.Scheduling; +using Wolverine.Tracking; +using Xunit; + +namespace Wolverine.Pulsar.Tests; + +public class PulsarNativeReliabilityTests : /*TransportComplianceFixture,*/ IAsyncLifetime +{ + public IHost WolverineHost; + + public PulsarNativeReliabilityTests() + { + } + + private IHostBuilder ConfigureBuilder() + { + return Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + + var topic = Guid.NewGuid().ToString(); + var topicPath = $"persistent://public/default/compliance{topic}"; + + opts.UsePulsar(b => { }); + + opts.IncludeType(); + + opts.PublishMessage() + .ToPulsarTopic(topicPath); + + opts.ListenToPulsarTopic(topicPath) + .WithSharedSubscriptionType() + .RetryLetterQueueing(new RetryLetterTopic([TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2)])) + .DeadLetterQueueing(DeadLetterTopic.DefaultNative) + .ProcessInline(); + //.BufferedInMemory(); + + + }); + } + + public async Task InitializeAsync() + { + WolverineHost = ConfigureBuilder().Build(); + await WolverineHost.StartAsync(); + } + + [Fact] + public async Task run_setup_with_simulated_exception_in_handler() + { + var session = await WolverineHost.TrackActivity(TimeSpan.FromSeconds(10)) + .WaitForMessageToBeReceivedAt(WolverineHost) + .DoNotAssertOnExceptionsDetected() + .IncludeExternalTransports() + .SendMessageAndWaitAsync(new SRMessage1()); + + + session.Sent.AllMessages(); + session.MovedToErrorQueue + .MessagesOf() + .Count() + .ShouldBe(1); + + session.Received + .MessagesOf() + .Count() + .ShouldBe(3); + + session.Requeued + .MessagesOf() + .Count() + .ShouldBe(2); + } + + + + public async Task DisposeAsync() + { + await WolverineHost.StopAsync(); + WolverineHost.Dispose(); + } + + +} + +public class SRMessage1; + + +public class SRMessageHandlers +{ + public Task Handle(SRMessage1 message) + { + throw new InvalidOperationException("Simulated exception"); + } + +} diff --git a/src/Wolverine/Runtime/MessageContext.cs b/src/Wolverine/Runtime/MessageContext.cs index 35c5f591e..e1993a244 100644 --- a/src/Wolverine/Runtime/MessageContext.cs +++ b/src/Wolverine/Runtime/MessageContext.cs @@ -142,20 +142,38 @@ public async Task MoveToRetryLetterQueueAsync(Exception exception) throw new InvalidOperationException("No Envelope is active for this context"); } - if (_channel is ISupportRetryLetterQueue c && c.NativeRetryLetterQueueEnabled) + var retryLetterQueue = tryGetRetryLetterQueue(_channel, Envelope); + if (retryLetterQueue is not null) { if (Envelope.Batch != null) { foreach (var envelope in Envelope.Batch) { - await c.MoveToRetryQueueAsync(envelope, exception); + await retryLetterQueue.MoveToRetryQueueAsync(envelope, exception); } } else { - await c.MoveToRetryQueueAsync(Envelope, exception); + await retryLetterQueue.MoveToRetryQueueAsync(Envelope, exception); } } + + + } + + private ISupportRetryLetterQueue? tryGetRetryLetterQueue(IChannelCallback? channel, Envelope e) + { + if (_channel is ISupportRetryLetterQueue { NativeRetryLetterQueueEnabled: true } c) + { + return c; + } + + if (Envelope.Listener is ISupportRetryLetterQueue { NativeRetryLetterQueueEnabled: true } c2) + { + return c2; + } + + return default; } public async Task MoveToDeadLetterQueueAsync(Exception exception) diff --git a/src/Wolverine/Tracking/EnvelopeHistory.cs b/src/Wolverine/Tracking/EnvelopeHistory.cs index ae65a18f1..78848b334 100644 --- a/src/Wolverine/Tracking/EnvelopeHistory.cs +++ b/src/Wolverine/Tracking/EnvelopeHistory.cs @@ -99,6 +99,7 @@ public void RecordLocally(EnvelopeRecord record) break; case MessageEventType.Requeued: + case MessageEventType.MovedToRetryQueue: // Do nothing, just informative break; From 8373a44cd864331fc5e8d798fa055f8fff8ed869 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Mon, 31 Mar 2025 14:34:47 +0200 Subject: [PATCH 18/30] #1326 bug fixes - better unit test asserts and proper condition waiting for DLQ envelope --- .../PulsarNativeReliabilityTests.cs | 66 +++++++++++++++++-- .../PulsarEnvelopeConstants.cs | 1 + .../Pulsar/Wolverine.Pulsar/PulsarListener.cs | 41 +++--------- src/Wolverine/Runtime/MessageContext.cs | 24 +++++-- src/Wolverine/Tracking/EnvelopeHistory.cs | 1 + src/Wolverine/Tracking/ITrackedSession.cs | 5 ++ src/Wolverine/Tracking/TrackedSession.cs | 1 + 7 files changed, 97 insertions(+), 42 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs index ad3a07d30..2481b5d75 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs @@ -38,8 +38,12 @@ private IHostBuilder ConfigureBuilder() .WithSharedSubscriptionType() .RetryLetterQueueing(new RetryLetterTopic([TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2)])) .DeadLetterQueueing(DeadLetterTopic.DefaultNative) - .ProcessInline(); - //.BufferedInMemory(); + //.ProcessInline(); + .BufferedInMemory(); + + //opts.ListenToPulsarTopic(topicPath + "-DLQ") + // .WithSharedSubscriptionType() + // .ProcessInline(); }); @@ -54,10 +58,11 @@ public async Task InitializeAsync() [Fact] public async Task run_setup_with_simulated_exception_in_handler() { - var session = await WolverineHost.TrackActivity(TimeSpan.FromSeconds(10)) - .WaitForMessageToBeReceivedAt(WolverineHost) + var session = await WolverineHost.TrackActivity(TimeSpan.FromSeconds(100)) + //.WaitForMessageToBeReceivedAt(WolverineHost) .DoNotAssertOnExceptionsDetected() .IncludeExternalTransports() + .WaitForCondition(new WaitForDeadLetteredMessage()) .SendMessageAndWaitAsync(new SRMessage1()); @@ -76,6 +81,32 @@ public async Task run_setup_with_simulated_exception_in_handler() .MessagesOf() .Count() .ShouldBe(2); + + session.MovedToRetryQueue + .MessagesOf() + .Count() + .ShouldBe(2); + + // TODO: I Guess the capture of the envelope headers occurs before we manipulate it + //var firstRequeuedEnvelope = session.MovedToRetryQueue.Envelopes().First(); + //firstRequeuedEnvelope.ShouldSatisfyAllConditions( + // () => firstRequeuedEnvelope.Headers.ContainsKey("DELAY_TIME").ShouldBeTrue(), + // () => firstRequeuedEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(1).TotalMilliseconds.ToString()) + //); + //var secondRequeuedEnvelope = session.MovedToRetryQueue.Envelopes().Skip(1).First(); + //secondRequeuedEnvelope.ShouldSatisfyAllConditions( + // () => secondRequeuedEnvelope.Headers.ContainsKey("DELAY_TIME").ShouldBeTrue(), + // () => secondRequeuedEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(2).TotalMilliseconds.ToString()) + //); + + + var firstEnvelope = session.MovedToErrorQueue.Envelopes().First(); + firstEnvelope.ShouldSatisfyAllConditions( + () => firstEnvelope.Headers.ContainsKey(PulsarEnvelopeConstants.Exception).ShouldBeTrue(), + () => firstEnvelope.Headers[PulsarEnvelopeConstants.ReconsumeTimes].ShouldBe("2"), + () => firstEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(2).TotalMilliseconds.ToString()) + ); + } @@ -94,9 +125,34 @@ public class SRMessage1; public class SRMessageHandlers { - public Task Handle(SRMessage1 message) + public Task Handle(SRMessage1 message, IMessageContext context) { throw new InvalidOperationException("Simulated exception"); } } + + + +public class WaitForDeadLetteredMessage : ITrackedCondition +{ + + private bool _found; + + public WaitForDeadLetteredMessage() + { + + } + + public void Record(EnvelopeRecord record) + { + if (record.Envelope.Message is T && record.MessageEventType == MessageEventType.MovedToErrorQueue ) + // && record.Envelope.Destination?.ToString().Contains(_dlqTopic) == true) + { + _found = true; + } + } + + public bool IsCompleted() => _found; +} + diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs index 6ca785822..32372642e 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs @@ -3,4 +3,5 @@ namespace Wolverine.Pulsar; public static class PulsarEnvelopeConstants { public const string ReconsumeTimes = "RECONSUMETIMES"; + public const string Exception = "EXCEPTION"; } \ No newline at end of file diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index c2e162cad..f845d39cd 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -226,7 +226,7 @@ public async Task TryRequeueAsync(Envelope envelope) public async Task MoveToErrorsAsync(Envelope envelope, Exception exception) { // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) - await moveToQueueAsync(envelope, exception); + await moveToQueueAsync(envelope, exception, true); } public bool NativeRetryLetterQueueEnabled { get; } @@ -255,7 +255,7 @@ public async Task MoveToRetryQueueAsync(Envelope envelope, Exception exception) } - private async Task moveToQueueAsync(Envelope envelope, Exception exception) + private async Task moveToQueueAsync(Envelope envelope, Exception exception, bool isDeadLettered = false) { if (envelope is PulsarEnvelope e) { @@ -269,13 +269,18 @@ private async Task moveToQueueAsync(Envelope envelope, Exception exception) { associatedConsumer = _retryConsumer; var retryCount = int.Parse(reconsumeTimesValue); - delayTime = _endpoint.RetryLetterTopic!.Retry[retryCount - 1]; + delayTime = !isDeadLettered ? _endpoint.RetryLetterTopic!.Retry[retryCount] : null; } else { associatedConsumer = _consumer; } + if (isDeadLettered) + { + e.Headers[PulsarEnvelopeConstants.Exception] = exception.ToString(); + } + await associatedConsumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection. If I understand correctly, Wolverine is/should handle original Wolverine message and its copies across Pulsar's topics as same identity? // TODO: e.Attempts / attempts header value is out of sync with Pulsar's RECONSUMETIMES header! @@ -284,36 +289,6 @@ private async Task moveToQueueAsync(Envelope envelope, Exception exception) } } - //public async Task MoveToRetryQueueAsync(Envelope envelope, Exception exception) - //{ - // // TODO: how to handle retries internally? - // // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) - - // if (envelope is PulsarEnvelope e) - // { - // if (_dlqClient != null) - // { - // var message = e.MessageData; - // // TODO: used to manage retries - refactor - // if (message.Properties.TryGetValue("RECONSUMETIMES", out var reconsumeTimesValue)) - // { - // var retryCount = int.Parse(reconsumeTimesValue); - // await _retryConsumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ - // //await _retryConsumer.Acknowledge(message); // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection. If I understand correctly, Wolverine is/should handle original Wolverine message and its copies across Pulsar's topics as same identity? - // //TODO: e.Attempts / attempts header value is out of sync with Pulsar's RECONSUMETIMES header! - // await _dlqClient.ReconsumeLater(message, delayTime: _endpoint.RetryLetterTopic!.Retry[retryCount - 1], cancellationToken: _cancellation); - // } - // else - // { - // // first time failure or no retry letter topic configured - // await _consumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ - // //await _retryConsumer.Acknowledge(message); // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? - // await _dlqClient.ReconsumeLater(message, delayTime: _endpoint.RetryLetterTopic!.Retry.First(), cancellationToken: _cancellation); - // } - // } - - // } - //} } diff --git a/src/Wolverine/Runtime/MessageContext.cs b/src/Wolverine/Runtime/MessageContext.cs index e1993a244..0a577735c 100644 --- a/src/Wolverine/Runtime/MessageContext.cs +++ b/src/Wolverine/Runtime/MessageContext.cs @@ -168,7 +168,22 @@ public async Task MoveToRetryLetterQueueAsync(Exception exception) return c; } - if (Envelope.Listener is ISupportRetryLetterQueue { NativeRetryLetterQueueEnabled: true } c2) + if (e.Listener is ISupportRetryLetterQueue { NativeRetryLetterQueueEnabled: true } c2) + { + return c2; + } + + return default; + } + + private ISupportDeadLetterQueue? tryGetDeadLetterQueue(IChannelCallback? channel, Envelope e) + { + if (_channel is ISupportDeadLetterQueue { NativeDeadLetterQueueEnabled: true } c) + { + return c; + } + + if (e.Listener is ISupportDeadLetterQueue { NativeDeadLetterQueueEnabled: true } c2) { return c2; } @@ -186,18 +201,19 @@ public async Task MoveToDeadLetterQueueAsync(Exception exception) throw new InvalidOperationException("No Envelope is active for this context"); } - if (_channel is ISupportDeadLetterQueue c && c.NativeDeadLetterQueueEnabled) + var deadLetterQueue = tryGetDeadLetterQueue(_channel, Envelope); + if (deadLetterQueue is not null) { if (Envelope.Batch != null) { foreach (var envelope in Envelope.Batch) { - await c.MoveToErrorsAsync(envelope, exception); + await deadLetterQueue.MoveToErrorsAsync(envelope, exception); } } else { - await c.MoveToErrorsAsync(Envelope, exception); + await deadLetterQueue.MoveToErrorsAsync(Envelope, exception); } return; diff --git a/src/Wolverine/Tracking/EnvelopeHistory.cs b/src/Wolverine/Tracking/EnvelopeHistory.cs index 78848b334..c8e39a002 100644 --- a/src/Wolverine/Tracking/EnvelopeHistory.cs +++ b/src/Wolverine/Tracking/EnvelopeHistory.cs @@ -156,6 +156,7 @@ public void RecordCrossApplication(EnvelopeRecord record) case MessageEventType.NoHandlers: case MessageEventType.NoRoutes: case MessageEventType.Requeued: + case MessageEventType.MovedToRetryQueue: break; diff --git a/src/Wolverine/Tracking/ITrackedSession.cs b/src/Wolverine/Tracking/ITrackedSession.cs index 0b7b6cf1e..5336de525 100644 --- a/src/Wolverine/Tracking/ITrackedSession.cs +++ b/src/Wolverine/Tracking/ITrackedSession.cs @@ -55,6 +55,11 @@ public interface ITrackedSession /// RecordCollection MovedToErrorQueue { get; } + /// + /// Records of all messages that were moved to the error queue + /// + RecordCollection MovedToRetryQueue { get; } + /// /// Records of all messages that were requeued /// diff --git a/src/Wolverine/Tracking/TrackedSession.cs b/src/Wolverine/Tracking/TrackedSession.cs index 7611cd1a6..b832d75ba 100644 --- a/src/Wolverine/Tracking/TrackedSession.cs +++ b/src/Wolverine/Tracking/TrackedSession.cs @@ -157,6 +157,7 @@ public void AssertCondition(string message, Func condition) public RecordCollection NoHandlers => new(MessageEventType.NoHandlers, this); public RecordCollection NoRoutes => new(MessageEventType.NoRoutes, this); public RecordCollection MovedToErrorQueue => new(MessageEventType.MovedToErrorQueue, this); + public RecordCollection MovedToRetryQueue => new(MessageEventType.MovedToRetryQueue, this); public RecordCollection Requeued => new(MessageEventType.Requeued, this); public RecordCollection Executed => new(MessageEventType.ExecutionFinished, this); From a6e8204f6ce72f61dacf21d5ae78212d64da91a7 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Mon, 31 Mar 2025 15:03:49 +0200 Subject: [PATCH 19/30] #1326 bug fixes - added unit test for only DLQ (without retry queue flow) --- .../PulsarNativeReliabilityTests.cs | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs index 2481b5d75..9c6240e5f 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs @@ -46,6 +46,17 @@ private IHostBuilder ConfigureBuilder() // .ProcessInline(); + var topicPath2 = $"persistent://public/default/no-retry-{topic}"; + opts.IncludeType(); + + opts.PublishMessage() + .ToPulsarTopic(topicPath2); + + opts.ListenToPulsarTopic(topicPath2) + .WithSharedSubscriptionType() + .DeadLetterQueueing(DeadLetterTopic.DefaultNative) + .ProcessInline(); + }); } @@ -109,7 +120,48 @@ public async Task run_setup_with_simulated_exception_in_handler() } - + [Fact] + public async Task run_setup_with_simulated_exception_in_handler_only_native_dead_lettered_queue() + { + var session = await WolverineHost.TrackActivity(TimeSpan.FromSeconds(100)) + .DoNotAssertOnExceptionsDetected() + .IncludeExternalTransports() + .WaitForCondition(new WaitForDeadLetteredMessage()) + .SendMessageAndWaitAsync(new SRMessage2()); + + + session.Sent.AllMessages(); + session.MovedToErrorQueue + .MessagesOf() + .Count() + .ShouldBe(1); + + session.Received + .MessagesOf() + .Count() + .ShouldBe(1); + + session.Requeued + .MessagesOf() + .Count() + .ShouldBe(0); + + session.MovedToRetryQueue + .MessagesOf() + .Count() + .ShouldBe(0); + + + + var firstEnvelope = session.MovedToErrorQueue.Envelopes().First(); + firstEnvelope.ShouldSatisfyAllConditions( + () => firstEnvelope.Headers.ContainsKey(PulsarEnvelopeConstants.Exception).ShouldBeTrue(), + () => firstEnvelope.Headers.ContainsKey(PulsarEnvelopeConstants.ReconsumeTimes).ShouldBeFalse() + ); + + } + + public async Task DisposeAsync() { @@ -121,6 +173,7 @@ public async Task DisposeAsync() } public class SRMessage1; +public class SRMessage2; public class SRMessageHandlers @@ -130,6 +183,11 @@ public Task Handle(SRMessage1 message, IMessageContext context) throw new InvalidOperationException("Simulated exception"); } + public Task Handle(SRMessage2 message, IMessageContext context) + { + throw new InvalidOperationException("Simulated exception"); + } + } From 844c8b2af621556b23fc08fb657e902319610604 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Tue, 1 Apr 2025 11:30:24 +0200 Subject: [PATCH 20/30] #1326 refactoring - as Jeremy suggested, I removed the action MoveToRetryLetterQueueAsync from the lifecycle --- .../Pulsar/Wolverine.Pulsar/PulsarListener.cs | 2 +- .../PulsarTransportExtensions.cs | 79 ++++++++++++++++--- src/Wolverine/ErrorHandling/FailureRule.cs | 2 +- .../ErrorHandling/MoveToRetryQueueSource.cs | 5 +- .../ErrorHandling/PolicyExpression.cs | 30 ++++++- src/Wolverine/IEnvelopeLifecycle.cs | 2 - src/Wolverine/Runtime/MessageContext.cs | 42 ---------- src/Wolverine/Tracking/ITrackedSession.cs | 2 +- 8 files changed, 99 insertions(+), 65 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index f845d39cd..8369884ce 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -249,7 +249,7 @@ public bool RetryLimitReached(Envelope envelope) public async Task MoveToRetryQueueAsync(Envelope envelope, Exception exception) { - // TODO: how to handle retries internally? + // TODO: how to handle retries internally (in Wolverine context, not Pulsar)? // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) await moveToQueueAsync(envelope, exception); } diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs index 07faccab8..92cf5e5d3 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs @@ -116,7 +116,31 @@ public PulsarListenerConfiguration SubscriptionType(SubscriptionType subscriptio } /// - /// Override the Pulsar subscription type for just this topic + /// Override the Pulsar subscription type to for just this topic + /// + /// + /// + public PulsarListenerConfiguration WithFailoverSubscriptionType() + { + add(e => { e.SubscriptionType = DotPulsar.SubscriptionType.Failover; }); + + return this; + } + + /// + /// Override the Pulsar subscription type to for just this topic + /// + /// + /// + public PulsarListenerConfiguration WithExclusiveSubscriptionType() + { + add(e => { e.SubscriptionType = DotPulsar.SubscriptionType.Exclusive; }); + + return this; + } + + /// + /// Override the Pulsar subscription type to for just this topic /// /// /// @@ -129,7 +153,7 @@ public PulsarSharedListenerConfiguration WithSharedSubscriptionType() /// - /// Override the Pulsar subscription type for just this topic + /// Override the Pulsar subscription type to for just this topic /// /// /// @@ -157,6 +181,36 @@ public PulsarListenerConfiguration CircuitBreaker(Action? return this; } + + /// + /// Customize the dead letter queueing for this specific endpoint + /// + /// Optional configuration + /// + public PulsarListenerConfiguration DeadLetterQueueing(DeadLetterTopic dlq) + { + add(e => + { + e.DeadLetterTopic = dlq; + }); + + return this; + } + + /// + /// Remove all dead letter queueing declarations from this queue + /// + /// + public PulsarListenerConfiguration DisableDeadLetterQueueing() + { + add(e => + { + e.DeadLetterTopic = null; + }); + + return this; + } + // /// // /// To optimize the message listener throughput, // /// start up multiple listening endpoints. This is @@ -181,7 +235,6 @@ public PulsarSharedListenerConfiguration(PulsarEndpoint endpoint) : base(endpoin { } - /// /// Customize the dead letter queueing for this specific endpoint /// @@ -222,19 +275,19 @@ public PulsarSharedListenerConfiguration RetryLetterQueueing(RetryLetterTopic rt { add(e => { - - // TODO: This is a bit of a hack to get the retry letter topic in place + e.RetryLetterTopic = rt; - var exceptionMatch = new AlwaysMatches(); // currently can't determine if endpoint listener needs it just based on exception, should handler that supports native resiliency, wrap the thrown exception into a new dedicated one? - var failureRule = new FailureRule(exceptionMatch, "PulsarNativeResiliency"); - - foreach (var _ in rt.Retry) - { - failureRule.AddSlot(new MoveToRetryQueueSource()); - } + //var exceptionMatch = new AlwaysMatches(); // currently can't determine if endpoint listener needs it just based on exception, should handler that supports native resiliency, wrap the thrown exception into a new dedicated one? + //var failureRule = new FailureRule(exceptionMatch, "PulsarNativeResiliency"); + //foreach (var _ in rt.Retry) + //{ + // failureRule.AddSlot(new MoveToRetryQueueSource()); + //} + //e.Runtime.Options.Policies.Failures.Add(failureRule); - e.Runtime.Options.Policies.Failures.Add(failureRule); + //e.Runtime.Options.Policies.OnAnyException().MoveToErrorQueue(); + e.Runtime.Options.Policies.OnAnyException().MoveToRetryQueue(rt.Retry.Count, "PulsarNativeResiliency"); }); diff --git a/src/Wolverine/ErrorHandling/FailureRule.cs b/src/Wolverine/ErrorHandling/FailureRule.cs index 51b94a303..87ecff400 100644 --- a/src/Wolverine/ErrorHandling/FailureRule.cs +++ b/src/Wolverine/ErrorHandling/FailureRule.cs @@ -13,7 +13,7 @@ public FailureRule(IExceptionMatch match) { Match = match; } - public FailureRule(IExceptionMatch match, string id) + public FailureRule(IExceptionMatch match, string? id) { Match = match; Id = id; diff --git a/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs b/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs index 64aea74a3..b447fc12e 100644 --- a/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs +++ b/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs @@ -37,11 +37,10 @@ public async ValueTask ExecuteAsync(IEnvelopeLifecycle lifecycle, lifecycle.Envelope.MessageType = lifecycle.Envelope.Message.GetType().ToMessageTypeName(); } - await lifecycle.MoveToRetryLetterQueueAsync(Exception); - + await retryListener.MoveToRetryQueueAsync(lifecycle.Envelope, Exception); activity?.AddEvent(new ActivityEvent(WolverineTracing.MovedToRetryQueue)); - runtime.MessageTracking.Requeued(lifecycle.Envelope); // TODO: new method + runtime.MessageTracking.Requeued(lifecycle.Envelope); // TODO: new method below or this? runtime.MessageTracking.MovedToRetryQueue(lifecycle.Envelope, Exception); } diff --git a/src/Wolverine/ErrorHandling/PolicyExpression.cs b/src/Wolverine/ErrorHandling/PolicyExpression.cs index 3c175fa21..097c1898a 100644 --- a/src/Wolverine/ErrorHandling/PolicyExpression.cs +++ b/src/Wolverine/ErrorHandling/PolicyExpression.cs @@ -71,9 +71,9 @@ internal class FailureActions : IAdditionalActions, IFailureActions private readonly FailureRule _rule; private readonly List _slots = new(); - public FailureActions(IExceptionMatch match, FailureRuleCollection parent) + public FailureActions(IExceptionMatch match, FailureRuleCollection parent, string? ruleId = null) { - _rule = new FailureRule(match); + _rule = new FailureRule(match, ruleId); parent.Add(_rule); } @@ -128,6 +128,17 @@ public IAdditionalActions MoveToErrorQueue() return this; } + public IAdditionalActions MoveToRetryQueue(int maxAttempts, string? ruleId = null) + { + for (var i = 0; i < maxAttempts - 1; i++) + { + var slot = _rule.AddSlot(new MoveToRetryQueueSource()); + _slots.Add(slot); + } + + return this; + } + public IAdditionalActions Requeue(int maxAttempts = 3) { if (maxAttempts > 25) @@ -297,6 +308,12 @@ public interface IFailureActions /// IAdditionalActions MoveToErrorQueue(); + /// + /// Immediately move the message to the retry queue when the exception + /// caught matches this criteria + /// + IAdditionalActions MoveToRetryQueue(int maxAttempts, string? ruleId = null); + /// /// Requeue the message back to the incoming transport, with the message being /// dead lettered when the maximum number of attempts is reached @@ -386,6 +403,15 @@ public IAdditionalActions MoveToErrorQueue() return new FailureActions(_match, _parent).MoveToErrorQueue(); } + /// + /// Immediately move the message to the retry queue when the exception + /// caught matches this criteria + /// + public IAdditionalActions MoveToRetryQueue(int maxAttempts, string? ruleId = null) + { + return new FailureActions(_match, _parent, ruleId).MoveToRetryQueue(maxAttempts); + } + /// /// Requeue the message back to the incoming transport, with the message being /// dead lettered when the maximum number of attempts is reached diff --git a/src/Wolverine/IEnvelopeLifecycle.cs b/src/Wolverine/IEnvelopeLifecycle.cs index ed42c1774..706f7d33a 100644 --- a/src/Wolverine/IEnvelopeLifecycle.cs +++ b/src/Wolverine/IEnvelopeLifecycle.cs @@ -24,8 +24,6 @@ public interface IEnvelopeLifecycle : IMessageBus Task ReScheduleAsync(DateTimeOffset scheduledTime); - Task MoveToRetryLetterQueueAsync(Exception exception); - Task MoveToDeadLetterQueueAsync(Exception exception); /// diff --git a/src/Wolverine/Runtime/MessageContext.cs b/src/Wolverine/Runtime/MessageContext.cs index 0a577735c..d97ef9dae 100644 --- a/src/Wolverine/Runtime/MessageContext.cs +++ b/src/Wolverine/Runtime/MessageContext.cs @@ -134,48 +134,6 @@ public async Task ReScheduleAsync(DateTimeOffset scheduledTime) await Storage.Inbox.ScheduleJobAsync(Envelope); } } - - public async Task MoveToRetryLetterQueueAsync(Exception exception) - { - if (_channel == null || Envelope == null) - { - throw new InvalidOperationException("No Envelope is active for this context"); - } - - var retryLetterQueue = tryGetRetryLetterQueue(_channel, Envelope); - if (retryLetterQueue is not null) - { - if (Envelope.Batch != null) - { - foreach (var envelope in Envelope.Batch) - { - await retryLetterQueue.MoveToRetryQueueAsync(envelope, exception); - } - } - else - { - await retryLetterQueue.MoveToRetryQueueAsync(Envelope, exception); - } - } - - - } - - private ISupportRetryLetterQueue? tryGetRetryLetterQueue(IChannelCallback? channel, Envelope e) - { - if (_channel is ISupportRetryLetterQueue { NativeRetryLetterQueueEnabled: true } c) - { - return c; - } - - if (e.Listener is ISupportRetryLetterQueue { NativeRetryLetterQueueEnabled: true } c2) - { - return c2; - } - - return default; - } - private ISupportDeadLetterQueue? tryGetDeadLetterQueue(IChannelCallback? channel, Envelope e) { if (_channel is ISupportDeadLetterQueue { NativeDeadLetterQueueEnabled: true } c) diff --git a/src/Wolverine/Tracking/ITrackedSession.cs b/src/Wolverine/Tracking/ITrackedSession.cs index 5336de525..b279cad98 100644 --- a/src/Wolverine/Tracking/ITrackedSession.cs +++ b/src/Wolverine/Tracking/ITrackedSession.cs @@ -56,7 +56,7 @@ public interface ITrackedSession RecordCollection MovedToErrorQueue { get; } /// - /// Records of all messages that were moved to the error queue + /// Records of all messages that were moved to the retry queue /// RecordCollection MovedToRetryQueue { get; } From b50c751f2d1e0d763cf4afaf3fbcffef4ca0a795 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Mon, 14 Apr 2025 11:56:42 +0200 Subject: [PATCH 21/30] #1326 refactoring - substituted ISupportRetryLetterQueue (to be deleted in next commit) with ISupportNativeScheduling for Pulsar's native retry letter functionality. --- .../PulsarNativeReliabilityTests.cs | 2 +- .../Wolverine.Pulsar/PulsarEnvelopeMapper.cs | 3 +- .../Pulsar/Wolverine.Pulsar/PulsarListener.cs | 15 +- .../PulsarTransportExtensions.cs | 177 +++++++++++------- src/Wolverine/Runtime/MessageContext.cs | 18 +- 5 files changed, 146 insertions(+), 69 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs index 9c6240e5f..803cff60b 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs @@ -36,8 +36,8 @@ private IHostBuilder ConfigureBuilder() opts.ListenToPulsarTopic(topicPath) .WithSharedSubscriptionType() - .RetryLetterQueueing(new RetryLetterTopic([TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2)])) .DeadLetterQueueing(DeadLetterTopic.DefaultNative) + .RetryLetterQueueing(new RetryLetterTopic([TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2)])) //.ProcessInline(); .BufferedInMemory(); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs index 7034ea24b..504c67ebc 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs @@ -25,7 +25,8 @@ protected override bool tryReadIncomingHeader(IMessage> i { // dirty hack, handler increments Attempt field int val = int.Parse(value); - value = (--val).ToString(); + value = (val).ToString(); + //value = (--val).ToString(); return true; } return incoming.Properties.TryGetValue(key, out value); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index 8369884ce..06730f1cd 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -8,7 +8,7 @@ namespace Wolverine.Pulsar; -internal class PulsarListener : IListener, ISupportDeadLetterQueue, ISupportRetryLetterQueue +internal class PulsarListener : IListener, ISupportDeadLetterQueue, ISupportRetryLetterQueue, ISupportNativeScheduling { private readonly CancellationToken _cancellation; private readonly IConsumer>? _consumer; @@ -254,6 +254,15 @@ public async Task MoveToRetryQueueAsync(Envelope envelope, Exception exception) await moveToQueueAsync(envelope, exception); } + public async Task MoveToScheduledUntilAsync(Envelope envelope, DateTimeOffset time) + { + if (NativeRetryLetterQueueEnabled && envelope is PulsarEnvelope e) + { + await moveToQueueAsync(e, e.Failure, false); + + } + } + private async Task moveToQueueAsync(Envelope envelope, Exception exception, bool isDeadLettered = false) { @@ -284,11 +293,15 @@ private async Task moveToQueueAsync(Envelope envelope, Exception exception, bool await associatedConsumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection. If I understand correctly, Wolverine is/should handle original Wolverine message and its copies across Pulsar's topics as same identity? // TODO: e.Attempts / attempts header value is out of sync with Pulsar's RECONSUMETIMES header! + + if (delayTime is null && _endpoint.RetryLetterTopic is not null) + delayTime = !isDeadLettered ? _endpoint.RetryLetterTopic!.Retry.First() : null; await _dlqClient.ReconsumeLater(message, delayTime: delayTime, cancellationToken: _cancellation); } } } + } diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs index 92cf5e5d3..747972d9d 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs @@ -74,6 +74,7 @@ public static PulsarListenerConfiguration ListenToPulsarTopic(this WolverineOpti endpoint.IsListener = true; return new PulsarListenerConfiguration(endpoint); } + } public class PulsarListenerConfiguration : ListenerConfiguration @@ -109,8 +110,9 @@ public PulsarListenerConfiguration SubscriptionType(SubscriptionType subscriptio e.SubscriptionType = subscriptionType; }); - if (subscriptionType is DotPulsar.SubscriptionType.Shared or DotPulsar.SubscriptionType.KeyShared) - new PulsarSharedListenerConfiguration(this._endpoint); + // TODO: check how to restrict it properly + //if (subscriptionType is DotPulsar.SubscriptionType.Shared or DotPulsar.SubscriptionType.KeyShared) + // return new PulsarSharedListenerConfiguration(this._endpoint); return this; } @@ -144,11 +146,11 @@ public PulsarListenerConfiguration WithExclusiveSubscriptionType() /// /// /// - public PulsarSharedListenerConfiguration WithSharedSubscriptionType() + public PulsarNativeResiliencyDeadLetterConfiguration WithSharedSubscriptionType() { add(e => { e.SubscriptionType = DotPulsar.SubscriptionType.Shared; }); - return new PulsarSharedListenerConfiguration(this._endpoint); + return new PulsarNativeResiliencyDeadLetterConfiguration(new PulsarListenerConfiguration(_endpoint)); } @@ -157,11 +159,11 @@ public PulsarSharedListenerConfiguration WithSharedSubscriptionType() /// /// /// - public PulsarSharedListenerConfiguration WithKeySharedSubscriptionType() + public PulsarNativeResiliencyDeadLetterConfiguration WithKeySharedSubscriptionType() { add(e => { e.SubscriptionType = DotPulsar.SubscriptionType.KeyShared; }); - return new PulsarSharedListenerConfiguration(this._endpoint); + return new PulsarNativeResiliencyDeadLetterConfiguration(new PulsarListenerConfiguration(_endpoint)); } /// @@ -192,23 +194,15 @@ public PulsarListenerConfiguration DeadLetterQueueing(DeadLetterTopic dlq) add(e => { e.DeadLetterTopic = dlq; + e.Runtime.Options.Policies.OnAnyException().MoveToErrorQueue(); }); return this; } - /// - /// Remove all dead letter queueing declarations from this queue - /// - /// - public PulsarListenerConfiguration DisableDeadLetterQueueing() + internal void Apply(Action action) { - add(e => - { - e.DeadLetterTopic = null; - }); - - return this; + add(action); } // /// @@ -229,87 +223,140 @@ public PulsarListenerConfiguration DisableDeadLetterQueueing() -public class PulsarSharedListenerConfiguration : ListenerConfiguration + +public class PulsarNativeResiliencyConfig { - public PulsarSharedListenerConfiguration(PulsarEndpoint endpoint) : base(endpoint) + public DeadLetterTopic DeadLetterTopic { get; set; } + public RetryLetterTopic? RetryLetterTopic { get; set; } + + public Action Apply() { + return endpoint => + { + + if (RetryLetterTopic is null && DeadLetterTopic is null) + { + endpoint.DeadLetterTopic = null; + endpoint.RetryLetterTopic = null; + return; + } + + if (RetryLetterTopic is null) + { + endpoint.DeadLetterTopic = DeadLetterTopic; + endpoint.Runtime.Options.Policies.OnAnyException().MoveToErrorQueue(); + + } + else if (RetryLetterTopic is not null) + { + if (endpoint.SubscriptionType is SubscriptionType.Failover or SubscriptionType.Exclusive) + { + throw new InvalidOperationException( + "Pulsar does not support Retry letter queueing with Failover or Exclusive subscription types. Please use Shared or KeyShared subscription types."); + } + + endpoint.DeadLetterTopic = DeadLetterTopic; + endpoint.RetryLetterTopic = RetryLetterTopic; + //endpoint.Runtime.Options.Policies.OnAnyException().MoveToRetryQueue(rt.Retry.Count, "PulsarNativeResiliency"); + endpoint.Runtime.Options.Policies.OnAnyException() + .ScheduleRetry(RetryLetterTopic.Retry.ToArray()) + .Then + .MoveToErrorQueue(); + + } + }; + } +} + +public abstract class PulsarNativeResiliencyConfiguration +{ + protected readonly PulsarListenerConfiguration Endpoint; + protected PulsarNativeResiliencyConfig NativeResiliencyConfig; + + protected PulsarNativeResiliencyConfiguration(PulsarListenerConfiguration endpoint) + { + Endpoint = endpoint; + NativeResiliencyConfig = new PulsarNativeResiliencyConfig(); + + } + + protected PulsarNativeResiliencyConfiguration(PulsarListenerConfiguration endpoint, PulsarNativeResiliencyConfig config) + { + Endpoint = endpoint; + NativeResiliencyConfig = config; + + } + +} + + +public class PulsarNativeResiliencyDeadLetterConfiguration : PulsarNativeResiliencyConfiguration +{ + + + public PulsarNativeResiliencyDeadLetterConfiguration(PulsarListenerConfiguration endpoint) + : base(endpoint) + { + + } /// /// Customize the dead letter queueing for this specific endpoint /// - /// Optional configuration + /// DLQ configuration /// - public PulsarSharedListenerConfiguration DeadLetterQueueing(DeadLetterTopic dlq) + public PulsarNativeResiliencyRetryLetterConfiguration DeadLetterQueueing(DeadLetterTopic dlq) { - add(e => - { - e.DeadLetterTopic = dlq; - }); + NativeResiliencyConfig.DeadLetterTopic = dlq; - return this; + return new PulsarNativeResiliencyRetryLetterConfiguration(Endpoint, NativeResiliencyConfig); } /// - /// Remove all dead letter queueing declarations from this queue + /// Disable native DLQ functionality for this queue /// /// - public PulsarSharedListenerConfiguration DisableDeadLetterQueueing() + public PulsarListenerConfiguration DisableDeadLetterQueueing() { - add(e => - { - e.DeadLetterTopic = null; - if (e.RetryLetterTopic is null && e.DeadLetterTopic is null) - e.Runtime.Options.Policies.Failures.Remove(rule => rule.Id == "PulsarNativeResiliency"); - }); + return this.Endpoint; + } +} + +public class PulsarNativeResiliencyRetryLetterConfiguration : PulsarNativeResiliencyConfiguration +{ + + public PulsarNativeResiliencyRetryLetterConfiguration(PulsarListenerConfiguration endpoint, PulsarNativeResiliencyConfig config) + : base(endpoint, config) + { + - return this; } /// - /// Customize the Retry letter queueing for this specific endpoint + /// Customize the retry letter queueing for this specific endpoint /// /// Optional configuration /// - public PulsarSharedListenerConfiguration RetryLetterQueueing(RetryLetterTopic rt) + public PulsarListenerConfiguration RetryLetterQueueing(RetryLetterTopic rt) { - add(e => - { - - e.RetryLetterTopic = rt; - - //var exceptionMatch = new AlwaysMatches(); // currently can't determine if endpoint listener needs it just based on exception, should handler that supports native resiliency, wrap the thrown exception into a new dedicated one? - //var failureRule = new FailureRule(exceptionMatch, "PulsarNativeResiliency"); - //foreach (var _ in rt.Retry) - //{ - // failureRule.AddSlot(new MoveToRetryQueueSource()); - //} - //e.Runtime.Options.Policies.Failures.Add(failureRule); - - //e.Runtime.Options.Policies.OnAnyException().MoveToErrorQueue(); - e.Runtime.Options.Policies.OnAnyException().MoveToRetryQueue(rt.Retry.Count, "PulsarNativeResiliency"); + NativeResiliencyConfig.RetryLetterTopic = rt; + Endpoint.Apply(NativeResiliencyConfig.Apply()); - }); - - return this; + return Endpoint; } /// - /// Remove all Retry letter queueing declarations from this queue + /// Disable native Retry letter functionality for this queue /// /// - public PulsarSharedListenerConfiguration DisableRetryLetterQueueing() + public PulsarListenerConfiguration DisableRetryLetterQueueing() { - add(e => - { - e.RetryLetterTopic = null; - if (e.RetryLetterTopic is null && e.DeadLetterTopic is null) - e.Runtime.Options.Policies.Failures.Remove(rule => rule.Id == "PulsarNativeResiliency"); - }); + NativeResiliencyConfig.RetryLetterTopic = null; + Endpoint.Apply(NativeResiliencyConfig.Apply()); - return this; + return Endpoint; } - } public class PulsarSubscriberConfiguration : SubscriberConfiguration diff --git a/src/Wolverine/Runtime/MessageContext.cs b/src/Wolverine/Runtime/MessageContext.cs index d97ef9dae..b18e4b814 100644 --- a/src/Wolverine/Runtime/MessageContext.cs +++ b/src/Wolverine/Runtime/MessageContext.cs @@ -125,7 +125,8 @@ public async Task ReScheduleAsync(DateTimeOffset scheduledTime) } Envelope.ScheduledTime = scheduledTime; - if (_channel is ISupportNativeScheduling c) + //if (_channel is ISupportNativeScheduling c) + if (tryGetRescheduler(_channel, Envelope) is ISupportNativeScheduling c) { await c.MoveToScheduledUntilAsync(Envelope, Envelope.ScheduledTime.Value); } @@ -134,6 +135,21 @@ public async Task ReScheduleAsync(DateTimeOffset scheduledTime) await Storage.Inbox.ScheduleJobAsync(Envelope); } } + + private ISupportNativeScheduling? tryGetRescheduler(IChannelCallback? channel, Envelope e) + { + if (e.Listener is ISupportNativeScheduling c2) + { + return c2; + } + + if (_channel is ISupportNativeScheduling c) + { + return c; + } + + return default; + } private ISupportDeadLetterQueue? tryGetDeadLetterQueue(IChannelCallback? channel, Envelope e) { if (_channel is ISupportDeadLetterQueue { NativeDeadLetterQueueEnabled: true } c) From d4c81acce7bff3a5e5c9185a16247919db2f706b Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Mon, 14 Apr 2025 20:27:37 +0200 Subject: [PATCH 22/30] #1326 refactoring - removed most of not needed constructs from the previously modified code --- .../PulsarNativeReliabilityTests.cs | 41 ++++---- .../Pulsar/Wolverine.Pulsar/PulsarListener.cs | 25 +---- .../PulsarTransportExtensions.cs | 4 +- src/Wolverine/ErrorHandling/FailureRule.cs | 6 -- .../ErrorHandling/MoveToErrorQueue.cs | 16 +--- .../ErrorHandling/MoveToRetryQueueSource.cs | 95 ------------------- .../ErrorHandling/PolicyExpression.cs | 30 +----- src/Wolverine/Logging/IMessageLogger.cs | 4 +- src/Wolverine/Runtime/MessageContext.cs | 1 + .../Runtime/WolverineRuntime.Tracking.cs | 14 +-- src/Wolverine/Runtime/WolverineTracing.cs | 6 -- src/Wolverine/Tracking/EnvelopeHistory.cs | 4 +- src/Wolverine/Tracking/ITrackedSession.cs | 8 +- src/Wolverine/Tracking/MessageEventType.cs | 2 +- src/Wolverine/Tracking/TrackedSession.cs | 2 +- src/Wolverine/Transports/IChannelCallback.cs | 11 --- 16 files changed, 46 insertions(+), 223 deletions(-) delete mode 100644 src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs index 803cff60b..ce710d8f8 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs @@ -37,7 +37,7 @@ private IHostBuilder ConfigureBuilder() opts.ListenToPulsarTopic(topicPath) .WithSharedSubscriptionType() .DeadLetterQueueing(DeadLetterTopic.DefaultNative) - .RetryLetterQueueing(new RetryLetterTopic([TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2)])) + .RetryLetterQueueing(new RetryLetterTopic([TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(2)])) //.ProcessInline(); .BufferedInMemory(); @@ -46,16 +46,17 @@ private IHostBuilder ConfigureBuilder() // .ProcessInline(); - var topicPath2 = $"persistent://public/default/no-retry-{topic}"; - opts.IncludeType(); + //var topicPath2 = $"persistent://public/default/no-retry-{topic}"; + //opts.IncludeType(); - opts.PublishMessage() - .ToPulsarTopic(topicPath2); + //opts.PublishMessage() + // .ToPulsarTopic(topicPath2); - opts.ListenToPulsarTopic(topicPath2) - .WithSharedSubscriptionType() - .DeadLetterQueueing(DeadLetterTopic.DefaultNative) - .ProcessInline(); + //opts.ListenToPulsarTopic(topicPath2) + // .WithSharedSubscriptionType() + // .DeadLetterQueueing(DeadLetterTopic.DefaultNative) + // .DisableRetryLetterQueueing() + // .ProcessInline(); }); } @@ -69,7 +70,7 @@ public async Task InitializeAsync() [Fact] public async Task run_setup_with_simulated_exception_in_handler() { - var session = await WolverineHost.TrackActivity(TimeSpan.FromSeconds(100)) + var session = await WolverineHost.TrackActivity(TimeSpan.FromSeconds(1000)) //.WaitForMessageToBeReceivedAt(WolverineHost) .DoNotAssertOnExceptionsDetected() .IncludeExternalTransports() @@ -86,17 +87,17 @@ public async Task run_setup_with_simulated_exception_in_handler() session.Received .MessagesOf() .Count() - .ShouldBe(3); + .ShouldBe(4); - session.Requeued + session.Rescheduled .MessagesOf() .Count() - .ShouldBe(2); + .ShouldBe(3); - session.MovedToRetryQueue - .MessagesOf() - .Count() - .ShouldBe(2); + //session.Requeued + // .MessagesOf() + // .Count() + // .ShouldBe(2); // TODO: I Guess the capture of the envelope headers occurs before we manipulate it //var firstRequeuedEnvelope = session.MovedToRetryQueue.Envelopes().First(); @@ -114,8 +115,8 @@ public async Task run_setup_with_simulated_exception_in_handler() var firstEnvelope = session.MovedToErrorQueue.Envelopes().First(); firstEnvelope.ShouldSatisfyAllConditions( () => firstEnvelope.Headers.ContainsKey(PulsarEnvelopeConstants.Exception).ShouldBeTrue(), - () => firstEnvelope.Headers[PulsarEnvelopeConstants.ReconsumeTimes].ShouldBe("2"), - () => firstEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(2).TotalMilliseconds.ToString()) + () => firstEnvelope.Headers[PulsarEnvelopeConstants.ReconsumeTimes].ShouldBe("3"), + () => firstEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(1).TotalMilliseconds.ToString()) ); } @@ -146,7 +147,7 @@ public async Task run_setup_with_simulated_exception_in_handler_only_native_dead .Count() .ShouldBe(0); - session.MovedToRetryQueue + session.Requeued .MessagesOf() .Count() .ShouldBe(0); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index 06730f1cd..d3bd670ec 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -8,7 +8,7 @@ namespace Wolverine.Pulsar; -internal class PulsarListener : IListener, ISupportDeadLetterQueue, ISupportRetryLetterQueue, ISupportNativeScheduling +internal class PulsarListener : IListener, ISupportDeadLetterQueue, ISupportNativeScheduling { private readonly CancellationToken _cancellation; private readonly IConsumer>? _consumer; @@ -230,29 +230,6 @@ public async Task MoveToErrorsAsync(Envelope envelope, Exception exception) } public bool NativeRetryLetterQueueEnabled { get; } - public bool RetryLimitReached(Envelope envelope) - { - if (NativeRetryLetterQueueEnabled && envelope is PulsarEnvelope e) - { - if (e.MessageData.Properties.TryGetValue(PulsarEnvelopeConstants.ReconsumeTimes, out var reconsumeTimesValue)) - { - var currentRetryCount = int.Parse(reconsumeTimesValue); - - return currentRetryCount >= _endpoint.RetryLetterTopic!.Retry.Count; - } - // first time failure - return false; - } - - return true; - } - - public async Task MoveToRetryQueueAsync(Envelope envelope, Exception exception) - { - // TODO: how to handle retries internally (in Wolverine context, not Pulsar)? - // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) - await moveToQueueAsync(envelope, exception); - } public async Task MoveToScheduledUntilAsync(Envelope envelope, DateTimeOffset time) { diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs index 747972d9d..d72d97a91 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs @@ -221,9 +221,6 @@ internal void Apply(Action action) // } } - - - public class PulsarNativeResiliencyConfig { public DeadLetterTopic DeadLetterTopic { get; set; } @@ -263,6 +260,7 @@ public Action Apply() .Then .MoveToErrorQueue(); + endpoint.Runtime.Options.EnableAutomaticFailureAcks = false; } }; } diff --git a/src/Wolverine/ErrorHandling/FailureRule.cs b/src/Wolverine/ErrorHandling/FailureRule.cs index 87ecff400..f8952219a 100644 --- a/src/Wolverine/ErrorHandling/FailureRule.cs +++ b/src/Wolverine/ErrorHandling/FailureRule.cs @@ -13,16 +13,10 @@ public FailureRule(IExceptionMatch match) { Match = match; } - public FailureRule(IExceptionMatch match, string? id) - { - Match = match; - Id = id; - } public FailureSlot this[int attempt] => _slots[attempt - 1]; public IExceptionMatch Match { get; } - public string? Id { get; } internal IContinuationSource? InfiniteSource { get; set; } public IEnumerator GetEnumerator() diff --git a/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs b/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs index 5da20a492..7357f4f77 100644 --- a/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs +++ b/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs @@ -16,23 +16,11 @@ public IContinuation Build(Exception ex, Envelope envelope) } } -internal class MoveToErrorQueueWithoutFailureAckSource : IContinuationSource -{ - public string Description => "Move to error queue without failure ack"; - - public IContinuation Build(Exception ex, Envelope envelope) - { - return new MoveToErrorQueue(ex, true); - } -} internal class MoveToErrorQueue : IContinuation { - private readonly bool _forceSkipSendFailureAcknowledgment; - - public MoveToErrorQueue(Exception exception, bool forceSkipSendFailureAcknowledgment = false) + public MoveToErrorQueue(Exception exception) { - _forceSkipSendFailureAcknowledgment = forceSkipSendFailureAcknowledgment; Exception = exception ?? throw new ArgumentNullException(nameof(exception)); } @@ -44,7 +32,7 @@ public async ValueTask ExecuteAsync(IEnvelopeLifecycle lifecycle, { // TODO -- at some point, we need a more systematic way of doing this var scheme = lifecycle.Envelope.Destination.Scheme; - if (!_forceSkipSendFailureAcknowledgment && runtime.Options.EnableAutomaticFailureAcks && scheme != TransportConstants.Local && scheme != "external-table") + if (runtime.Options.EnableAutomaticFailureAcks && scheme != TransportConstants.Local && scheme != "external-table") { await lifecycle.SendFailureAcknowledgementAsync( $"Moved message {lifecycle.Envelope!.Id} to the Error Queue.\n{Exception}"); diff --git a/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs b/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs deleted file mode 100644 index b447fc12e..000000000 --- a/src/Wolverine/ErrorHandling/MoveToRetryQueueSource.cs +++ /dev/null @@ -1,95 +0,0 @@ -using System.Diagnostics; -using Wolverine.Runtime; -using Wolverine.Transports; -using Wolverine.Transports.Local; -using Wolverine.Util; - -namespace Wolverine.ErrorHandling; - -internal class MoveToRetryQueueSource : IContinuationSource -{ - public string Description => "Move to retry queue"; - - public IContinuation Build(Exception ex, Envelope envelope) - { - return new MoveToRetryQueue(ex); - } -} - -internal class MoveToRetryQueue : IContinuation -{ - public MoveToRetryQueue(Exception exception) - { - Exception = exception ?? throw new ArgumentNullException(nameof(exception)); - } - - public Exception Exception { get; } - - public async ValueTask ExecuteAsync(IEnvelopeLifecycle lifecycle, - IWolverineRuntime runtime, - DateTimeOffset now, Activity? activity) - { - if (lifecycle.Envelope.Listener is ISupportRetryLetterQueue retryListener && - !retryListener.RetryLimitReached(lifecycle.Envelope)) - { - if (lifecycle.Envelope.Message != null) - { - lifecycle.Envelope.MessageType = lifecycle.Envelope.Message.GetType().ToMessageTypeName(); - } - - await retryListener.MoveToRetryQueueAsync(lifecycle.Envelope, Exception); - activity?.AddEvent(new ActivityEvent(WolverineTracing.MovedToRetryQueue)); - - runtime.MessageTracking.Requeued(lifecycle.Envelope); // TODO: new method below or this? - runtime.MessageTracking.MovedToRetryQueue(lifecycle.Envelope, Exception); - - } - else - { - await buildFallbackContinuation(lifecycle) - .ExecuteAsync(lifecycle, runtime, now, activity); - } - } - - private IContinuation buildFallbackContinuation(IEnvelopeLifecycle lifecycle) - { - var cs = new MoveToErrorQueueWithoutFailureAckSource(); - var continuation = cs.Build(Exception, lifecycle.Envelope); - return continuation; - } - - public override string ToString() - { - return "Move to Retry Queue"; - } - - protected bool Equals(MoveToRetryQueue other) - { - return Equals(Exception, other.Exception); - } - - public override bool Equals(object? obj) - { - if (ReferenceEquals(null, obj)) - { - return false; - } - - if (ReferenceEquals(this, obj)) - { - return true; - } - - if (obj.GetType() != GetType()) - { - return false; - } - - return Equals((MoveToRetryQueue)obj); - } - - public override int GetHashCode() - { - return Exception.GetHashCode(); - } -} \ No newline at end of file diff --git a/src/Wolverine/ErrorHandling/PolicyExpression.cs b/src/Wolverine/ErrorHandling/PolicyExpression.cs index 097c1898a..3c175fa21 100644 --- a/src/Wolverine/ErrorHandling/PolicyExpression.cs +++ b/src/Wolverine/ErrorHandling/PolicyExpression.cs @@ -71,9 +71,9 @@ internal class FailureActions : IAdditionalActions, IFailureActions private readonly FailureRule _rule; private readonly List _slots = new(); - public FailureActions(IExceptionMatch match, FailureRuleCollection parent, string? ruleId = null) + public FailureActions(IExceptionMatch match, FailureRuleCollection parent) { - _rule = new FailureRule(match, ruleId); + _rule = new FailureRule(match); parent.Add(_rule); } @@ -128,17 +128,6 @@ public IAdditionalActions MoveToErrorQueue() return this; } - public IAdditionalActions MoveToRetryQueue(int maxAttempts, string? ruleId = null) - { - for (var i = 0; i < maxAttempts - 1; i++) - { - var slot = _rule.AddSlot(new MoveToRetryQueueSource()); - _slots.Add(slot); - } - - return this; - } - public IAdditionalActions Requeue(int maxAttempts = 3) { if (maxAttempts > 25) @@ -308,12 +297,6 @@ public interface IFailureActions /// IAdditionalActions MoveToErrorQueue(); - /// - /// Immediately move the message to the retry queue when the exception - /// caught matches this criteria - /// - IAdditionalActions MoveToRetryQueue(int maxAttempts, string? ruleId = null); - /// /// Requeue the message back to the incoming transport, with the message being /// dead lettered when the maximum number of attempts is reached @@ -403,15 +386,6 @@ public IAdditionalActions MoveToErrorQueue() return new FailureActions(_match, _parent).MoveToErrorQueue(); } - /// - /// Immediately move the message to the retry queue when the exception - /// caught matches this criteria - /// - public IAdditionalActions MoveToRetryQueue(int maxAttempts, string? ruleId = null) - { - return new FailureActions(_match, _parent, ruleId).MoveToRetryQueue(maxAttempts); - } - /// /// Requeue the message back to the incoming transport, with the message being /// dead lettered when the maximum number of attempts is reached diff --git a/src/Wolverine/Logging/IMessageLogger.cs b/src/Wolverine/Logging/IMessageLogger.cs index e35f4a82a..b200f3236 100644 --- a/src/Wolverine/Logging/IMessageLogger.cs +++ b/src/Wolverine/Logging/IMessageLogger.cs @@ -74,11 +74,11 @@ public interface IMessageTracker /// - /// Called when Wolverine moves an envelope into the retry letter queue + /// Called when message execution was rescheduled at some later time /// /// /// - void MovedToRetryQueue(Envelope envelope, Exception ex); + void Rescheduled(Envelope envelope); /// /// Called when Wolverine moves an envelope into the dead letter queue diff --git a/src/Wolverine/Runtime/MessageContext.cs b/src/Wolverine/Runtime/MessageContext.cs index b18e4b814..96f31b68e 100644 --- a/src/Wolverine/Runtime/MessageContext.cs +++ b/src/Wolverine/Runtime/MessageContext.cs @@ -124,6 +124,7 @@ public async Task ReScheduleAsync(DateTimeOffset scheduledTime) throw new InvalidOperationException("No Envelope is active for this context"); } + Runtime.MessageTracking.Rescheduled(Envelope); Envelope.ScheduledTime = scheduledTime; //if (_channel is ISupportNativeScheduling c) if (tryGetRescheduler(_channel, Envelope) is ISupportNativeScheduling c) diff --git a/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs b/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs index c606509cc..d20c06f93 100644 --- a/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs +++ b/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs @@ -15,7 +15,7 @@ public sealed partial class WolverineRuntime : IMessageTracker public const int UndeliverableEventId = 108; private static readonly Action _movedToErrorQueue; - private static readonly Action _movedToRetryQueue; + private static readonly Action _rescheduled; private static readonly Action _noHandler; private static readonly Action _noRoutes; private static readonly Action _received; @@ -42,8 +42,8 @@ static WolverineRuntime() _noRoutes = LoggerMessage.Define(LogLevel.Information, NoRoutesEventId, "No routes can be determined for {envelope}"); - _movedToRetryQueue = LoggerMessage.Define(LogLevel.Error, MovedToErrorQueueId, - "Envelope {envelope} was moved to the retry queue"); + _rescheduled = LoggerMessage.Define(LogLevel.Error, MovedToErrorQueueId, + "Envelope {envelope} was rescheduled to queue"); _movedToErrorQueue = LoggerMessage.Define(LogLevel.Error, MovedToErrorQueueId, "Envelope {envelope} was moved to the error queue"); @@ -132,10 +132,11 @@ public void NoRoutesFor(Envelope envelope) _noRoutes(Logger, envelope, null); } - public void MovedToRetryQueue(Envelope envelope, Exception ex) + + public void Rescheduled(Envelope envelope) { - ActiveSession?.Record(MessageEventType.MovedToRetryQueue, envelope, _serviceName, _uniqueNodeId); - _movedToRetryQueue(Logger, envelope, ex); + ActiveSession?.Record(MessageEventType.Rescheduled, envelope, _serviceName, _uniqueNodeId); + _rescheduled(Logger, envelope, envelope.Failure); } public void MovedToErrorQueue(Envelope envelope, Exception ex) @@ -153,6 +154,7 @@ public void Requeued(Envelope envelope) { Logger.LogInformation("Requeue for message {Id} of message type {MessageType}", envelope.Id, envelope.MessageType); ActiveSession?.Record(MessageEventType.Requeued, envelope, _serviceName, _uniqueNodeId); + _rescheduled(Logger, envelope, null); } [Obsolete("Try to eliminate this")] diff --git a/src/Wolverine/Runtime/WolverineTracing.cs b/src/Wolverine/Runtime/WolverineTracing.cs index 08e5e4fc0..94de07af7 100644 --- a/src/Wolverine/Runtime/WolverineTracing.cs +++ b/src/Wolverine/Runtime/WolverineTracing.cs @@ -31,12 +31,6 @@ public const string /// public const string EnvelopeDiscarded = "wolverine.envelope.discarded"; - - /// - /// ActivityEvent marking when an incoming envelope is being moved to the retry queue - /// - public const string MovedToRetryQueue = "wolverine.error.retry.queued"; - /// /// ActivityEvent marking when an incoming envelope is being moved to the error queue /// diff --git a/src/Wolverine/Tracking/EnvelopeHistory.cs b/src/Wolverine/Tracking/EnvelopeHistory.cs index c8e39a002..a9a4aaf81 100644 --- a/src/Wolverine/Tracking/EnvelopeHistory.cs +++ b/src/Wolverine/Tracking/EnvelopeHistory.cs @@ -99,7 +99,7 @@ public void RecordLocally(EnvelopeRecord record) break; case MessageEventType.Requeued: - case MessageEventType.MovedToRetryQueue: + case MessageEventType.Rescheduled: // Do nothing, just informative break; @@ -156,7 +156,7 @@ public void RecordCrossApplication(EnvelopeRecord record) case MessageEventType.NoHandlers: case MessageEventType.NoRoutes: case MessageEventType.Requeued: - case MessageEventType.MovedToRetryQueue: + case MessageEventType.Rescheduled: break; diff --git a/src/Wolverine/Tracking/ITrackedSession.cs b/src/Wolverine/Tracking/ITrackedSession.cs index b279cad98..dba088fa1 100644 --- a/src/Wolverine/Tracking/ITrackedSession.cs +++ b/src/Wolverine/Tracking/ITrackedSession.cs @@ -56,14 +56,14 @@ public interface ITrackedSession RecordCollection MovedToErrorQueue { get; } /// - /// Records of all messages that were moved to the retry queue + /// Records of all messages that were requeued /// - RecordCollection MovedToRetryQueue { get; } + RecordCollection Requeued { get; } /// - /// Records of all messages that were requeued + /// Records of all messages that were rescheduled sometime in the future /// - RecordCollection Requeued { get; } + RecordCollection Rescheduled { get; } /// /// Message processing records for messages that were executed. Note that this includes message diff --git a/src/Wolverine/Tracking/MessageEventType.cs b/src/Wolverine/Tracking/MessageEventType.cs index ac9cd0c58..0f3b68692 100644 --- a/src/Wolverine/Tracking/MessageEventType.cs +++ b/src/Wolverine/Tracking/MessageEventType.cs @@ -13,6 +13,6 @@ public enum MessageEventType NoRoutes, MovedToErrorQueue, Requeued, - MovedToRetryQueue, + Rescheduled, } #endregion \ No newline at end of file diff --git a/src/Wolverine/Tracking/TrackedSession.cs b/src/Wolverine/Tracking/TrackedSession.cs index b832d75ba..a9a15cafb 100644 --- a/src/Wolverine/Tracking/TrackedSession.cs +++ b/src/Wolverine/Tracking/TrackedSession.cs @@ -157,7 +157,7 @@ public void AssertCondition(string message, Func condition) public RecordCollection NoHandlers => new(MessageEventType.NoHandlers, this); public RecordCollection NoRoutes => new(MessageEventType.NoRoutes, this); public RecordCollection MovedToErrorQueue => new(MessageEventType.MovedToErrorQueue, this); - public RecordCollection MovedToRetryQueue => new(MessageEventType.MovedToRetryQueue, this); + public RecordCollection Rescheduled => new(MessageEventType.Rescheduled, this); public RecordCollection Requeued => new(MessageEventType.Requeued, this); public RecordCollection Executed => new(MessageEventType.ExecutionFinished, this); diff --git a/src/Wolverine/Transports/IChannelCallback.cs b/src/Wolverine/Transports/IChannelCallback.cs index bed88f60e..10960df3d 100644 --- a/src/Wolverine/Transports/IChannelCallback.cs +++ b/src/Wolverine/Transports/IChannelCallback.cs @@ -10,17 +10,6 @@ public interface ISupportDeadLetterQueue Task MoveToErrorsAsync(Envelope envelope, Exception exception); } -/// -/// Marks an IChannelCallback as supporting a native retry letter queue -/// functionality -/// -public interface ISupportRetryLetterQueue -{ - bool NativeRetryLetterQueueEnabled { get; } - bool RetryLimitReached(Envelope envelope); - Task MoveToRetryQueueAsync(Envelope envelope, Exception exception); -} - /// /// Marks an IChannelCallback as supporting native scheduled send /// From ac343d941970a4107e0158d19384ce47532fe970 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Wed, 16 Apr 2025 08:40:42 +0200 Subject: [PATCH 23/30] #1326 refactoring - removed not needed construct from the previously modified code --- .../PulsarNativeReliabilityTests.cs | 49 +++++++++---------- src/Wolverine/Logging/IMessageLogger.cs | 8 --- src/Wolverine/Runtime/MessageContext.cs | 5 +- src/Wolverine/Tracking/ITrackedSession.cs | 5 -- src/Wolverine/Tracking/TrackedSession.cs | 1 - 5 files changed, 26 insertions(+), 42 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs index ce710d8f8..68f8723e1 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs @@ -37,7 +37,7 @@ private IHostBuilder ConfigureBuilder() opts.ListenToPulsarTopic(topicPath) .WithSharedSubscriptionType() .DeadLetterQueueing(DeadLetterTopic.DefaultNative) - .RetryLetterQueueing(new RetryLetterTopic([TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(2)])) + .RetryLetterQueueing(new RetryLetterTopic([TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3)])) //.ProcessInline(); .BufferedInMemory(); @@ -89,34 +89,37 @@ public async Task run_setup_with_simulated_exception_in_handler() .Count() .ShouldBe(4); - session.Rescheduled + session.Requeued .MessagesOf() .Count() .ShouldBe(3); - //session.Requeued - // .MessagesOf() - // .Count() - // .ShouldBe(2); // TODO: I Guess the capture of the envelope headers occurs before we manipulate it - //var firstRequeuedEnvelope = session.MovedToRetryQueue.Envelopes().First(); - //firstRequeuedEnvelope.ShouldSatisfyAllConditions( - // () => firstRequeuedEnvelope.Headers.ContainsKey("DELAY_TIME").ShouldBeTrue(), - // () => firstRequeuedEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(1).TotalMilliseconds.ToString()) - //); - //var secondRequeuedEnvelope = session.MovedToRetryQueue.Envelopes().Skip(1).First(); - //secondRequeuedEnvelope.ShouldSatisfyAllConditions( - // () => secondRequeuedEnvelope.Headers.ContainsKey("DELAY_TIME").ShouldBeTrue(), - // () => secondRequeuedEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(2).TotalMilliseconds.ToString()) - //); + var firstRequeuedEnvelope = session.Requeued.Envelopes().First(); + firstRequeuedEnvelope.ShouldSatisfyAllConditions( + () => firstRequeuedEnvelope.Attempts.ShouldBe(1), + () => firstRequeuedEnvelope.Headers.ContainsKey("DELAY_TIME").ShouldBeFalse() + ); + var secondRequeuedEnvelope = session.Requeued.Envelopes().Skip(1).First(); + secondRequeuedEnvelope.ShouldSatisfyAllConditions( + () => secondRequeuedEnvelope.Attempts.ShouldBe(2), + () => secondRequeuedEnvelope.Headers.ContainsKey("DELAY_TIME").ShouldBeTrue(), + () => secondRequeuedEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(4).TotalMilliseconds.ToString()) + ); + var thirdRequeuedEnvelope = session.Requeued.Envelopes().Skip(2).First(); + thirdRequeuedEnvelope.ShouldSatisfyAllConditions( + () => thirdRequeuedEnvelope.Attempts.ShouldBe(3), + () => thirdRequeuedEnvelope.Headers.ContainsKey("DELAY_TIME").ShouldBeTrue(), + () => thirdRequeuedEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(4).TotalMilliseconds.ToString()) // TODO: delay is not respected (always uses first specified delay) + ); - var firstEnvelope = session.MovedToErrorQueue.Envelopes().First(); - firstEnvelope.ShouldSatisfyAllConditions( - () => firstEnvelope.Headers.ContainsKey(PulsarEnvelopeConstants.Exception).ShouldBeTrue(), - () => firstEnvelope.Headers[PulsarEnvelopeConstants.ReconsumeTimes].ShouldBe("3"), - () => firstEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(1).TotalMilliseconds.ToString()) + + var dlqEnvelope = session.MovedToErrorQueue.Envelopes().First(); + dlqEnvelope.ShouldSatisfyAllConditions( + () => dlqEnvelope.Headers.ContainsKey(PulsarEnvelopeConstants.Exception).ShouldBeTrue(), + () => dlqEnvelope.Headers[PulsarEnvelopeConstants.ReconsumeTimes].ShouldBe("3") ); } @@ -147,10 +150,6 @@ public async Task run_setup_with_simulated_exception_in_handler_only_native_dead .Count() .ShouldBe(0); - session.Requeued - .MessagesOf() - .Count() - .ShouldBe(0); diff --git a/src/Wolverine/Logging/IMessageLogger.cs b/src/Wolverine/Logging/IMessageLogger.cs index b200f3236..408a8f807 100644 --- a/src/Wolverine/Logging/IMessageLogger.cs +++ b/src/Wolverine/Logging/IMessageLogger.cs @@ -72,14 +72,6 @@ public interface IMessageTracker /// void NoRoutesFor(Envelope envelope); - - /// - /// Called when message execution was rescheduled at some later time - /// - /// - /// - void Rescheduled(Envelope envelope); - /// /// Called when Wolverine moves an envelope into the dead letter queue /// diff --git a/src/Wolverine/Runtime/MessageContext.cs b/src/Wolverine/Runtime/MessageContext.cs index 96f31b68e..7a98b2bb8 100644 --- a/src/Wolverine/Runtime/MessageContext.cs +++ b/src/Wolverine/Runtime/MessageContext.cs @@ -124,9 +124,8 @@ public async Task ReScheduleAsync(DateTimeOffset scheduledTime) throw new InvalidOperationException("No Envelope is active for this context"); } - Runtime.MessageTracking.Rescheduled(Envelope); + Runtime.MessageTracking.Requeued(Envelope); Envelope.ScheduledTime = scheduledTime; - //if (_channel is ISupportNativeScheduling c) if (tryGetRescheduler(_channel, Envelope) is ISupportNativeScheduling c) { await c.MoveToScheduledUntilAsync(Envelope, Envelope.ScheduledTime.Value); @@ -144,7 +143,7 @@ public async Task ReScheduleAsync(DateTimeOffset scheduledTime) return c2; } - if (_channel is ISupportNativeScheduling c) + if (channel is ISupportNativeScheduling c) { return c; } diff --git a/src/Wolverine/Tracking/ITrackedSession.cs b/src/Wolverine/Tracking/ITrackedSession.cs index dba088fa1..8b8c076c5 100644 --- a/src/Wolverine/Tracking/ITrackedSession.cs +++ b/src/Wolverine/Tracking/ITrackedSession.cs @@ -60,11 +60,6 @@ public interface ITrackedSession /// RecordCollection Requeued { get; } - /// - /// Records of all messages that were rescheduled sometime in the future - /// - RecordCollection Rescheduled { get; } - /// /// Message processing records for messages that were executed. Note that this includes message /// executions that failed and additional attempts as a separate record in the case of retries diff --git a/src/Wolverine/Tracking/TrackedSession.cs b/src/Wolverine/Tracking/TrackedSession.cs index a9a15cafb..7611cd1a6 100644 --- a/src/Wolverine/Tracking/TrackedSession.cs +++ b/src/Wolverine/Tracking/TrackedSession.cs @@ -157,7 +157,6 @@ public void AssertCondition(string message, Func condition) public RecordCollection NoHandlers => new(MessageEventType.NoHandlers, this); public RecordCollection NoRoutes => new(MessageEventType.NoRoutes, this); public RecordCollection MovedToErrorQueue => new(MessageEventType.MovedToErrorQueue, this); - public RecordCollection Rescheduled => new(MessageEventType.Rescheduled, this); public RecordCollection Requeued => new(MessageEventType.Requeued, this); public RecordCollection Executed => new(MessageEventType.ExecutionFinished, this); From 908b4deb0be98a0c108139ac55fa11902a60b1af Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Thu, 24 Apr 2025 15:28:46 +0200 Subject: [PATCH 24/30] #1326 refactoring - removed not needed construct from the previously modified code - removed DotPulsar.Extensions.Resiliency (retry and delay message functionality was not working properly) and created "in-house" retry and DLQ functionalities --- .../PulsarNativeReliabilityTests.cs | 20 +-- .../PulsarEnvelopeConstants.cs | 6 + .../Pulsar/Wolverine.Pulsar/PulsarListener.cs | 136 ++++++++++++------ .../Wolverine.Pulsar/PulsarTransport.cs | 47 +++--- .../Wolverine.Pulsar/Wolverine.Pulsar.csproj | 1 - 5 files changed, 143 insertions(+), 67 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs index 68f8723e1..b561def54 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs @@ -46,17 +46,17 @@ private IHostBuilder ConfigureBuilder() // .ProcessInline(); - //var topicPath2 = $"persistent://public/default/no-retry-{topic}"; - //opts.IncludeType(); + var topicPath2 = $"persistent://public/default/no-retry-{topic}"; + opts.IncludeType(); - //opts.PublishMessage() - // .ToPulsarTopic(topicPath2); + opts.PublishMessage() + .ToPulsarTopic(topicPath2); - //opts.ListenToPulsarTopic(topicPath2) - // .WithSharedSubscriptionType() - // .DeadLetterQueueing(DeadLetterTopic.DefaultNative) - // .DisableRetryLetterQueueing() - // .ProcessInline(); + opts.ListenToPulsarTopic(topicPath2) + .WithSharedSubscriptionType() + .DeadLetterQueueing(DeadLetterTopic.DefaultNative) + .DisableRetryLetterQueueing() + .ProcessInline(); }); } @@ -112,7 +112,7 @@ public async Task run_setup_with_simulated_exception_in_handler() thirdRequeuedEnvelope.ShouldSatisfyAllConditions( () => thirdRequeuedEnvelope.Attempts.ShouldBe(3), () => thirdRequeuedEnvelope.Headers.ContainsKey("DELAY_TIME").ShouldBeTrue(), - () => thirdRequeuedEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(4).TotalMilliseconds.ToString()) // TODO: delay is not respected (always uses first specified delay) + () => thirdRequeuedEnvelope.Headers["DELAY_TIME"].ShouldBe(TimeSpan.FromSeconds(2).TotalMilliseconds.ToString()) ); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs index 32372642e..b2dd4c99a 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeConstants.cs @@ -3,5 +3,11 @@ namespace Wolverine.Pulsar; public static class PulsarEnvelopeConstants { public const string ReconsumeTimes = "RECONSUMETIMES"; + public const string DelayTimeMetadataKey = "DELAY_TIME"; + public const string RealTopicMetadataKey = "REAL_TOPIC"; + public const string TopicMetadataKey = "TOPIC_NAME"; + public const string MessageIdMetadataKey = "MESSAGE_ID"; + public const string RealSubscriptionMetadataKey = "REAL_SUBSCRIPTION"; + public const string OriginMessageIdMetadataKey = "ORIGIN_MESSAGE_ID"; public const string Exception = "EXCEPTION"; } \ No newline at end of file diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index d3bd670ec..2376cc6db 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.Globalization; using DotPulsar; using DotPulsar.Abstractions; using DotPulsar.Extensions; @@ -17,9 +18,10 @@ internal class PulsarListener : IListener, ISupportDeadLetterQueue, ISupportNati private readonly Task? _receivingLoop; private readonly Task? _receivingRetryLoop; private readonly PulsarSender _sender; - private DeadLetterPolicy? _dlqClient; private IReceiver _receiver; private PulsarEndpoint _endpoint; + private IProducer>? _retryLetterQueueProducer; + private IProducer>? _dlqProducer; public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IReceiver receiver, PulsarTransport transport, @@ -74,7 +76,7 @@ public PulsarListener(IWolverineRuntime runtime, PulsarEndpoint endpoint, IRecei }, combined.Token); - if (_dlqClient != null) + if (NativeRetryLetterQueueEnabled) { _retryConsumer = createRetryConsumer(endpoint, transport); _receivingRetryLoop = Task.Run(async () => @@ -104,15 +106,18 @@ private void trySetupNativeResiliency(PulsarEndpoint endpoint, PulsarTransport t return; } - var topicDql = NativeDeadLetterQueueEnabled ? getDeadLetteredTopicUri(endpoint) : null; - var topicRetry = NativeRetryLetterQueueEnabled ? getRetryLetterTopicUri(endpoint) : null; - var retryCount = NativeRetryLetterQueueEnabled ? endpoint.RetryLetterTopic!.Retry.Count : 0; + if (endpoint.RetryLetterTopic is not null) + { + + _retryLetterQueueProducer = transport.Client!.NewProducer() + .Topic(getRetryLetterTopicUri(endpoint)!.ToString()) + .Create(); + } + + _dlqProducer = transport.Client!.NewProducer() + .Topic(getDeadLetteredTopicUri(endpoint).ToString()) + .Create(); - _dlqClient = new DeadLetterPolicy( - topicDql != null ? transport.Client!.NewProducer().Topic(topicDql.ToString()) : null, - topicRetry != null ? transport.Client!.NewProducer().Topic(topicRetry.ToString()) : null, - retryCount - ); } @@ -180,11 +185,17 @@ public async ValueTask DisposeAsync() await _retryConsumer.DisposeAsync(); } - if (_dlqClient != null) + if (_retryLetterQueueProducer != null) { - await _dlqClient.DisposeAsync(); + await _retryLetterQueueProducer.DisposeAsync(); } + if (_dlqProducer != null) + { + await _dlqProducer.DisposeAsync(); + } + + await _sender.DisposeAsync(); _receivingLoop!.Dispose(); @@ -225,8 +236,11 @@ public async Task TryRequeueAsync(Envelope envelope) public bool NativeDeadLetterQueueEnabled { get; } public async Task MoveToErrorsAsync(Envelope envelope, Exception exception) { - // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) - await moveToQueueAsync(envelope, exception, true); + if (NativeRetryLetterQueueEnabled && envelope is PulsarEnvelope e) + { + // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) + await moveToQueueAsync(envelope, exception, true); + } } public bool NativeRetryLetterQueueEnabled { get; } @@ -236,7 +250,6 @@ public async Task MoveToScheduledUntilAsync(Envelope envelope, DateTimeOffset ti if (NativeRetryLetterQueueEnabled && envelope is PulsarEnvelope e) { await moveToQueueAsync(e, e.Failure, false); - } } @@ -245,40 +258,83 @@ private async Task moveToQueueAsync(Envelope envelope, Exception exception, bool { if (envelope is PulsarEnvelope e) { - if (_dlqClient != null) + var messageMetadata = BuildMessageMetadata(envelope, e, exception, isDeadLettered); + + + IConsumer>? associatedConsumer; + IProducer> associatedProducer; + + if (NativeRetryLetterQueueEnabled && !isDeadLettered) { - var message = e.MessageData; - IConsumer>? associatedConsumer; - TimeSpan? delayTime = null; + associatedConsumer = _retryConsumer!; + associatedProducer = _retryLetterQueueProducer!; + } + else + { + associatedConsumer = _consumer!; + associatedProducer = _dlqProducer!; + } - if (message.TryGetMessageProperty(PulsarEnvelopeConstants.ReconsumeTimes, out var reconsumeTimesValue)) - { - associatedConsumer = _retryConsumer; - var retryCount = int.Parse(reconsumeTimesValue); - delayTime = !isDeadLettered ? _endpoint.RetryLetterTopic!.Retry[retryCount] : null; - } - else - { - associatedConsumer = _consumer; - } - if (isDeadLettered) - { - e.Headers[PulsarEnvelopeConstants.Exception] = exception.ToString(); - } + await associatedConsumer.Acknowledge(e.MessageData, + _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ + // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection. If I understand correctly, Wolverine is/should handle original Wolverine message and its copies across Pulsar's topics as same identity? + // TODO: e.Attempts / attempts header value is out of sync with Pulsar's RECONSUMETIMES header! - await associatedConsumer!.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ - // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection. If I understand correctly, Wolverine is/should handle original Wolverine message and its copies across Pulsar's topics as same identity? - // TODO: e.Attempts / attempts header value is out of sync with Pulsar's RECONSUMETIMES header! - if (delayTime is null && _endpoint.RetryLetterTopic is not null) - delayTime = !isDeadLettered ? _endpoint.RetryLetterTopic!.Retry.First() : null; - await _dlqClient.ReconsumeLater(message, delayTime: delayTime, cancellationToken: _cancellation); - } + await associatedProducer.Send(messageMetadata, e.MessageData.Data, _cancellation) + .ConfigureAwait(false); } } + private MessageMetadata BuildMessageMetadata(Envelope envelope, PulsarEnvelope e, Exception exception, + bool isDeadLettered) + { + var messageMetadata = new MessageMetadata(); + foreach (var property in e.Headers) + { + messageMetadata[property.Key] = property.Value; + } + + //reconsumeTimesValue = GetReconsumeHeader(messageMetadata); + + if (!e.Headers.TryGetValue(PulsarEnvelopeConstants.RealTopicMetadataKey, out var originTopicNameStr)) + { + originTopicNameStr = envelope.Headers[EnvelopeConstants.ReplyUriKey]; + + } + + messageMetadata[PulsarEnvelopeConstants.RealTopicMetadataKey] = originTopicNameStr; + + var eid = e.Headers.GetValueOrDefault(PulsarEnvelopeConstants.OriginMessageIdMetadataKey, e.MessageData.MessageId.ToString()); + + if (!e.Headers.ContainsKey(PulsarEnvelopeConstants.OriginMessageIdMetadataKey)) + { + messageMetadata[PulsarEnvelopeConstants.OriginMessageIdMetadataKey] = eid; + } + + if (NativeRetryLetterQueueEnabled) + { + + if (!isDeadLettered) + { + messageMetadata[PulsarEnvelopeConstants.ReconsumeTimes] = envelope.Attempts.ToString(); + var delayTime = _endpoint.RetryLetterTopic!.Retry[envelope.Attempts - 1]; + messageMetadata[PulsarEnvelopeConstants.DelayTimeMetadataKey] = delayTime.TotalMilliseconds.ToString(CultureInfo.InvariantCulture); + messageMetadata.DeliverAtTimeAsDateTimeOffset = DateTimeOffset.UtcNow.Add(delayTime); + } + else + { + //messageMetadata[PulsarEnvelopeConstants.DelayTimeMetadataKey] = null; + messageMetadata.DeliverAtTimeAsDateTimeOffset = DateTimeOffset.UtcNow; + e.Headers[PulsarEnvelopeConstants.Exception] = exception.ToString(); + } + } + + + return messageMetadata; + } } diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs index b35243f22..553edfae2 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs @@ -31,22 +31,37 @@ public PulsarTransport() : base(ProtocolName, "Pulsar") public RetryLetterTopic? RetryLetterTopic { get; internal set; } // TODO: should we even have a default or just per endpoint based? - private IEnumerable enabledDeadLetterTopics() - { - if (DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage) - { - yield return DeadLetterTopic; - } - - foreach (var queue in endpoints()) - { - if (queue.IsPersistent && queue.Role == EndpointRole.Application && queue.DeadLetterTopic != null && - queue.DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage) - { - yield return queue.DeadLetterTopic; - } - } - } + //private IEnumerable enabledDeadLetterTopics() + //{ + // if (DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage) + // { + // yield return DeadLetterTopic; + // } + + // foreach (var queue in endpoints()) + // { + // if (queue.IsPersistent && queue.Role == EndpointRole.Application && queue.DeadLetterTopic != null && + // queue.DeadLetterTopic.Mode != DeadLetterTopicMode.WolverineStorage) + // { + // yield return queue.DeadLetterTopic; + // } + // } + //} + + //public IEnumerable enabledRetryLetterTopics() + //{ + // if (RetryLetterTopic != null) + // { + // yield return RetryLetterTopic; + // } + // foreach (var queue in endpoints()) + // { + // if (queue.IsPersistent && queue.Role == EndpointRole.Application && queue.RetryLetterTopic != null) + // { + // yield return queue.RetryLetterTopic; + // } + // } + //} public ValueTask DisposeAsync() { diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/Wolverine.Pulsar.csproj b/src/Transports/Pulsar/Wolverine.Pulsar/Wolverine.Pulsar.csproj index a2178e27f..5a8765c9e 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/Wolverine.Pulsar.csproj +++ b/src/Transports/Pulsar/Wolverine.Pulsar/Wolverine.Pulsar.csproj @@ -14,7 +14,6 @@ - From 58e21002cae2a76c34189198922ed65956e713f3 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Thu, 24 Apr 2025 23:12:50 +0200 Subject: [PATCH 25/30] #1326 refactoring --- .../PulsarNativeContinuationSource.cs | 21 ++++++++ .../PulsarNativeResiliencyContinuation.cs | 45 +++++++++++++++++ .../PulsarNativeResiliencyPolicy.cs | 22 ++++++++ .../Pulsar/Wolverine.Pulsar/PulsarListener.cs | 35 ++++++------- .../PulsarTransportExtensions.cs | 50 ++++++++++++++----- 5 files changed, 144 insertions(+), 29 deletions(-) create mode 100644 src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeContinuationSource.cs create mode 100644 src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeResiliencyContinuation.cs create mode 100644 src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeResiliencyPolicy.cs diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeContinuationSource.cs b/src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeContinuationSource.cs new file mode 100644 index 000000000..e37168120 --- /dev/null +++ b/src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeContinuationSource.cs @@ -0,0 +1,21 @@ +using Wolverine.ErrorHandling; +using Wolverine.Runtime; + +namespace Wolverine.Pulsar.ErrorHandling; + +public class PulsarNativeContinuationSource : IContinuationSource +{ + public string Description { get; } + + public IContinuation Build(Exception ex, Envelope envelope) + { + // Only handle Pulsar envelopes/listeners + if (envelope.Listener is PulsarListener) + { + return new PulsarNativeResiliencyContinuation(ex); + } + + // Return null to let the next continuation source handle it + return null; + } +} \ No newline at end of file diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeResiliencyContinuation.cs b/src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeResiliencyContinuation.cs new file mode 100644 index 000000000..eeb5fc2d9 --- /dev/null +++ b/src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeResiliencyContinuation.cs @@ -0,0 +1,45 @@ +using System.Diagnostics; +using Wolverine.ErrorHandling; +using Wolverine.Runtime; + +namespace Wolverine.Pulsar.ErrorHandling; + +public class PulsarNativeResiliencyContinuation : IContinuation +{ + private readonly Exception _exception; + + public PulsarNativeResiliencyContinuation(Exception exception) + { + _exception = exception; + } + + public async ValueTask ExecuteAsync(IEnvelopeLifecycle lifecycle, IWolverineRuntime runtime, DateTimeOffset now, Activity? activity) + { + var context = lifecycle as MessageContext; + var envelope = context?.Envelope; + + if (envelope?.Listener is PulsarListener listener) + { + if (listener.NativeRetryLetterQueueEnabled && listener.RetryLetterTopic!.Retry.Count >= envelope.Attempts) + { + // Use native retry if enabled and attempts are within bounds + //await listener.MoveToScheduledUntilAsync(envelope, now); + await new ScheduledRetryContinuation(listener.RetryLetterTopic!.Retry[envelope.Attempts - 1]) + .ExecuteAsync(lifecycle, runtime, now, activity); + return; + } + + if (listener.NativeDeadLetterQueueEnabled) + { + await new MoveToErrorQueue(_exception) + .ExecuteAsync(lifecycle, runtime, now, activity); + //await listener.MoveToErrorsAsync(envelope, _exception); + } + + return; + } + + // Fall back to MoveToErrorQueue for other listener types + await new MoveToErrorQueue(_exception).ExecuteAsync(lifecycle, runtime, now, activity); + } +} \ No newline at end of file diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeResiliencyPolicy.cs b/src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeResiliencyPolicy.cs new file mode 100644 index 000000000..5416a0672 --- /dev/null +++ b/src/Transports/Pulsar/Wolverine.Pulsar/ErrorHandling/PulsarNativeResiliencyPolicy.cs @@ -0,0 +1,22 @@ +using Wolverine.Configuration; +using Wolverine.ErrorHandling; +using Wolverine.ErrorHandling.Matches; + +namespace Wolverine.Pulsar.ErrorHandling; + +public class PulsarNativeResiliencyPolicy : IWolverinePolicy +{ + public void Apply(WolverineOptions options) + { + var rule = new FailureRule(new AlwaysMatches()); + + rule.AddSlot(new PulsarNativeContinuationSource()); + + // Set the same source as the InfiniteSource to handle all other attempts + rule.InfiniteSource = new PulsarNativeContinuationSource(); + + // Add this rule to the global failure collection + // This ensures it will be checked before other rules + options.Policies.Failures.Add(rule); + } +} \ No newline at end of file diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index 2376cc6db..e6127a7e2 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -234,9 +234,10 @@ public async Task TryRequeueAsync(Envelope envelope) } public bool NativeDeadLetterQueueEnabled { get; } + public RetryLetterTopic? RetryLetterTopic => _endpoint.RetryLetterTopic; public async Task MoveToErrorsAsync(Envelope envelope, Exception exception) { - if (NativeRetryLetterQueueEnabled && envelope is PulsarEnvelope e) + if (NativeDeadLetterQueueEnabled && envelope is PulsarEnvelope e) { // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) await moveToQueueAsync(envelope, exception, true); @@ -307,30 +308,30 @@ private MessageMetadata BuildMessageMetadata(Envelope envelope, PulsarEnvelope e messageMetadata[PulsarEnvelopeConstants.RealTopicMetadataKey] = originTopicNameStr; - var eid = e.Headers.GetValueOrDefault(PulsarEnvelopeConstants.OriginMessageIdMetadataKey, e.MessageData.MessageId.ToString()); + var eid = e.Headers.GetValueOrDefault(PulsarEnvelopeConstants.OriginMessageIdMetadataKey, + e.MessageData.MessageId.ToString()); if (!e.Headers.ContainsKey(PulsarEnvelopeConstants.OriginMessageIdMetadataKey)) { messageMetadata[PulsarEnvelopeConstants.OriginMessageIdMetadataKey] = eid; } - if (NativeRetryLetterQueueEnabled) - { - if (!isDeadLettered) - { - messageMetadata[PulsarEnvelopeConstants.ReconsumeTimes] = envelope.Attempts.ToString(); - var delayTime = _endpoint.RetryLetterTopic!.Retry[envelope.Attempts - 1]; - messageMetadata[PulsarEnvelopeConstants.DelayTimeMetadataKey] = delayTime.TotalMilliseconds.ToString(CultureInfo.InvariantCulture); - messageMetadata.DeliverAtTimeAsDateTimeOffset = DateTimeOffset.UtcNow.Add(delayTime); - } - else - { - //messageMetadata[PulsarEnvelopeConstants.DelayTimeMetadataKey] = null; - messageMetadata.DeliverAtTimeAsDateTimeOffset = DateTimeOffset.UtcNow; - e.Headers[PulsarEnvelopeConstants.Exception] = exception.ToString(); - } + if (!isDeadLettered) + { + messageMetadata[PulsarEnvelopeConstants.ReconsumeTimes] = envelope.Attempts.ToString(); + var delayTime = _endpoint.RetryLetterTopic!.Retry[envelope.Attempts - 1]; + messageMetadata[PulsarEnvelopeConstants.DelayTimeMetadataKey] = + delayTime.TotalMilliseconds.ToString(CultureInfo.InvariantCulture); + messageMetadata.DeliverAtTimeAsDateTimeOffset = DateTimeOffset.UtcNow.Add(delayTime); } + else + { + //messageMetadata[PulsarEnvelopeConstants.DelayTimeMetadataKey] = null; + messageMetadata.DeliverAtTimeAsDateTimeOffset = DateTimeOffset.UtcNow; + e.Headers[PulsarEnvelopeConstants.Exception] = exception.ToString(); + } + return messageMetadata; diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs index d72d97a91..154867028 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransportExtensions.cs @@ -1,9 +1,10 @@ +using System.Net; using DotPulsar; using DotPulsar.Abstractions; using JasperFx.Core.Reflection; using Wolverine.Configuration; using Wolverine.ErrorHandling; -using Wolverine.ErrorHandling.Matches; +using Wolverine.Pulsar.ErrorHandling; namespace Wolverine.Pulsar; @@ -30,6 +31,12 @@ internal static PulsarTransport PulsarTransport(this WolverineOptions endpoints) /// public static void UsePulsar(this WolverineOptions endpoints, Action configure) { + // doesn't apply the policy?!?: + //endpoints.Policies.Add(); + //endpoints.Policies.Add(new PulsarNativeResiliencyPolicy()); + + new PulsarNativeResiliencyPolicy().Apply(endpoints); + configure(endpoints.PulsarTransport().Builder); } @@ -226,11 +233,11 @@ public class PulsarNativeResiliencyConfig public DeadLetterTopic DeadLetterTopic { get; set; } public RetryLetterTopic? RetryLetterTopic { get; set; } + public Action Apply() { return endpoint => { - if (RetryLetterTopic is null && DeadLetterTopic is null) { endpoint.DeadLetterTopic = null; @@ -238,32 +245,51 @@ public Action Apply() return; } - if (RetryLetterTopic is null) + // Set the DLQ configuration regardless + if (DeadLetterTopic is not null) { endpoint.DeadLetterTopic = DeadLetterTopic; - endpoint.Runtime.Options.Policies.OnAnyException().MoveToErrorQueue(); - } - else if (RetryLetterTopic is not null) + + if (RetryLetterTopic is not null) { + // Validate subscription type if (endpoint.SubscriptionType is SubscriptionType.Failover or SubscriptionType.Exclusive) { throw new InvalidOperationException( "Pulsar does not support Retry letter queueing with Failover or Exclusive subscription types. Please use Shared or KeyShared subscription types."); } - endpoint.DeadLetterTopic = DeadLetterTopic; + // Set retry configuration endpoint.RetryLetterTopic = RetryLetterTopic; - //endpoint.Runtime.Options.Policies.OnAnyException().MoveToRetryQueue(rt.Retry.Count, "PulsarNativeResiliency"); - endpoint.Runtime.Options.Policies.OnAnyException() - .ScheduleRetry(RetryLetterTopic.Retry.ToArray()) - .Then - .MoveToErrorQueue(); endpoint.Runtime.Options.EnableAutomaticFailureAcks = false; } + + //if (RetryLetterTopic is null) + //{ + // // Just move to error queue with no retry + // endpoint.Runtime.Options.Policies.OnAnyException().MoveToErrorQueue(); + //} + //else + //{ + + // // Set retry configuration + // endpoint.RetryLetterTopic = RetryLetterTopic; + + // // Configure retry policy + + // //endpoint.IncomingRules + // endpoint.Runtime.Options.Policies.OnAnyException() + // .ScheduleRetry(RetryLetterTopic.Retry.ToArray()) + // .Then + // .MoveToErrorQueue(); + + // endpoint.Runtime.Options.EnableAutomaticFailureAcks = false; + //} }; } + } public abstract class PulsarNativeResiliencyConfiguration From 4462d5aaa2d38d62bdf7890e2a881bcdad033937 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Fri, 25 Apr 2025 08:48:12 +0200 Subject: [PATCH 26/30] #1326 clean-up --- .../PulsarNativeReliabilityTests.cs | 4 ---- .../Pulsar/Wolverine.Pulsar/PulsarTransport.cs | 1 - src/Wolverine/ErrorHandling/FailureRule.cs | 5 ----- src/Wolverine/Runtime/WolverineRuntime.Tracking.cs | 10 ++-------- src/Wolverine/Tracking/EnvelopeHistory.cs | 2 -- src/Wolverine/Tracking/ITrackedSession.cs | 2 +- src/Wolverine/Tracking/MessageEventType.cs | 3 +-- 7 files changed, 4 insertions(+), 23 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs index b561def54..6f202d3f2 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/PulsarNativeReliabilityTests.cs @@ -41,10 +41,6 @@ private IHostBuilder ConfigureBuilder() //.ProcessInline(); .BufferedInMemory(); - //opts.ListenToPulsarTopic(topicPath + "-DLQ") - // .WithSharedSubscriptionType() - // .ProcessInline(); - var topicPath2 = $"persistent://public/default/no-retry-{topic}"; opts.IncludeType(); diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs index 553edfae2..5717f5bc2 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarTransport.cs @@ -11,7 +11,6 @@ public class PulsarTransport : TransportBase, IAsyncDisposable { public const string ProtocolName = "pulsar"; - private readonly LightweightCache _endpoints; public PulsarTransport() : base(ProtocolName, "Pulsar") diff --git a/src/Wolverine/ErrorHandling/FailureRule.cs b/src/Wolverine/ErrorHandling/FailureRule.cs index f8952219a..2e7b01022 100644 --- a/src/Wolverine/ErrorHandling/FailureRule.cs +++ b/src/Wolverine/ErrorHandling/FailureRule.cs @@ -8,7 +8,6 @@ public class FailureRule : IEnumerable { private readonly List _slots = new(); - public FailureRule(IExceptionMatch match) { Match = match; @@ -56,8 +55,4 @@ public FailureSlot AddSlot(IContinuationSource source) return slot; } - - - - } \ No newline at end of file diff --git a/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs b/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs index d20c06f93..c2c179d9d 100644 --- a/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs +++ b/src/Wolverine/Runtime/WolverineRuntime.Tracking.cs @@ -13,6 +13,7 @@ public sealed partial class WolverineRuntime : IMessageTracker public const int NoRoutesEventId = 107; public const int MovedToErrorQueueId = 108; public const int UndeliverableEventId = 108; + public const int RescheduledEventId = 109; private static readonly Action _movedToErrorQueue; private static readonly Action _rescheduled; @@ -42,7 +43,7 @@ static WolverineRuntime() _noRoutes = LoggerMessage.Define(LogLevel.Information, NoRoutesEventId, "No routes can be determined for {envelope}"); - _rescheduled = LoggerMessage.Define(LogLevel.Error, MovedToErrorQueueId, + _rescheduled = LoggerMessage.Define(LogLevel.Error, RescheduledEventId, "Envelope {envelope} was rescheduled to queue"); _movedToErrorQueue = LoggerMessage.Define(LogLevel.Error, MovedToErrorQueueId, @@ -132,13 +133,6 @@ public void NoRoutesFor(Envelope envelope) _noRoutes(Logger, envelope, null); } - - public void Rescheduled(Envelope envelope) - { - ActiveSession?.Record(MessageEventType.Rescheduled, envelope, _serviceName, _uniqueNodeId); - _rescheduled(Logger, envelope, envelope.Failure); - } - public void MovedToErrorQueue(Envelope envelope, Exception ex) { ActiveSession?.Record(MessageEventType.MovedToErrorQueue, envelope, _serviceName, _uniqueNodeId); diff --git a/src/Wolverine/Tracking/EnvelopeHistory.cs b/src/Wolverine/Tracking/EnvelopeHistory.cs index a9a4aaf81..ae65a18f1 100644 --- a/src/Wolverine/Tracking/EnvelopeHistory.cs +++ b/src/Wolverine/Tracking/EnvelopeHistory.cs @@ -99,7 +99,6 @@ public void RecordLocally(EnvelopeRecord record) break; case MessageEventType.Requeued: - case MessageEventType.Rescheduled: // Do nothing, just informative break; @@ -156,7 +155,6 @@ public void RecordCrossApplication(EnvelopeRecord record) case MessageEventType.NoHandlers: case MessageEventType.NoRoutes: case MessageEventType.Requeued: - case MessageEventType.Rescheduled: break; diff --git a/src/Wolverine/Tracking/ITrackedSession.cs b/src/Wolverine/Tracking/ITrackedSession.cs index 8b8c076c5..0b7b6cf1e 100644 --- a/src/Wolverine/Tracking/ITrackedSession.cs +++ b/src/Wolverine/Tracking/ITrackedSession.cs @@ -58,7 +58,7 @@ public interface ITrackedSession /// /// Records of all messages that were requeued /// - RecordCollection Requeued { get; } + RecordCollection Requeued { get; } /// /// Message processing records for messages that were executed. Note that this includes message diff --git a/src/Wolverine/Tracking/MessageEventType.cs b/src/Wolverine/Tracking/MessageEventType.cs index 0f3b68692..d71a2a55a 100644 --- a/src/Wolverine/Tracking/MessageEventType.cs +++ b/src/Wolverine/Tracking/MessageEventType.cs @@ -12,7 +12,6 @@ public enum MessageEventType NoHandlers, NoRoutes, MovedToErrorQueue, - Requeued, - Rescheduled, + Requeued } #endregion \ No newline at end of file From f7fd39b1fdf32a6805f66894b6f854c40feffc7c Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Fri, 25 Apr 2025 08:52:08 +0200 Subject: [PATCH 27/30] #1326 clean-up --- src/Wolverine/ErrorHandling/FailureRuleCollection.cs | 5 ----- src/Wolverine/ErrorHandling/MoveToErrorQueue.cs | 1 - 2 files changed, 6 deletions(-) diff --git a/src/Wolverine/ErrorHandling/FailureRuleCollection.cs b/src/Wolverine/ErrorHandling/FailureRuleCollection.cs index 54be76904..6c7019d92 100644 --- a/src/Wolverine/ErrorHandling/FailureRuleCollection.cs +++ b/src/Wolverine/ErrorHandling/FailureRuleCollection.cs @@ -95,9 +95,4 @@ internal void Add(FailureRule rule) { _rules.Add(rule); } - - internal void Remove(Func rule) - { - _rules.RemoveAll(new Predicate(rule)); - } } \ No newline at end of file diff --git a/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs b/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs index 7357f4f77..a864de071 100644 --- a/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs +++ b/src/Wolverine/ErrorHandling/MoveToErrorQueue.cs @@ -16,7 +16,6 @@ public IContinuation Build(Exception ex, Envelope envelope) } } - internal class MoveToErrorQueue : IContinuation { public MoveToErrorQueue(Exception exception) From 707f40a5fe91164ef8aa832b60c349611ed9acc9 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Fri, 25 Apr 2025 09:04:59 +0200 Subject: [PATCH 28/30] #1326 clean-up --- .../Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs | 4 ---- src/Wolverine/Runtime/MessageContext.cs | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs index 504c67ebc..394053304 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEnvelopeMapper.cs @@ -23,10 +23,6 @@ protected override bool tryReadIncomingHeader(IMessage> i { if (key == EnvelopeConstants.AttemptsKey && incoming.Properties.TryGetValue(PulsarEnvelopeConstants.ReconsumeTimes, out value)) { - // dirty hack, handler increments Attempt field - int val = int.Parse(value); - value = (val).ToString(); - //value = (--val).ToString(); return true; } return incoming.Properties.TryGetValue(key, out value); diff --git a/src/Wolverine/Runtime/MessageContext.cs b/src/Wolverine/Runtime/MessageContext.cs index 7a98b2bb8..55474c6c2 100644 --- a/src/Wolverine/Runtime/MessageContext.cs +++ b/src/Wolverine/Runtime/MessageContext.cs @@ -138,6 +138,7 @@ public async Task ReScheduleAsync(DateTimeOffset scheduledTime) private ISupportNativeScheduling? tryGetRescheduler(IChannelCallback? channel, Envelope e) { + // TODO: is that ok, or should we modify Task ISupportNativeScheduling.MoveToScheduledUntilAsync(Envelope envelope, DateTimeOffset time) in DurableReceiver and BufferedReceiver? if (e.Listener is ISupportNativeScheduling c2) { return c2; From a19f6104756130238dbdaf29aec9d9261bc48db8 Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Fri, 25 Apr 2025 09:14:33 +0200 Subject: [PATCH 29/30] #1326 clean-up --- src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs index e6127a7e2..c4e321232 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarListener.cs @@ -239,7 +239,6 @@ public async Task MoveToErrorsAsync(Envelope envelope, Exception exception) { if (NativeDeadLetterQueueEnabled && envelope is PulsarEnvelope e) { - // TODO: Currently only ISupportDeadLetterQueue exists, should we introduce ISupportRetryLetterQueue concept? Because now on (first) exception, Wolverine calls this method (concept of retry letter queue is not set for Pulsar) await moveToQueueAsync(envelope, exception, true); } } @@ -261,7 +260,6 @@ private async Task moveToQueueAsync(Envelope envelope, Exception exception, bool { var messageMetadata = BuildMessageMetadata(envelope, e, exception, isDeadLettered); - IConsumer>? associatedConsumer; IProducer> associatedProducer; @@ -276,12 +274,9 @@ private async Task moveToQueueAsync(Envelope envelope, Exception exception, bool associatedProducer = _dlqProducer!; } - await associatedConsumer.Acknowledge(e.MessageData, _cancellation); // TODO: check: original message should be acked and copy is sent to retry/DLQ // TODO: check: what to do with the original message on Wolverine side? I Guess it should be acked? or we could use some kind of RequeueContinuation in FailureRuleCollection. If I understand correctly, Wolverine is/should handle original Wolverine message and its copies across Pulsar's topics as same identity? - // TODO: e.Attempts / attempts header value is out of sync with Pulsar's RECONSUMETIMES header! - await associatedProducer.Send(messageMetadata, e.MessageData.Data, _cancellation) .ConfigureAwait(false); From 70c46864a0684874c1fc2385128ff30cb05cd23a Mon Sep 17 00:00:00 2001 From: Rok Povodnik Date: Fri, 25 Apr 2025 09:20:43 +0200 Subject: [PATCH 30/30] #1326 clean-up --- src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs index 3eb6736b5..139d6d5da 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs +++ b/src/Transports/Pulsar/Wolverine.Pulsar/PulsarEndpoint.cs @@ -108,11 +108,6 @@ protected override ISender CreateSender(IWolverineRuntime runtime) return new PulsarSender(runtime, this, _parent, runtime.Cancellation); } - public override ValueTask InitializeAsync(ILogger logger) - { - return base.InitializeAsync(logger); - } - public override bool TryBuildDeadLetterSender(IWolverineRuntime runtime, out ISender? deadLetterSender) { return base.TryBuildDeadLetterSender(runtime, out deadLetterSender);