From 9d91dd236b2b73a7c70c38d9657f4c6c23c9c160 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Mon, 10 Mar 2025 11:30:37 -0700 Subject: [PATCH 01/42] Add SQS event source to test tool --- .../Amazon.Lambda.TestTool.csproj | 3 + .../Commands/RunCommand.cs | 11 +- .../Commands/Settings/RunCommandSettings.cs | 7 + .../SQSEventSourceBackgroundService.cs | 240 ++++++++++++++++++ .../SQSEventSourceBackgroundServiceConfig.cs | 42 +++ .../SQSEventSource/SQSEventSourceConfig.cs | 53 ++++ .../SQSEventSource/SQSEventSourceProcess.cs | 206 +++++++++++++++ 7 files changed, 560 insertions(+), 2 deletions(-) create mode 100644 Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs create mode 100644 Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundServiceConfig.cs create mode 100644 Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceConfig.cs create mode 100644 Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj index 2e3fd629c..081f1c76a 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj @@ -26,7 +26,10 @@ + + + diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs index 09d03e856..3097ef35c 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs @@ -7,6 +7,7 @@ using Amazon.Lambda.TestTool.Extensions; using Amazon.Lambda.TestTool.Models; using Amazon.Lambda.TestTool.Processes; +using Amazon.Lambda.TestTool.Processes.SQSEventSource; using Amazon.Lambda.TestTool.Services; using Amazon.Lambda.TestTool.Services.IO; using Spectre.Console.Cli; @@ -31,10 +32,10 @@ public override async Task ExecuteAsync(CommandContext context, RunCommandS { EvaluateEnvironmentVariables(settings); - if (!settings.LambdaEmulatorPort.HasValue && !settings.ApiGatewayEmulatorPort.HasValue) + if (!settings.LambdaEmulatorPort.HasValue && !settings.ApiGatewayEmulatorPort.HasValue && string.IsNullOrEmpty(settings.SQSEventSourceConfig)) { throw new ArgumentException("At least one of the following parameters must be set: " + - "--lambda-emulator-port or --api-gateway-emulator-port"); + "--lambda-emulator-port, --api-gateway-emulator-port or --sqs-eventsource-config"); } var tasks = new List(); @@ -74,6 +75,12 @@ public override async Task ExecuteAsync(CommandContext context, RunCommandS tasks.Add(apiGatewayEmulatorProcess.RunningTask); } + if (!string.IsNullOrEmpty(settings.SQSEventSourceConfig)) + { + var sqsEventSourceProcess = SQSEventSourceProcess.Startup(settings, cancellationTokenSource.Token); + tasks.Add(sqsEventSourceProcess.RunningTask); + } + await Task.WhenAny(tasks); return CommandReturnCodes.Success; diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs index cdabc377a..873895e8e 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs @@ -53,4 +53,11 @@ public sealed class RunCommandSettings : CommandSettings [CommandOption("--api-gateway-emulator-port ")] [Description("The port number used for the test tool's API Gateway emulator.")] public int? ApiGatewayEmulatorPort { get; set; } + + /// + /// JSON configuration for an SQS event source that will poll messages from a queue and forward the messages to the events. + /// + [CommandOption("--sqs-eventsource-config ")] + [Description("The JSON configuration for an SQS event source that will poll messages from a queue and forward the messages to the events. If the value is a file path the file will be read as the JSON value.")] + public string? SQSEventSourceConfig { get; set; } } diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs new file mode 100644 index 000000000..8070513e4 --- /dev/null +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs @@ -0,0 +1,240 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.Model; +using Amazon.Lambda.SQSEvents; +using Amazon.Runtime; +using Amazon.SQS.Model; +using Amazon.SQS; +using System.Text.Json; + +namespace Amazon.Lambda.TestTool.Processes.SQSEventSource; + +/// +/// IHostedService that will run continuially polling the SQS queue for messages and invoking the connected +/// Lambda function with the polled messages. +/// +public class SQSEventSourceBackgroundService : BackgroundService +{ + private static readonly List DefaultAttributesToReceive = new List { "All" }; + private static readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + private readonly ILogger _logger; + private readonly IAmazonSQS _sqsClient; + private readonly IAmazonLambda _lambdaClient; + private readonly SQSEventSourceBackgroundServiceConfig _config; + + /// + /// Constructs instance of SQSEventSourceBackgroundService. + /// + /// + /// + /// + public SQSEventSourceBackgroundService(ILogger logger, IAmazonSQS sqsClient, SQSEventSourceBackgroundServiceConfig config) + { + _logger = logger; + _sqsClient = sqsClient; + _config = config; + + _lambdaClient = new AmazonLambdaClient(new BasicAWSCredentials("accessKey", "secretKey"), new AmazonLambdaConfig + { + ServiceURL = _config.LambdaRuntimeApi + }); + } + + private async Task GetQueueArn(CancellationToken stoppingToken) + { + var response = await _sqsClient.GetQueueAttributesAsync(new GetQueueAttributesRequest + { + QueueUrl = _config.QueueUrl, + AttributeNames = new List { "QueueArn" } + }, stoppingToken); + + return response.QueueARN; + } + + /// + /// Execute the SQSEventSourceBackgroundService. + /// + /// + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var queueArn = await GetQueueArn(stoppingToken); + while (!stoppingToken.IsCancellationRequested) + { + try + { + _logger.LogDebug("Polling {queueUrl} for messages", _config.QueueUrl); + // Read a message from the queue using the ExternalCommands console application. + var response = await _sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest + { + QueueUrl = _config.QueueUrl, + WaitTimeSeconds = 20, + MessageAttributeNames = DefaultAttributesToReceive, + MessageSystemAttributeNames = DefaultAttributesToReceive, + MaxNumberOfMessages = _config.BatchSize, + VisibilityTimeout = _config.VisibilityTimeout, + }, stoppingToken); + + if (stoppingToken.IsCancellationRequested) + { + return; + } + if (response.Messages == null || response.Messages.Count == 0) + { + _logger.LogDebug("No messages received from while polling SQS"); + // Since there are no messages, sleep a bit to wait for messages to come. + await Task.Delay(1000); + continue; + } + + + var lambdaPayload = new + { + Records = ConvertToLambdaMessages(response.Messages, _sqsClient.Config.RegionEndpoint.SystemName, queueArn) + }; + + var invokeRequest = new InvokeRequest + { + InvocationType = InvocationType.RequestResponse, + FunctionName = _config.FunctionName, + Payload = JsonSerializer.Serialize(lambdaPayload, _jsonOptions) + }; + + _logger.LogInformation("Invoking Lambda function {functionName} function with {messageCount} messages", _config.FunctionName, lambdaPayload.Records.Count); + var lambdaResponse = await _lambdaClient.InvokeAsync(invokeRequest, stoppingToken); + + if (lambdaResponse.FunctionError != null) + { + _logger.LogError("Invoking Lambda {function} function with {messageCount} failed with error {errorMessage}", _config.FunctionName, response.Messages.Count, lambdaResponse.FunctionError); + continue; + } + + if (!_config.DisableMessageDelete) + { + List messagesToDelete; + if (lambdaResponse.Payload != null && lambdaResponse.Payload.Length > 0) + { + var partialResponse = JsonSerializer.Deserialize(lambdaResponse.Payload); + if (partialResponse == null) + { + lambdaResponse.Payload.Position = 0; + using var reader = new StreamReader(lambdaResponse.Payload); + var payloadString = reader.ReadToEnd(); + _logger.LogError("Failed to deserialize response from Lambda function into SQSBatchResponse. Response payload:\n{payload}", payloadString); + continue; + } + + if (partialResponse.BatchItemFailures == null || partialResponse.BatchItemFailures.Count == 0) + { + _logger.LogDebug("Partial SQS response received with no failures"); + messagesToDelete = response.Messages; + } + else + { + _logger.LogDebug("Partial SQS response received with {count} failures", partialResponse.BatchItemFailures.Count); + messagesToDelete = new List(); + foreach (var message in response.Messages) + { + if (partialResponse.BatchItemFailures.FirstOrDefault(x => string.Equals(x.ItemIdentifier, message.MessageId)) == null) + { + messagesToDelete.Add(message); + } + } + } + } + else + { + _logger.LogDebug("No partial response received. All messages eligible for deletion"); + messagesToDelete = response.Messages; + } + + var deleteRequest = new DeleteMessageBatchRequest + { + QueueUrl = _config.QueueUrl, + Entries = messagesToDelete.Select(m => new DeleteMessageBatchRequestEntry { Id = m.MessageId, ReceiptHandle = m.ReceiptHandle }).ToList() + }; + + _logger.LogDebug("Deleting {messageCount} messages from queue", deleteRequest.Entries.Count); + await _sqsClient.DeleteMessageBatchAsync(deleteRequest, stoppingToken); + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (TaskCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (Exception e) + { + _logger.LogWarning(e, "Exception occurred in SQS poller for {queueUrl}: {message}", _config.QueueUrl, e.Message); + + // Add a delay before restarting loop in case the exception was a transient error that needs a little time to reset. + await Task.Delay(3000); + } + } + } + + /// + /// Convert from the SDK's list of messages to the Lambda event's SQS message type. + /// + /// + /// + /// + /// + internal static List ConvertToLambdaMessages(List message, string awsRegion, string queueArn) + { + return message.Select(m => ConvertToLambdaMessage(m, awsRegion, queueArn)).ToList(); + } + + /// + /// Convert from the SDK's SQS message to the Lambda event's SQS message type. + /// + /// + /// + /// + /// + internal static SQSEvent.SQSMessage ConvertToLambdaMessage(Message message, string awsRegion, string queueArn) + { + var lambdaMessage = new SQSEvent.SQSMessage + { + AwsRegion = awsRegion, + Body = message.Body, + EventSource = "aws:sqs", + EventSourceArn = queueArn, + Md5OfBody = message.MD5OfBody, + Md5OfMessageAttributes = message.MD5OfMessageAttributes, + MessageId = message.MessageId, + ReceiptHandle = message.ReceiptHandle, + }; + + if (message.MessageAttributes != null && message.MessageAttributes.Count > 0) + { + lambdaMessage.MessageAttributes = new Dictionary(); + foreach (var kvp in message.MessageAttributes) + { + var lambdaAttribute = new SQSEvent.MessageAttribute + { + DataType = kvp.Value.DataType, + StringValue = kvp.Value.StringValue, + BinaryValue = kvp.Value.BinaryValue + }; + + lambdaMessage.MessageAttributes.Add(kvp.Key, lambdaAttribute); + } + } + + if (message.Attributes != null && message.Attributes.Count > 0) + { + lambdaMessage.Attributes = message.Attributes; + } + + return lambdaMessage; + } +} diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundServiceConfig.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundServiceConfig.cs new file mode 100644 index 000000000..118c19386 --- /dev/null +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundServiceConfig.cs @@ -0,0 +1,42 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.TestTool.Processes.SQSEventSource; + +/// +/// Configuration for the SQSEventSourceBackgroundService service. +/// +public class SQSEventSourceBackgroundServiceConfig +{ + /// + /// The batch size to read and send to Lambda function. This is the upper bound of messages to read and send. + /// SQS will return with less then batch size if there are not enough messages in the queue. + /// + public required int BatchSize { get; init; } = SQSEventSourceProcess.DefaultBatchSize; + + /// + /// If true the SQSEventSourceBackgroundService will skip deleting messages from the queue after the Lambda function returns. + /// + public required bool DisableMessageDelete { get; init; } + + /// + /// The Lambda function to send the SQS messages to delete to. + /// + public required string FunctionName { get; init; } + + /// + /// The endpoint where the emulated Lambda runtime API is running. The Lambda function identified by FunctionName must be listening for events from this endpoint. + /// + public required string LambdaRuntimeApi { get; init; } + + /// + /// The SQS queue url to poll for messages. + /// + public required string QueueUrl { get; init; } + + /// + /// The visibility timeout used for messages read. This is the length the message will not be visible to be read + /// again once it is returned in the receive call. + /// + public required int VisibilityTimeout { get; init; } = SQSEventSourceProcess.DefaultVisiblityTimeout; +} diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceConfig.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceConfig.cs new file mode 100644 index 000000000..6a6fc7e61 --- /dev/null +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceConfig.cs @@ -0,0 +1,53 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.TestTool.Processes.SQSEventSource; + +/// +/// This class represents the input values from the user. +/// +internal class SQSEventSourceConfig +{ + /// + /// The batch size to read and send to Lambda function. This is the upper bound of messages to read and send. + /// SQS will return with less then batch size if there are not enough messages in the queue. + /// + public int? BatchSize { get; set; } + + /// + /// If true the SQSEventSourceBackgroundService will skip deleting messages from the queue after the Lambda function returns. + /// + public bool? DisableMessageDelete { get; set; } + + /// + /// The Lambda function to send the SQS messages to delete to. + /// If not set the default function will be used. + /// + public string? FunctionName { get; set; } + + /// + /// The endpoint where the emulated Lambda runtime API is running. The Lambda function identified by FunctionName must be listening for events from this endpoint. + /// If not set the current Test Tool instance will be used assuming it is running a Lambda runtime api emulator. + /// + public string? LambdaRuntimeApi { get; set; } + /// + /// The AWS profile to use for credentials for fetching messages from the queue. + /// + public string? Profile { get; set; } + + /// + /// The queue url where messages should be polled from. + /// + public string? QueueUrl { get; set; } + + /// + /// The AWS region the queue is in. + /// + public string? Region { get; set; } + + /// + /// The visibility timeout used for messages read. This is the length the message will not be visible to be read + /// again once it is returned in the receive call. + /// + public int? VisibilityTimeout { get; set; } +} diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs new file mode 100644 index 000000000..0194b4115 --- /dev/null +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs @@ -0,0 +1,206 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.SQS; +using Amazon.Lambda.TestTool.Commands.Settings; +using Amazon.Lambda.TestTool.Services; +using System.Text.Json; + +namespace Amazon.Lambda.TestTool.Processes.SQSEventSource; + +/// +/// Process for handling SQS event source for Lambda functions. +/// +public class SQSEventSourceProcess +{ + internal const int DefaultBatchSize = 10; + internal const int DefaultVisiblityTimeout = 30; + + /// + /// The API Gateway emulator task that was started. + /// + public required Task RunningTask { get; init; } + + /// + /// Startup SQS event sources + /// + /// + /// + /// + /// + public static SQSEventSourceProcess Startup(RunCommandSettings settings, CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(settings.SQSEventSourceConfig)) + { + throw new InvalidOperationException($"The {nameof(RunCommandSettings.SQSEventSourceConfig)} can not be null when starting the SQS event source process"); + } + + var sqsEventSourceConfigs = LoadSQSEventSourceConfig(settings.SQSEventSourceConfig); + + var tasks = new List(); + + // Spin up a separate SQSEventSourceBackgroundService for each SQS event source config listed in the SQSEventSourceConfig + foreach (var sqsEventSourceConfig in sqsEventSourceConfigs) + { + var builder = Host.CreateApplicationBuilder(); + + var sqsConfig = new AmazonSQSConfig(); + if (!string.IsNullOrEmpty(sqsEventSourceConfig.Profile)) + { + sqsConfig.Profile = new Profile(sqsEventSourceConfig.Profile); + } + + if (!string.IsNullOrEmpty(sqsEventSourceConfig.Region)) + { + sqsConfig.RegionEndpoint = RegionEndpoint.GetBySystemName(sqsEventSourceConfig.Region); + } + + var sqsClient = new AmazonSQSClient(sqsConfig); + builder.Services.AddSingleton(sqsClient); + + var queueUrl = sqsEventSourceConfig.QueueUrl; + if (string.IsNullOrEmpty(queueUrl)) + { + throw new InvalidOperationException("QueueUrl is a required property for SQS event source config"); + } + + var lambdaRuntimeApi = sqsEventSourceConfig.LambdaRuntimeApi; + if (string.IsNullOrEmpty(lambdaRuntimeApi)) + { + if (!settings.LambdaEmulatorPort.HasValue) + { + throw new InvalidOperationException("No Lambda runtime api endpoint was given as part of the SQS event source config and the current " + + "instance of the test tool is not running the Lambda runtime api. Either provide a Lambda runtime api endpoint or set a port for " + + "the lambda runtime api when starting the test tool."); + } + lambdaRuntimeApi = $"http://{settings.LambdaEmulatorHost}:{settings.LambdaEmulatorPort}/"; + } + + var backgroundServiceConfig = new SQSEventSourceBackgroundServiceConfig + { + BatchSize = sqsEventSourceConfig.BatchSize ?? DefaultBatchSize, + DisableMessageDelete = sqsEventSourceConfig.DisableMessageDelete ?? false, + FunctionName = sqsEventSourceConfig.FunctionName ?? LambdaRuntimeApi.DefaultFunctionName, + LambdaRuntimeApi = lambdaRuntimeApi, + QueueUrl = queueUrl, + VisibilityTimeout = sqsEventSourceConfig.VisibilityTimeout ?? DefaultVisiblityTimeout + }; + + builder.Services.AddSingleton(backgroundServiceConfig); + builder.Services.AddHostedService(); + + var app = builder.Build(); + var task = app.RunAsync(cancellationToken); + tasks.Add(task); + } + + var combinedTask = Task.WhenAll(tasks); + + + return new SQSEventSourceProcess + { + RunningTask = combinedTask + }; + } + + internal static List LoadSQSEventSourceConfig(string sqsEventSourceConfigJson) + { + if (File.Exists(sqsEventSourceConfigJson)) + { + sqsEventSourceConfigJson = File.ReadAllText(sqsEventSourceConfigJson); + } + + sqsEventSourceConfigJson = sqsEventSourceConfigJson.Trim(); + + List? configs = null; + + var jsonOptions = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }; + + + if (sqsEventSourceConfigJson.StartsWith('[')) + { + configs = JsonSerializer.Deserialize>(sqsEventSourceConfigJson, jsonOptions); + if (configs == null) + { + throw new InvalidOperationException("Failed to parse SQS event source JSON config: " + sqsEventSourceConfigJson); + } + } + else if (sqsEventSourceConfigJson.StartsWith('{')) + { + var config = JsonSerializer.Deserialize(sqsEventSourceConfigJson, jsonOptions); + if (config == null) + { + throw new InvalidOperationException("Failed to parse SQS event source JSON config: " + sqsEventSourceConfigJson); + } + + configs = new List { config }; + } + else if (Uri.TryCreate(sqsEventSourceConfigJson, UriKind.Absolute, out _)) + { + configs = new List { new SQSEventSourceConfig { QueueUrl = sqsEventSourceConfigJson } }; + } + else + { + var config = new SQSEventSourceConfig(); + var tokens = sqsEventSourceConfigJson.Split(','); + foreach(var token in tokens) + { + if (string.IsNullOrWhiteSpace(token)) + continue; + + var keyValuePair = token.Split('='); + if (keyValuePair.Length != 2) + { + throw new InvalidOperationException("Failed to parse SQS event source config. Format should be \"QueueUrl=,FunctionName=,...\""); + } + + switch (keyValuePair[0].ToLower().Trim()) + { + case "batchsize": + if (!int.TryParse(keyValuePair[1].Trim(), out var batchSize)) + { + throw new InvalidOperationException("Value for batch size is not a formatted integer"); + } + config.BatchSize = batchSize; + break; + case "disablemessagedelete": + if (!bool.TryParse(keyValuePair[1].Trim(), out var disableMessageDelete)) + { + throw new InvalidOperationException("Value for disable message delete is not a formatted boolean"); + } + config.DisableMessageDelete = disableMessageDelete; + break; + case "functionname": + config.FunctionName = keyValuePair[1].Trim(); + break; + case "lambdaruntimeapi": + config.LambdaRuntimeApi = keyValuePair[1].Trim(); + break; + case "profile": + config.Profile = keyValuePair[1].Trim(); + break; + case "queueurl": + config.QueueUrl = keyValuePair[1].Trim(); + break; + case "region": + config.Region = keyValuePair[1].Trim(); + break; + case "visibilitytimeout": + if (!int.TryParse(keyValuePair[1].Trim(), out var visibilityTimeout)) + { + throw new InvalidOperationException("Value for visibility timeout is not a formatted integer"); + } + config.VisibilityTimeout = visibilityTimeout; + break; + } + } + + configs = new List { config }; + } + + return configs; + } +} From 49e45706267c36566e8f39c80185b148e46856c8 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Mon, 10 Mar 2025 23:45:59 -0700 Subject: [PATCH 02/42] Add unit tests --- .../src/Amazon.Lambda.TestTool/Constants.cs | 6 + .../SQSEventSourceBackgroundService.cs | 1 + .../SQSEventSource/SQSEventSourceProcess.cs | 94 +++++++++--- .../ConvertSDKToLambdaEventTests.cs | 78 ++++++++++ .../ParseSQSEventSourceConfigTests.cs | 142 ++++++++++++++++++ 5 files changed, 297 insertions(+), 24 deletions(-) create mode 100644 Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/SQSEventSource/ConvertSDKToLambdaEventTests.cs create mode 100644 Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/SQSEventSource/ParseSQSEventSourceConfigTests.cs diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs index 11a940278..f788f7a2b 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs @@ -84,4 +84,10 @@ public abstract class Constants /// The Visual Studio Marketplace link for the AWS Toolkit for Visual Studio. /// public const string LinkVsToolkitMarketplace = "https://marketplace.visualstudio.com/items?itemName=AmazonWebServices.AWSToolkitforVisualStudio2022"; + + /// + /// Prefix used for values of command line arguments that support the value being stored in an environment variable. + /// This used in the Aspire integration where it is often easier to pass configuration via environment variables. + /// + public const string ARGUMENT_ENVIRONMENT_VARIABLE_PREFIX = "env:"; } diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs index 8070513e4..affc796db 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs @@ -63,6 +63,7 @@ private async Task GetQueueArn(CancellationToken stoppingToken) /// protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + // The queue arn is needed for creating the Lambda event. var queueArn = await GetQueueArn(stoppingToken); while (!stoppingToken.IsCancellationRequested) { diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs index 0194b4115..d93a7a338 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs @@ -17,7 +17,7 @@ public class SQSEventSourceProcess internal const int DefaultVisiblityTimeout = 30; /// - /// The API Gateway emulator task that was started. + /// The Parent task for all of the tasks started for each list SQS event source. /// public required Task RunningTask { get; init; } @@ -39,7 +39,7 @@ public static SQSEventSourceProcess Startup(RunCommandSettings settings, Cancell var tasks = new List(); - // Spin up a separate SQSEventSourceBackgroundService for each SQS event source config listed in the SQSEventSourceConfig + // Create a separate SQSEventSourceBackgroundService for each SQS event source config listed in the SQSEventSourceConfig foreach (var sqsEventSourceConfig in sqsEventSourceConfigs) { var builder = Host.CreateApplicationBuilder(); @@ -94,23 +94,47 @@ public static SQSEventSourceProcess Startup(RunCommandSettings settings, Cancell tasks.Add(task); } - var combinedTask = Task.WhenAll(tasks); - - return new SQSEventSourceProcess { - RunningTask = combinedTask + RunningTask = Task.WhenAll(tasks) }; } - internal static List LoadSQSEventSourceConfig(string sqsEventSourceConfigJson) + /// + /// Load the SQS event source configs. The format of the config can be either JSON or comma delimited key pairs. + /// With the JSON format it is possible to configure multiple event sources but special care is required + /// escaping the quotes. The JSON format also provides consistency with the API Gateway configuration. + /// + /// The comma delimited key pairs allows users to configure a single SQS event source without having + /// to deal with escaping quotes. + /// + /// If the value of sqsEventSourceConfigString points to a file that exists the contents of the file + /// will be read and sued for the value for SQS event source config. + /// + /// If the value of sqsEventSourceConfigString starts with "env:" then it assume the suffix of the value + /// is an environment variable containing the config. This is used by the .NET Aspire integration because + /// the values required for the config are resolved after command line arguments are setup in Aspire. + /// + /// + /// + /// + internal static List LoadSQSEventSourceConfig(string sqsEventSourceConfigString) { - if (File.Exists(sqsEventSourceConfigJson)) + if (sqsEventSourceConfigString.StartsWith(Constants.ARGUMENT_ENVIRONMENT_VARIABLE_PREFIX, StringComparison.CurrentCultureIgnoreCase)) { - sqsEventSourceConfigJson = File.ReadAllText(sqsEventSourceConfigJson); + var envVariable = sqsEventSourceConfigString.Substring(Constants.ARGUMENT_ENVIRONMENT_VARIABLE_PREFIX.Length); + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(envVariable))) + { + throw new InvalidOperationException($"Environment variable {envVariable} for the SQS event source config was empty"); + } + sqsEventSourceConfigString = Environment.GetEnvironmentVariable(envVariable)!; + } + else if (File.Exists(sqsEventSourceConfigString)) + { + sqsEventSourceConfigString = File.ReadAllText(sqsEventSourceConfigString); } - sqsEventSourceConfigJson = sqsEventSourceConfigJson.Trim(); + sqsEventSourceConfigString = sqsEventSourceConfigString.Trim(); List? configs = null; @@ -119,33 +143,55 @@ internal static List LoadSQSEventSourceConfig(string sqsEv PropertyNameCaseInsensitive = true }; - - if (sqsEventSourceConfigJson.StartsWith('[')) + // Check to see if the config is in JSON array format. + // The JSON format provides consistency with the API Gateway config style. + if (sqsEventSourceConfigString.StartsWith('[')) { - configs = JsonSerializer.Deserialize>(sqsEventSourceConfigJson, jsonOptions); - if (configs == null) + try { - throw new InvalidOperationException("Failed to parse SQS event source JSON config: " + sqsEventSourceConfigJson); + configs = JsonSerializer.Deserialize>(sqsEventSourceConfigString, jsonOptions); + if (configs == null) + { + throw new InvalidOperationException("Failed to parse SQS event source JSON config: " + sqsEventSourceConfigString); + } + } + catch(JsonException e) + { + throw new InvalidOperationException("Failed to parse SQS event source JSON config: " + sqsEventSourceConfigString, e); } } - else if (sqsEventSourceConfigJson.StartsWith('{')) + // Config is a single object JSON document. + // The JSON format provides consistency with the API Gateway config style. + else if (sqsEventSourceConfigString.StartsWith('{')) { - var config = JsonSerializer.Deserialize(sqsEventSourceConfigJson, jsonOptions); - if (config == null) + try { - throw new InvalidOperationException("Failed to parse SQS event source JSON config: " + sqsEventSourceConfigJson); - } + var config = JsonSerializer.Deserialize(sqsEventSourceConfigString, jsonOptions); + if (config == null) + { + throw new InvalidOperationException("Failed to parse SQS event source JSON config: " + sqsEventSourceConfigString); + } - configs = new List { config }; + configs = new List { config }; + } + catch (JsonException e) + { + throw new InvalidOperationException("Failed to parse SQS event source JSON config: " + sqsEventSourceConfigString, e); + } } - else if (Uri.TryCreate(sqsEventSourceConfigJson, UriKind.Absolute, out _)) + // Config is a QueueUrl only. The current test tool instance will be assumed the Lambda runtime api and the + // messages will be sent to the default function. Support this format allows for an + // simple CLI experience of just providing a single value for the default scenario. + else if (Uri.TryCreate(sqsEventSourceConfigString, UriKind.Absolute, out _)) { - configs = new List { new SQSEventSourceConfig { QueueUrl = sqsEventSourceConfigJson } }; + configs = new List { new SQSEventSourceConfig { QueueUrl = sqsEventSourceConfigString } }; } + // Config is in comma delimited key value pair format. This format allows setting all the parameters without having + // to deal with escaping quotes like the JSON format. else { var config = new SQSEventSourceConfig(); - var tokens = sqsEventSourceConfigJson.Split(','); + var tokens = sqsEventSourceConfigString.Split(','); foreach(var token in tokens) { if (string.IsNullOrWhiteSpace(token)) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/SQSEventSource/ConvertSDKToLambdaEventTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/SQSEventSource/ConvertSDKToLambdaEventTests.cs new file mode 100644 index 000000000..c8722551d --- /dev/null +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/SQSEventSource/ConvertSDKToLambdaEventTests.cs @@ -0,0 +1,78 @@ +using Amazon.Lambda.TestTool.Processes.SQSEventSource; +using Amazon.SQS.Model; +using Xunit; + +namespace Amazon.Lambda.TestTool.UnitTests.SQSEventSource; + +public class ConvertSDKToLambdaEventTests +{ + [Fact] + public void ConvertSDKMessageFull() + { + var sdkMessage = new Message + { + Attributes = new Dictionary { { "key1", "value1" }, { "key2", "value2" } }, + Body = "theBody", + MD5OfBody = "theBodyMD5", + MD5OfMessageAttributes = "attributesMD5", + MessageAttributes = new Dictionary + { + { "key1", new MessageAttributeValue{StringValue = "value1", DataType = "String"} }, + { "key2", new MessageAttributeValue{BinaryValue = new MemoryStream(), DataType = "Binary"} } + }, + MessageId = "id", + ReceiptHandle = "receiptHandle" + }; + + var eventMessage = SQSEventSourceBackgroundService.ConvertToLambdaMessage(sdkMessage, "us-west-2", "queueArn"); + Assert.Equal("us-west-2", eventMessage.AwsRegion); + Assert.Equal("queueArn", eventMessage.EventSourceArn); + Assert.Equal("aws:sqs", eventMessage.EventSource); + + Assert.Equal(sdkMessage.Attributes, eventMessage.Attributes); + Assert.Equal("theBody", eventMessage.Body); + Assert.Equal("theBodyMD5", eventMessage.Md5OfBody); + Assert.Equal("attributesMD5", eventMessage.Md5OfMessageAttributes); + Assert.Equal("id", eventMessage.MessageId); + Assert.Equal("receiptHandle", eventMessage.ReceiptHandle); + + Assert.Equal(2, eventMessage.MessageAttributes.Count); + + Assert.Equal("value1", eventMessage.MessageAttributes["key1"].StringValue); + Assert.Null(eventMessage.MessageAttributes["key1"].BinaryValue); + Assert.Equal("String", eventMessage.MessageAttributes["key1"].DataType); + + Assert.Null(eventMessage.MessageAttributes["key2"].StringValue); + Assert.NotNull(eventMessage.MessageAttributes["key2"].BinaryValue); + Assert.Equal("Binary", eventMessage.MessageAttributes["key2"].DataType); + } + + [Fact] + public void ConvertSDKMessageWithNullCollections() + { + var sdkMessage = new Message + { + Attributes = null, + Body = "theBody", + MD5OfBody = "theBodyMD5", + MD5OfMessageAttributes = "attributesMD5", + MessageAttributes = null, + MessageId = "id", + ReceiptHandle = "receiptHandle" + }; + + var eventMessage = SQSEventSourceBackgroundService.ConvertToLambdaMessage(sdkMessage, "us-west-2", "queueArn"); + Assert.Equal("us-west-2", eventMessage.AwsRegion); + Assert.Equal("queueArn", eventMessage.EventSourceArn); + Assert.Equal("aws:sqs", eventMessage.EventSource); + + Assert.Equal("theBody", eventMessage.Body); + Assert.Equal("theBodyMD5", eventMessage.Md5OfBody); + Assert.Equal("attributesMD5", eventMessage.Md5OfMessageAttributes); + Assert.Equal("id", eventMessage.MessageId); + Assert.Equal("receiptHandle", eventMessage.ReceiptHandle); + + Assert.Null(eventMessage.Attributes); + Assert.Null(eventMessage.MessageAttributes); + } +} diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/SQSEventSource/ParseSQSEventSourceConfigTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/SQSEventSource/ParseSQSEventSourceConfigTests.cs new file mode 100644 index 000000000..b9686f219 --- /dev/null +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/SQSEventSource/ParseSQSEventSourceConfigTests.cs @@ -0,0 +1,142 @@ +using Amazon.Lambda.TestTool.Processes.SQSEventSource; +using Xunit; + +namespace Amazon.Lambda.TestTool.UnitTests.SQSEventSource; + +public class ParseSQSEventSourceConfigTests +{ + [Fact] + public void ParseValidJsonObject() + { + string json = """ +{ + "QueueUrl" : "https://amazonsqs/queueurl", + "FunctionName" : "LambdaFunction", + "BatchSize" : 5, + "DisableMessageDelete" : true, + "LambdaRuntimeApi" : "http://localhost:7777/", + "Profile" : "beta", + "Region" : "us-east-23", + "VisibilityTimeout" : 50 +} +"""; + + var configs = SQSEventSourceProcess.LoadSQSEventSourceConfig(json); + Assert.Single(configs); + Assert.Equal("https://amazonsqs/queueurl", configs[0].QueueUrl); + Assert.Equal("LambdaFunction", configs[0].FunctionName); + Assert.Equal(5, configs[0].BatchSize); + Assert.True(configs[0].DisableMessageDelete); + Assert.Equal("http://localhost:7777/", configs[0].LambdaRuntimeApi); + Assert.Equal("beta", configs[0].Profile); + Assert.Equal("us-east-23", configs[0].Region); + Assert.Equal(50, configs[0].VisibilityTimeout); + + } + + [Fact] + public void ParseInvalidJsonObject() + { + string json = """ +{ + "aaa" +} +"""; + + Assert.Throws(() => SQSEventSourceProcess.LoadSQSEventSourceConfig(json)); + } + + + [Fact] + public void ParseValidJsonArray() + { + string json = """ +[ + { + "QueueUrl" : "https://amazonsqs/queueurl", + "FunctionName" : "LambdaFunction", + "BatchSize" : 5, + "DisableMessageDelete" : true, + "LambdaRuntimeApi" : "http://localhost:7777/", + "Profile" : "beta", + "Region" : "us-east-23", + "VisibilityTimeout" : 50 + }, + { + "QueueUrl" : "https://amazonsqs/queueurl", + "FunctionName" : "LambdaFunction", + "BatchSize" : 5, + "DisableMessageDelete" : true, + "LambdaRuntimeApi" : "http://localhost:7777/", + "Profile" : "beta", + "Region" : "us-east-23", + "VisibilityTimeout" : 50 + } +] +"""; + + var configs = SQSEventSourceProcess.LoadSQSEventSourceConfig(json); + Assert.Equal(2, configs.Count); + + foreach (var config in configs) + { + Assert.Equal("https://amazonsqs/queueurl", config.QueueUrl); + Assert.Equal("LambdaFunction", config.FunctionName); + Assert.Equal(5, config.BatchSize); + Assert.True(config.DisableMessageDelete); + Assert.Equal("http://localhost:7777/", config.LambdaRuntimeApi); + Assert.Equal("beta", config.Profile); + Assert.Equal("us-east-23", config.Region); + Assert.Equal(50, config.VisibilityTimeout); + } + } + + [Fact] + public void ParseInvalidJsonArray() + { + string json = """ +[ + {"aaa"} +] +"""; + + Assert.Throws(() => SQSEventSourceProcess.LoadSQSEventSourceConfig(json)); + } + + [Fact] + public void ParseQueueUrl() + { + var configs = SQSEventSourceProcess.LoadSQSEventSourceConfig("https://amazonsqs/queueurl"); + Assert.Single(configs); + Assert.Equal("https://amazonsqs/queueurl", configs[0].QueueUrl); + } + + [Fact] + public void ParseKeyPairs() + { + var configs = SQSEventSourceProcess.LoadSQSEventSourceConfig( + "QueueUrl=https://amazonsqs/queueurl ,functionName =LambdaFunction, batchSize=5, DisableMessageDelete=true," + + "LambdaRuntimeApi=http://localhost:7777/ ,Profile=beta,Region=us-east-23,VisibilityTimeout=50"); + + Assert.Single(configs); + Assert.Equal("https://amazonsqs/queueurl", configs[0].QueueUrl); + Assert.Equal("LambdaFunction", configs[0].FunctionName); + Assert.Equal(5, configs[0].BatchSize); + Assert.True(configs[0].DisableMessageDelete); + Assert.Equal("http://localhost:7777/", configs[0].LambdaRuntimeApi); + Assert.Equal("beta", configs[0].Profile); + Assert.Equal("us-east-23", configs[0].Region); + Assert.Equal(50, configs[0].VisibilityTimeout); + } + + [Theory] + [InlineData("novalue")] + [InlineData("BatchSize=noint")] + [InlineData("VisibilityTimeout=noint")] + [InlineData("DisableMessageDelete=nobool")] + [InlineData("QueueUrl=https://amazonsqs/queueurl FunctionName =LambdaFunction")] + public void InvalidKeyPairString(string keyPairConfig) + { + Assert.Throws(() => SQSEventSourceProcess.LoadSQSEventSourceConfig(keyPairConfig)); + } +} From a10ddaf0fe26ab5792b3f37557ea3b7b4f302fb2 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 11 Mar 2025 09:55:58 -0700 Subject: [PATCH 03/42] Add integ tests --- .../Commands/RunCommand.cs | 10 + .../src/Amazon.Lambda.TestTool/Constants.cs | 2 +- .../SQSEventSourceBackgroundService.cs | 18 +- .../SQSEventSource/SQSEventSourceProcess.cs | 15 +- ...on.Lambda.TestTool.IntegrationTests.csproj | 1 + .../SQSEventSourceTests.cs | 389 ++++++++++++++++++ .../TestEnvironmentManager.cs | 8 + 7 files changed, 420 insertions(+), 23 deletions(-) create mode 100644 Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs create mode 100644 Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestEnvironmentManager.cs diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs index 3097ef35c..b90fa3e81 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs @@ -139,5 +139,15 @@ private void EvaluateEnvironmentVariables(RunCommandSettings settings) throw new ArgumentException($"Value for {API_GATEWAY_EMULATOR_PORT} environment variable was not a valid port number"); } } + + if (settings.SQSEventSourceConfig != null && settings.SQSEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix, StringComparison.CurrentCultureIgnoreCase)) + { + var envVariable = settings.SQSEventSourceConfig.Substring(Constants.ArgumentEnvironmentVariablePrefix.Length); + if (!environmentVariables.Contains(envVariable)) + { + throw new InvalidOperationException($"Environment variable {envVariable} for the SQS event source config was empty"); + } + settings.SQSEventSourceConfig = environmentVariables[envVariable]?.ToString(); + } } } diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs index f788f7a2b..abd63eed7 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs @@ -89,5 +89,5 @@ public abstract class Constants /// Prefix used for values of command line arguments that support the value being stored in an environment variable. /// This used in the Aspire integration where it is often easier to pass configuration via environment variables. /// - public const string ARGUMENT_ENVIRONMENT_VARIABLE_PREFIX = "env:"; + public const string ArgumentEnvironmentVariablePrefix = "env:"; } diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs index affc796db..1fe8d753e 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs @@ -93,7 +93,6 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) continue; } - var lambdaPayload = new { Records = ConvertToLambdaMessages(response.Messages, _sqsClient.Config.RegionEndpoint.SystemName, queueArn) @@ -141,7 +140,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) messagesToDelete = new List(); foreach (var message in response.Messages) { - if (partialResponse.BatchItemFailures.FirstOrDefault(x => string.Equals(x.ItemIdentifier, message.MessageId)) == null) + if (!partialResponse.BatchItemFailures.Any(x => string.Equals(x.ItemIdentifier, message.MessageId))) { messagesToDelete.Add(message); } @@ -154,14 +153,17 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) messagesToDelete = response.Messages; } - var deleteRequest = new DeleteMessageBatchRequest + if (messagesToDelete.Count > 0) { - QueueUrl = _config.QueueUrl, - Entries = messagesToDelete.Select(m => new DeleteMessageBatchRequestEntry { Id = m.MessageId, ReceiptHandle = m.ReceiptHandle }).ToList() - }; + var deleteRequest = new DeleteMessageBatchRequest + { + QueueUrl = _config.QueueUrl, + Entries = messagesToDelete.Select(m => new DeleteMessageBatchRequestEntry { Id = m.MessageId, ReceiptHandle = m.ReceiptHandle }).ToList() + }; - _logger.LogDebug("Deleting {messageCount} messages from queue", deleteRequest.Entries.Count); - await _sqsClient.DeleteMessageBatchAsync(deleteRequest, stoppingToken); + _logger.LogDebug("Deleting {messageCount} messages from queue", deleteRequest.Entries.Count); + await _sqsClient.DeleteMessageBatchAsync(deleteRequest, stoppingToken); + } } } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs index d93a7a338..3848d4c42 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs @@ -110,26 +110,13 @@ public static SQSEventSourceProcess Startup(RunCommandSettings settings, Cancell /// /// If the value of sqsEventSourceConfigString points to a file that exists the contents of the file /// will be read and sued for the value for SQS event source config. - /// - /// If the value of sqsEventSourceConfigString starts with "env:" then it assume the suffix of the value - /// is an environment variable containing the config. This is used by the .NET Aspire integration because - /// the values required for the config are resolved after command line arguments are setup in Aspire. /// /// /// /// internal static List LoadSQSEventSourceConfig(string sqsEventSourceConfigString) { - if (sqsEventSourceConfigString.StartsWith(Constants.ARGUMENT_ENVIRONMENT_VARIABLE_PREFIX, StringComparison.CurrentCultureIgnoreCase)) - { - var envVariable = sqsEventSourceConfigString.Substring(Constants.ARGUMENT_ENVIRONMENT_VARIABLE_PREFIX.Length); - if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(envVariable))) - { - throw new InvalidOperationException($"Environment variable {envVariable} for the SQS event source config was empty"); - } - sqsEventSourceConfigString = Environment.GetEnvironmentVariable(envVariable)!; - } - else if (File.Exists(sqsEventSourceConfigString)) + if (File.Exists(sqsEventSourceConfigString)) { sqsEventSourceConfigString = File.ReadAllText(sqsEventSourceConfigString); } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Amazon.Lambda.TestTool.IntegrationTests.csproj b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Amazon.Lambda.TestTool.IntegrationTests.csproj index 7c1381008..eda8904a5 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Amazon.Lambda.TestTool.IntegrationTests.csproj +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Amazon.Lambda.TestTool.IntegrationTests.csproj @@ -20,6 +20,7 @@ + diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs new file mode 100644 index 000000000..62932f0dd --- /dev/null +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -0,0 +1,389 @@ +using Amazon.Lambda.TestTool.Commands; +using Amazon.Lambda.TestTool.Commands.Settings; +using Amazon.Lambda.TestTool.Models; +using Amazon.Lambda.TestTool.Services.IO; +using Amazon.Lambda.TestTool.Services; +using Amazon.SQS; +using Moq; +using Spectre.Console.Cli; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; +using Amazon.Lambda.RuntimeSupport; +using Amazon.Lambda.Serialization.SystemTextJson; +using Amazon.Lambda.APIGatewayEvents; +using Amazon.Lambda.Core; +using Amazon.Lambda.SQSEvents; +using Amazon.SQS.Model; +using Amazon.Lambda.TestTool.Tests.Common; + +namespace Amazon.Lambda.TestTool.IntegrationTests; + +public class SQSEventSourceTests : BaseApiGatewayTest +{ + public SQSEventSourceTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper) + { + } + + [Fact] + public async Task ProcessSingleMessage() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + foreach(var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + await CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [Fact] + public async Task SQSEventSourceComesFromEnvironmentVariable() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + await CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [Fact] + public async Task ProcessMessagesFromMultipleEventSources() + { + var sqsClient = new AmazonSQSClient(); + var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; + var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; + + var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; + var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; + + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + + var sqsEventSourceConfig = """ +[ + { + "QueueUrl" : "queueUrl1", + "FunctionName" : "SQSProcessor" + }, + { + "QueueUrl" : "queueUrl2", + "FunctionName" : "SQSProcessor" + } +] +""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); + await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Equal(2, listOfProcessedMessages.Count); + Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); + } + finally + { + await CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl1); + await sqsClient.DeleteQueueAsync(queueUrl2); + Console.SetError(consoleError); + } + } + + [Fact] + public async Task MessageNotDeleted() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + await CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [Fact] + public async Task LambdaThrowsErrorAndMessageNotDeleted() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + + throw new Exception("Failed to process message"); + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + await CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [Fact] + public async Task PartialFailureResponse() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + await sqsClient.SendMessageAsync(queueUrl, "Message1"); + + var lambdaPort = GetFreePort(); + + // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + + var sqsResponse = new SQSBatchResponse(); + sqsResponse.BatchItemFailures = new List + { + new SQSBatchResponse.BatchItemFailure + { + ItemIdentifier = evnt.Records[0].MessageId + } + }; + + return sqsResponse; + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + // Since the message was never deleted by the event source it should still be eligibl for reading. + var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); + Assert.Single(response.Messages); + } + finally + { + await CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + + private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) + { + var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); + return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; + } + + private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) + { + Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); + + var environmentVariables = new Dictionary { }; + + if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) + { + var tokens = sqsEventSourceConfig.Split('&'); + if (tokens.Length == 2) + { + sqsEventSourceConfig = tokens[0]; + var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); + var envValue = tokens[1]; + environmentVariables[envName] = envValue; + } + } + + var settings = new RunCommandSettings + { + LambdaEmulatorPort = lambdaPort, + NoLaunchWindow = true, + SQSEventSourceConfig = sqsEventSourceConfig + }; + + var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); + var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); + _ = command.ExecuteAsync(context, settings, cancellationTokenSource); + + await Task.Delay(2000, cancellationTokenSource.Token); + } +} diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestEnvironmentManager.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestEnvironmentManager.cs new file mode 100644 index 000000000..d614a06fe --- /dev/null +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestEnvironmentManager.cs @@ -0,0 +1,8 @@ +using Amazon.Lambda.TestTool.Services.IO; + +namespace Amazon.Lambda.TestTool.Tests.Common; + +public class TestEnvironmentManager(System.Collections.IDictionary dictionary) : IEnvironmentManager +{ + public System.Collections.IDictionary GetEnvironmentVariables() => dictionary; +} From 7ef91deaa4b70f446f9b2ffdc4624e7cd558b102 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 11 Mar 2025 09:57:48 -0700 Subject: [PATCH 04/42] Add change log file --- .../changes/e390422f-955d-4699-97cf-67725872e746.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .autover/changes/e390422f-955d-4699-97cf-67725872e746.json diff --git a/.autover/changes/e390422f-955d-4699-97cf-67725872e746.json b/.autover/changes/e390422f-955d-4699-97cf-67725872e746.json new file mode 100644 index 000000000..e71fd5f0e --- /dev/null +++ b/.autover/changes/e390422f-955d-4699-97cf-67725872e746.json @@ -0,0 +1,11 @@ +{ + "Projects": [ + { + "Name": "Amazon.Lambda.TestTool", + "Type": "Minor", + "ChangelogMessages": [ + "Add SQS event source support" + ] + } + ] +} \ No newline at end of file From ae43475f4f489dc0ad5557282449c7541c62ede9 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 11 Mar 2025 10:13:40 -0700 Subject: [PATCH 05/42] Update switches description --- .../Commands/Settings/RunCommandSettings.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs index 873895e8e..1be04a4dc 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/Settings/RunCommandSettings.cs @@ -55,9 +55,10 @@ public sealed class RunCommandSettings : CommandSettings public int? ApiGatewayEmulatorPort { get; set; } /// - /// JSON configuration for an SQS event source that will poll messages from a queue and forward the messages to the events. + /// The configuration for the SQS event source. The format of the config is a comma delimited key pairs. For example \"QueueUrl=queue-url,FunctionName=function-name,VisibilityTimeout=100\". + /// Possible keys are: BatchSize, DisableMessageDelete, FunctionName, LambdaRuntimeApi, Profile, QueueUrl, Region, VisibilityTimeout /// - [CommandOption("--sqs-eventsource-config ")] - [Description("The JSON configuration for an SQS event source that will poll messages from a queue and forward the messages to the events. If the value is a file path the file will be read as the JSON value.")] + [CommandOption("--sqs-eventsource-config ")] + [Description("The configuration for the SQS event source. The format of the config is a comma delimited key pairs. For example \"QueueUrl=,FunctionName=,VisibilityTimeout=100\". Possible keys are: BatchSize, DisableMessageDelete, FunctionName, LambdaRuntimeApi, Profile, QueueUrl, Region, VisibilityTimeout")] public string? SQSEventSourceConfig { get; set; } } From 597d9432df8f2e6955de5a47d9f203554ce4d4d7 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 11 Mar 2025 11:12:03 -0700 Subject: [PATCH 06/42] Add delay to handle SQS eventual consistency --- .../SQSEventSourceTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 62932f0dd..6a2ade3a4 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -351,6 +351,8 @@ public async Task PartialFailureResponse() private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) { + // Add a delay to handle SQS eventual consistency. + await Task.Delay(5000); var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; } From fd1612f4b4ccb1fb5a367a40d5c25bedaaf79905 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 11 Mar 2025 13:21:58 -0700 Subject: [PATCH 07/42] Add dispose of TcpListener when looking for a free port. --- .../BaseApiGatewayTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index bdc9c0622..2cb175e09 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -156,7 +156,7 @@ protected int GetFreePort() { var random = new Random(); var port = random.Next(49152, 65535); - var listener = new TcpListener(IPAddress.Loopback, port); + using var listener = new TcpListener(IPAddress.Loopback, port); try { listener.Start(); From de271ac18f98d226e8e6b9022805100b650b3268 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 11 Mar 2025 14:25:36 -0700 Subject: [PATCH 08/42] Rework how random ports are picked in tests --- .../BaseApiGatewayTest.cs | 5 ++-- .../Helpers/TestHelpers.cs | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index 2cb175e09..ed3767b78 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -14,6 +14,8 @@ using System.Text; using System.Net.Http; using System.Net.Http.Headers; +using System.Security.Cryptography; +using Amazon.Lambda.TestTool.Tests.Common.Helpers; namespace Amazon.Lambda.TestTool.IntegrationTests; @@ -154,8 +156,7 @@ protected async Task TestEndpoint(string routeName, int api protected int GetFreePort() { - var random = new Random(); - var port = random.Next(49152, 65535); + var port = TestHelpers.GetRandomIntegerInRange(49152, 65535); using var listener = new TcpListener(IPAddress.Loopback, port); try { diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/Helpers/TestHelpers.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/Helpers/TestHelpers.cs index 79c618d48..71c8b14a4 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/Helpers/TestHelpers.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/Helpers/TestHelpers.cs @@ -1,6 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +using System.Security.Cryptography; + namespace Amazon.Lambda.TestTool.Tests.Common.Helpers; public static class TestHelpers @@ -51,4 +53,25 @@ public static int GetNextApiGatewayPort() { return Interlocked.Increment(ref _maxApiGatewayPort); } + + private static RandomNumberGenerator rng = RandomNumberGenerator.Create(); + public static int GetRandomIntegerInRange(int minValue, int maxValue) + { + if (minValue >= maxValue) + throw new ArgumentOutOfRangeException(nameof(minValue), "minValue must be less than maxValue"); + + // Create a byte array to hold the random bytes + byte[] randomBytes = new byte[4]; + + // Fill the array with random bytes + rng.GetBytes(randomBytes); + + // Convert the bytes to an integer + int randomInteger = BitConverter.ToInt32(randomBytes, 0); + + // Make sure the random integer is within the desired range + // Apply modulus to get a number in the range [0, maxValue - minValue] + int range = maxValue - minValue; + return (Math.Abs(randomInteger) % range) + minValue; + } } From e484507716a620c73a6f4b0aeacf128d1f60eed6 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 11 Mar 2025 14:51:03 -0700 Subject: [PATCH 09/42] Add collection attribute to SQS integ tests --- .../SQSEventSourceTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 6a2ade3a4..8f37ac639 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -19,6 +19,7 @@ namespace Amazon.Lambda.TestTool.IntegrationTests; +[Collection("SQSEventSourceTests")] public class SQSEventSourceTests : BaseApiGatewayTest { public SQSEventSourceTests(ITestOutputHelper testOutputHelper) From 128befde04116f29db82a1cc3834b81d48c1ddcc Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 11 Mar 2025 15:14:20 -0700 Subject: [PATCH 10/42] Add [RetryFact] for new SQS integ tests --- .../Amazon.Lambda.TestTool.csproj | 4 ++-- .../BaseApiGatewayTest.cs | 2 +- .../BasicApiGatewayTests.cs | 6 +++--- .../BinaryResponseTests.cs | 2 +- .../EdgeCaseTests.cs | 4 ++-- .../LargePayloadTests.cs | 4 ++-- .../SQSEventSourceTests.cs | 13 +++++++------ 7 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj index 081f1c76a..b0e270f1a 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj @@ -1,4 +1,4 @@ - + @@ -15,7 +15,7 @@ true Amazon.Lambda.TestTool dotnet-lambda-test-tool - 0.9.1 + 0.9.888 NU5100 Major README.md diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index ed3767b78..313302b8b 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -142,7 +142,7 @@ protected async Task TestEndpoint(string routeName, int api } } - protected async Task<(int lambdaPort, int apiGatewayPort)> GetFreePorts() + protected (int lambdaPort, int apiGatewayPort) GetFreePorts() { var lambdaPort = GetFreePort(); int apiGatewayPort; diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs index a61ef07fa..edd0f49fd 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs @@ -23,7 +23,7 @@ public BasicApiGatewayTests(ITestOutputHelper testOutputHelper) : base(testOutpu [RetryFact] public async Task TestLambdaToUpperV2() { - var ports = await GetFreePorts(); + var ports = GetFreePorts(); var lambdaPort = ports.lambdaPort; var apiGatewayPort = ports.apiGatewayPort; @@ -68,7 +68,7 @@ public async Task TestLambdaToUpperV2() [RetryFact] public async Task TestLambdaToUpperRest() { - var ports = await GetFreePorts(); + var ports = GetFreePorts(); var lambdaPort = ports.lambdaPort; var apiGatewayPort = ports.apiGatewayPort; @@ -113,7 +113,7 @@ public async Task TestLambdaToUpperRest() [RetryFact] public async Task TestLambdaToUpperV1() { - var ports = await GetFreePorts(); + var ports = GetFreePorts(); var lambdaPort = ports.lambdaPort; var apiGatewayPort = ports.apiGatewayPort; diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs index 3dc575d76..23f46a472 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs @@ -23,7 +23,7 @@ public BinaryResponseTests(ITestOutputHelper testOutputHelper) : base(testOutput [RetryFact] public async Task TestLambdaBinaryResponse() { - var ports = await GetFreePorts(); + var ports = GetFreePorts(); var lambdaPort = ports.lambdaPort; var apiGatewayPort = ports.apiGatewayPort; diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs index a0d830a2f..6c5fccfb6 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs @@ -23,7 +23,7 @@ public EdgeCaseTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper [RetryFact] public async Task TestLambdaReturnString() { - var ports = await GetFreePorts(); + var ports = GetFreePorts(); var lambdaPort = ports.lambdaPort; var apiGatewayPort = ports.apiGatewayPort; @@ -64,7 +64,7 @@ public async Task TestLambdaReturnString() [RetryFact] public async Task TestLambdaWithNullEndpoint() { - var ports = await GetFreePorts(); + var ports = GetFreePorts(); var lambdaPort = ports.lambdaPort; var apiGatewayPort = ports.apiGatewayPort; diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs index 4a8628eed..c7faf251d 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs @@ -24,7 +24,7 @@ public LargePayloadTests(ITestOutputHelper testOutputHelper) : base(testOutputHe [InlineData(ApiGatewayEmulatorMode.HttpV1)] public async Task TestLambdaWithLargeRequestPayload_RestAndV1(ApiGatewayEmulatorMode mode) { - var ports = await GetFreePorts(); + var ports = GetFreePorts(); var lambdaPort = ports.lambdaPort; var apiGatewayPort = ports.apiGatewayPort; @@ -71,7 +71,7 @@ public async Task TestLambdaWithLargeRequestPayload_RestAndV1(ApiGatewayEmulator [Fact] public async Task TestLambdaWithLargeRequestPayload_HttpV2() { - var ports = await GetFreePorts(); + var ports = GetFreePorts(); var lambdaPort = ports.lambdaPort; var apiGatewayPort = ports.apiGatewayPort; diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 8f37ac639..913e83b4b 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -16,6 +16,7 @@ using Amazon.Lambda.SQSEvents; using Amazon.SQS.Model; using Amazon.Lambda.TestTool.Tests.Common; +using Amazon.Lambda.TestTool.Tests.Common.Retries; namespace Amazon.Lambda.TestTool.IntegrationTests; @@ -27,7 +28,7 @@ public SQSEventSourceTests(ITestOutputHelper testOutputHelper) { } - [Fact] + [RetryFact] public async Task ProcessSingleMessage() { var sqsClient = new AmazonSQSClient(); @@ -75,7 +76,7 @@ public async Task ProcessSingleMessage() } } - [Fact] + [RetryFact] public async Task SQSEventSourceComesFromEnvironmentVariable() { var sqsClient = new AmazonSQSClient(); @@ -123,7 +124,7 @@ public async Task SQSEventSourceComesFromEnvironmentVariable() } } - [Fact] + [RetryFact] public async Task ProcessMessagesFromMultipleEventSources() { var sqsClient = new AmazonSQSClient(); @@ -189,7 +190,7 @@ public async Task ProcessMessagesFromMultipleEventSources() } } - [Fact] + [RetryFact] public async Task MessageNotDeleted() { var sqsClient = new AmazonSQSClient(); @@ -237,7 +238,7 @@ public async Task MessageNotDeleted() } } - [Fact] + [RetryFact] public async Task LambdaThrowsErrorAndMessageNotDeleted() { var sqsClient = new AmazonSQSClient(); @@ -287,7 +288,7 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() } } - [Fact] + [RetryFact] public async Task PartialFailureResponse() { var sqsClient = new AmazonSQSClient(); From 016f318a2bce58c2d6b1caa3a69d44b1a2080d2e Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 11 Mar 2025 15:45:26 -0700 Subject: [PATCH 11/42] Do more checks to see if port is available --- .../BaseApiGatewayTest.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index 313302b8b..f65604d0c 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -16,6 +16,7 @@ using System.Net.Http.Headers; using System.Security.Cryptography; using Amazon.Lambda.TestTool.Tests.Common.Helpers; +using Microsoft.AspNetCore.Hosting.Server; namespace Amazon.Lambda.TestTool.IntegrationTests; @@ -161,6 +162,11 @@ protected int GetFreePort() try { listener.Start(); + using TcpClient client = new TcpClient("127.0.0.1", port); + using NetworkStream stream = client.GetStream(); + stream.WriteByte(0x01); + stream.Flush(); + return port; } catch (SocketException) From 2eca32124a801dcc58148b64a50bdca5ae402310 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 11 Mar 2025 17:17:24 -0700 Subject: [PATCH 12/42] Add more logging for tests. --- .../Commands/RunCommand.cs | 4 +-- .../Processes/ApiGatewayEmulatorProcess.cs | 7 ++++- .../Processes/TestToolProcess.cs | 4 ++- .../ToolInteractiveLoggerProvider.cs | 31 +++++++++++++++++++ .../BaseApiGatewayTest.cs | 13 +++++--- .../SQSEventSourceTests.cs | 17 +++++----- .../TestOutputToolInteractiveService.cs | 31 +++++++++++++++++++ .../ApiGatewayEmulatorProcessTests.cs | 6 ++-- .../RuntimeApiTests.cs | 8 +++-- 9 files changed, 99 insertions(+), 22 deletions(-) create mode 100644 Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Utilities/ToolInteractiveLoggerProvider.cs create mode 100644 Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs index b90fa3e81..a046bad12 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs @@ -42,7 +42,7 @@ public override async Task ExecuteAsync(CommandContext context, RunCommandS if (settings.LambdaEmulatorPort.HasValue) { - var testToolProcess = TestToolProcess.Startup(settings, cancellationTokenSource.Token); + var testToolProcess = TestToolProcess.Startup(settings, toolInteractiveService, cancellationTokenSource.Token); tasks.Add(testToolProcess.RunningTask); if (!settings.NoLaunchWindow) @@ -71,7 +71,7 @@ public override async Task ExecuteAsync(CommandContext context, RunCommandS } var apiGatewayEmulatorProcess = - ApiGatewayEmulatorProcess.Startup(settings, cancellationTokenSource.Token); + ApiGatewayEmulatorProcess.Startup(settings, toolInteractiveService, cancellationTokenSource.Token); tasks.Add(apiGatewayEmulatorProcess.RunningTask); } diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/ApiGatewayEmulatorProcess.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/ApiGatewayEmulatorProcess.cs index 0ddb448a1..9694a5afd 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/ApiGatewayEmulatorProcess.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/ApiGatewayEmulatorProcess.cs @@ -40,7 +40,7 @@ public class ApiGatewayEmulatorProcess /// /// Creates the Web API and runs it in the background. /// - public static ApiGatewayEmulatorProcess Startup(RunCommandSettings settings, CancellationToken cancellationToken = default) + public static ApiGatewayEmulatorProcess Startup(RunCommandSettings settings, IToolInteractiveService toolInteractiveService, CancellationToken cancellationToken = default) { if (settings.ApiGatewayEmulatorMode is null) { @@ -62,6 +62,11 @@ public static ApiGatewayEmulatorProcess Startup(RunCommandSettings settings, Can var app = builder.Build(); + if (!app.Environment.IsProduction()) + { + app.Services.GetRequiredService().AddProvider(new ToolInteractiveLoggerProvider(toolInteractiveService)); + } + app.MapHealthChecks("/__lambda_test_tool_apigateway_health__"); app.Lifetime.ApplicationStarted.Register(() => diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/TestToolProcess.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/TestToolProcess.cs index 5309315c4..38eb88870 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/TestToolProcess.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/TestToolProcess.cs @@ -37,7 +37,7 @@ public class TestToolProcess /// /// Creates the Web Application and runs it in the background. /// - public static TestToolProcess Startup(RunCommandSettings settings, CancellationToken cancellationToken = default) + public static TestToolProcess Startup(RunCommandSettings settings, IToolInteractiveService toolInteractiveService, CancellationToken cancellationToken = default) { var builder = WebApplication.CreateBuilder(); @@ -82,6 +82,8 @@ public static TestToolProcess Startup(RunCommandSettings settings, CancellationT } else { + app.Services.GetRequiredService().AddProvider(new ToolInteractiveLoggerProvider(toolInteractiveService)); + // nosemgrep: csharp.lang.security.stacktrace-disclosure.stacktrace-disclosure app.UseDeveloperExceptionPage(); app.UseStaticFiles(); diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Utilities/ToolInteractiveLoggerProvider.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Utilities/ToolInteractiveLoggerProvider.cs new file mode 100644 index 000000000..f29f73719 --- /dev/null +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Utilities/ToolInteractiveLoggerProvider.cs @@ -0,0 +1,31 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Microsoft.Extensions.Logging.Abstractions; +using Amazon.Lambda.TestTool.Services; + +namespace Amazon.Lambda.TestTool.Utilities; + +internal class ToolInteractiveLoggerProvider(IToolInteractiveService toolInteractiveSerivce) : ILoggerProvider +{ + private Logger _logger = new Logger(toolInteractiveSerivce); + public ILogger CreateLogger(string categoryName) => _logger; + public void Dispose() { } + + class Logger(IToolInteractiveService toolInteractiveSerivce) : ILogger + { + public IDisposable? BeginScope(TState state) + where TState : notnull + { + return null; + } + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + var message = formatter(state, exception); + toolInteractiveSerivce.WriteLine(message); + } + } +} diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index f65604d0c..faffe1a84 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -17,24 +17,29 @@ using System.Security.Cryptography; using Amazon.Lambda.TestTool.Tests.Common.Helpers; using Microsoft.AspNetCore.Hosting.Server; +using Amazon.Lambda.TestTool.Tests.Common; namespace Amazon.Lambda.TestTool.IntegrationTests; public abstract class BaseApiGatewayTest { protected readonly ITestOutputHelper TestOutputHelper; + protected readonly TestOutputToolInteractiveService InteractiveService; protected readonly Mock MockEnvironmentManager; - protected readonly Mock MockInteractiveService; protected readonly Mock MockRemainingArgs; protected CancellationTokenSource CancellationTokenSource; protected static readonly object TestLock = new(); protected BaseApiGatewayTest(ITestOutputHelper testOutputHelper) { + // Enable the intneral logging of runtime support. + Environment.SetEnvironmentVariable("LAMBDA_RUNTIMESUPPORT_DEBUG", "true"); + TestOutputHelper = testOutputHelper; MockEnvironmentManager = new Mock(); - MockInteractiveService = new Mock(); MockRemainingArgs = new Mock(); + + InteractiveService = new TestOutputToolInteractiveService(testOutputHelper); CancellationTokenSource = new CancellationTokenSource(); } @@ -66,7 +71,7 @@ protected async Task StartTestToolProcessAsync(ApiGatewayEmulatorMode apiGateway ApiGatewayEmulatorPort = apiGatewayPort }; - var command = new RunCommand(MockInteractiveService.Object, MockEnvironmentManager.Object); + var command = new RunCommand(InteractiveService, MockEnvironmentManager.Object); var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); _ = command.ExecuteAsync(context, settings, cancellationTokenSource); @@ -196,7 +201,7 @@ protected async Task StartTestToolProcessWithNullEndpoint(ApiGatewayEmulatorMode ApiGatewayEmulatorPort = apiGatewayPort }; - var command = new RunCommand(MockInteractiveService.Object, MockEnvironmentManager.Object); + var command = new RunCommand(InteractiveService, MockEnvironmentManager.Object); var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); _ = command.ExecuteAsync(context, settings, cancellationTokenSource); diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 913e83b4b..3bba55f21 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -1,17 +1,11 @@ using Amazon.Lambda.TestTool.Commands; using Amazon.Lambda.TestTool.Commands.Settings; -using Amazon.Lambda.TestTool.Models; -using Amazon.Lambda.TestTool.Services.IO; -using Amazon.Lambda.TestTool.Services; using Amazon.SQS; -using Moq; using Spectre.Console.Cli; using Xunit; using Xunit.Abstractions; -using Xunit.Sdk; using Amazon.Lambda.RuntimeSupport; using Amazon.Lambda.Serialization.SystemTextJson; -using Amazon.Lambda.APIGatewayEvents; using Amazon.Lambda.Core; using Amazon.Lambda.SQSEvents; using Amazon.SQS.Model; @@ -45,7 +39,8 @@ public async Task ProcessSingleMessage() var listOfProcessedMessages = new List(); var handler = (SQSEvent evnt, ILambdaContext context) => { - foreach(var message in evnt.Records) + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) { listOfProcessedMessages.Add(message); } @@ -93,6 +88,7 @@ public async Task SQSEventSourceComesFromEnvironmentVariable() var listOfProcessedMessages = new List(); var handler = (SQSEvent evnt, ILambdaContext context) => { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); foreach (var message in evnt.Records) { listOfProcessedMessages.Add(message); @@ -158,6 +154,7 @@ public async Task ProcessMessagesFromMultipleEventSources() var listOfProcessedMessages = new List(); var handler = (SQSEvent evnt, ILambdaContext context) => { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); foreach (var message in evnt.Records) { listOfProcessedMessages.Add(message); @@ -207,6 +204,7 @@ public async Task MessageNotDeleted() var listOfProcessedMessages = new List(); var handler = (SQSEvent evnt, ILambdaContext context) => { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); foreach (var message in evnt.Records) { listOfProcessedMessages.Add(message); @@ -248,13 +246,13 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() try { Console.SetError(TextWriter.Null); - var lambdaPort = GetFreePort(); var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); var listOfProcessedMessages = new List(); var handler = (SQSEvent evnt, ILambdaContext context) => { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); foreach (var message in evnt.Records) { listOfProcessedMessages.Add(message); @@ -308,6 +306,7 @@ public async Task PartialFailureResponse() var listOfProcessedMessages = new List(); var handler = (SQSEvent evnt, ILambdaContext context) => { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); foreach (var message in evnt.Records) { listOfProcessedMessages.Add(message); @@ -384,7 +383,7 @@ private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSour SQSEventSourceConfig = sqsEventSourceConfig }; - var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); + var command = new RunCommand(InteractiveService, new TestEnvironmentManager(environmentVariables)); var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); _ = command.ExecuteAsync(context, settings, cancellationTokenSource); diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs new file mode 100644 index 000000000..a2da09af5 --- /dev/null +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs @@ -0,0 +1,31 @@ +using Amazon.Lambda.TestTool.Services; +using Xunit.Abstractions; + + +namespace Amazon.Lambda.TestTool.Tests.Common; + +public class TestOutputToolInteractiveService(ITestOutputHelper testOutputHelper) : IToolInteractiveService +{ + public void WriteErrorLine(string? message) + { + try + { + testOutputHelper.WriteLine("Error: " + message); + } + catch (Exception) + { + // This can happen when Xunit thinks where is no active test + } + } + public void WriteLine(string? message) + { + try + { + testOutputHelper.WriteLine(message); + } + catch(Exception) + { + // This can happen when Xunit thinks where is no active test + } + } +} diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/Processes/ApiGatewayEmulatorProcessTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/Processes/ApiGatewayEmulatorProcessTests.cs index f6d80d58b..e1ca45511 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/Processes/ApiGatewayEmulatorProcessTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/Processes/ApiGatewayEmulatorProcessTests.cs @@ -5,12 +5,14 @@ using Amazon.Lambda.TestTool.Commands.Settings; using Amazon.Lambda.TestTool.Models; using Amazon.Lambda.TestTool.Processes; +using Amazon.Lambda.TestTool.Tests.Common; using Amazon.Lambda.TestTool.Tests.Common.Helpers; using Xunit; +using Xunit.Abstractions; namespace Amazon.Lambda.TestTool.UnitTests.Processes; -public class ApiGatewayEmulatorProcessTests +public class ApiGatewayEmulatorProcessTests(ITestOutputHelper testOutputHelper) { [Theory] [InlineData(ApiGatewayEmulatorMode.Rest, HttpStatusCode.Forbidden, "{\"message\":\"Missing Authentication Token\"}")] @@ -27,7 +29,7 @@ public async Task RouteNotFound(ApiGatewayEmulatorMode mode, HttpStatusCode stat var apiUrl = $"http://{settings.LambdaEmulatorHost}:{settings.ApiGatewayEmulatorPort}/__lambda_test_tool_apigateway_health__"; // Act - var process = ApiGatewayEmulatorProcess.Startup(settings, cancellationSource.Token); + var process = ApiGatewayEmulatorProcess.Startup(settings, new TestOutputToolInteractiveService(testOutputHelper), cancellationSource.Token); var isApiRunning = await TestHelpers.WaitForApiToStartAsync(apiUrl); var response = await TestHelpers.SendRequest($"{process.ServiceUrl}/invalid"); await cancellationSource.CancelAsync(); diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/RuntimeApiTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/RuntimeApiTests.cs index d03c6a8e6..ca8434ec8 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/RuntimeApiTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/RuntimeApiTests.cs @@ -14,10 +14,12 @@ using Microsoft.Extensions.DependencyInjection; using Xunit; using Environment = System.Environment; +using Xunit.Abstractions; +using Amazon.Lambda.TestTool.Tests.Common; namespace Amazon.Lambda.TestTool.UnitTests; -public class RuntimeApiTests +public class RuntimeApiTests(ITestOutputHelper testOutputHelper) { #if DEBUG [Fact] @@ -34,7 +36,7 @@ public async Task AddEventToDataStore() var options = new RunCommandSettings(); options.LambdaEmulatorPort = lambdaPort; Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); - var testToolProcess = TestToolProcess.Startup(options, cancellationTokenSource.Token); + var testToolProcess = TestToolProcess.Startup(options, new TestOutputToolInteractiveService(testOutputHelper), cancellationTokenSource.Token); try { var lambdaClient = ConstructLambdaServiceClient(testToolProcess.ServiceUrl); @@ -87,7 +89,7 @@ public async Task InvokeRequestResponse() var options = new RunCommandSettings(); options.LambdaEmulatorPort = lambdaPort; Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); - var testToolProcess = TestToolProcess.Startup(options, cancellationTokenSource.Token); + var testToolProcess = TestToolProcess.Startup(options, new TestOutputToolInteractiveService(testOutputHelper), cancellationTokenSource.Token); try { var handler = (string input, ILambdaContext context) => From 0bc3a19901c9b71e804d88b9e54f92082d186476 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 13 Mar 2025 13:16:35 -0700 Subject: [PATCH 13/42] Add started polling info log message --- .../Processes/SQSEventSource/SQSEventSourceBackgroundService.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs index 1fe8d753e..32d52462f 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs @@ -65,6 +65,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { // The queue arn is needed for creating the Lambda event. var queueArn = await GetQueueArn(stoppingToken); + _logger.LogInformation("Starting polling for messages on SQS queue: {queueArn}", queueArn); while (!stoppingToken.IsCancellationRequested) { try From 3108beb8e2636d5c478b3f791cb26761b1a34eda Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Sat, 22 Mar 2025 00:50:04 -0700 Subject: [PATCH 14/42] Address PR comments --- .../Amazon.Lambda.TestTool.csproj | 8 ++-- .../src/Amazon.Lambda.TestTool/Constants.cs | 2 +- .../SQSEventSourceBackgroundService.cs | 48 +++++++++---------- .../SQSEventSourceBackgroundServiceConfig.cs | 6 +-- .../SQSEventSource/SQSEventSourceConfig.cs | 5 +- .../SQSEventSource/SQSEventSourceProcess.cs | 19 ++++---- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj index b0e270f1a..1841f48fc 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Amazon.Lambda.TestTool.csproj @@ -2,7 +2,7 @@ - Exe + Exe A tool to help debug and test your .NET AWS Lambda functions locally. net8.0 enable @@ -15,9 +15,9 @@ true Amazon.Lambda.TestTool dotnet-lambda-test-tool - 0.9.888 + 0.9.1 NU5100 - Major + Major README.md @@ -41,7 +41,7 @@ - + diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs index abd63eed7..680f99398 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Constants.cs @@ -86,7 +86,7 @@ public abstract class Constants public const string LinkVsToolkitMarketplace = "https://marketplace.visualstudio.com/items?itemName=AmazonWebServices.AWSToolkitforVisualStudio2022"; /// - /// Prefix used for values of command line arguments that support the value being stored in an environment variable. + /// Prefix this is used for values of command line arguments that support the value being stored in an environment variable. /// This used in the Aspire integration where it is often easier to pass configuration via environment variables. /// public const string ArgumentEnvironmentVariablePrefix = "env:"; diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs index 32d52462f..0d8f79ece 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs @@ -7,11 +7,12 @@ using Amazon.SQS.Model; using Amazon.SQS; using System.Text.Json; +using Amazon.Lambda.TestTool.Services; namespace Amazon.Lambda.TestTool.Processes.SQSEventSource; /// -/// IHostedService that will run continuially polling the SQS queue for messages and invoking the connected +/// IHostedService that will run continually polling the SQS queue for messages and invoking the connected /// Lambda function with the polled messages. /// public class SQSEventSourceBackgroundService : BackgroundService @@ -24,25 +25,22 @@ public class SQSEventSourceBackgroundService : BackgroundService private readonly ILogger _logger; private readonly IAmazonSQS _sqsClient; - private readonly IAmazonLambda _lambdaClient; + private readonly ILambdaClient _lambdaClient; private readonly SQSEventSourceBackgroundServiceConfig _config; /// - /// Constructs instance of SQSEventSourceBackgroundService. + /// Constructs instance of . /// - /// - /// - /// - public SQSEventSourceBackgroundService(ILogger logger, IAmazonSQS sqsClient, SQSEventSourceBackgroundServiceConfig config) + /// The logger + /// The SQS client used to poll messages from a queue. + /// The config of the service + /// The Lambda client that can use a different endpoint for each invoke request. + public SQSEventSourceBackgroundService(ILogger logger, IAmazonSQS sqsClient, SQSEventSourceBackgroundServiceConfig config, ILambdaClient lambdaClient) { _logger = logger; _sqsClient = sqsClient; _config = config; - - _lambdaClient = new AmazonLambdaClient(new BasicAWSCredentials("accessKey", "secretKey"), new AmazonLambdaConfig - { - ServiceURL = _config.LambdaRuntimeApi - }); + _lambdaClient = lambdaClient; } private async Task GetQueueArn(CancellationToken stoppingToken) @@ -59,8 +57,8 @@ private async Task GetQueueArn(CancellationToken stoppingToken) /// /// Execute the SQSEventSourceBackgroundService. /// - /// - /// + /// CancellationToken used to end the service. + /// Task for the background service. protected override async Task ExecuteAsync(CancellationToken stoppingToken) { // The queue arn is needed for creating the Lambda event. @@ -107,7 +105,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) }; _logger.LogInformation("Invoking Lambda function {functionName} function with {messageCount} messages", _config.FunctionName, lambdaPayload.Records.Count); - var lambdaResponse = await _lambdaClient.InvokeAsync(invokeRequest, stoppingToken); + var lambdaResponse = await _lambdaClient.InvokeAsync(invokeRequest, _config.LambdaRuntimeApi); if (lambdaResponse.FunctionError != null) { @@ -188,22 +186,22 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) /// /// Convert from the SDK's list of messages to the Lambda event's SQS message type. /// - /// - /// - /// - /// - internal static List ConvertToLambdaMessages(List message, string awsRegion, string queueArn) + /// List of messages using the SDK's .NET type + /// The aws region the messages came from. + /// The SQS queue arn the messages came from. + /// List of messages using the Lambda event's .NET type. + internal static List ConvertToLambdaMessages(List messages, string awsRegion, string queueArn) { - return message.Select(m => ConvertToLambdaMessage(m, awsRegion, queueArn)).ToList(); + return messages.Select(m => ConvertToLambdaMessage(m, awsRegion, queueArn)).ToList(); } /// /// Convert from the SDK's SQS message to the Lambda event's SQS message type. /// - /// - /// - /// - /// + /// Message using the SDK's .NET type + /// The aws region the message came from. + /// The SQS queue arn the message came from. + /// Messages using the Lambda event's .NET type. internal static SQSEvent.SQSMessage ConvertToLambdaMessage(Message message, string awsRegion, string queueArn) { var lambdaMessage = new SQSEvent.SQSMessage diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundServiceConfig.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundServiceConfig.cs index 118c19386..b07cc386c 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundServiceConfig.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundServiceConfig.cs @@ -4,18 +4,18 @@ namespace Amazon.Lambda.TestTool.Processes.SQSEventSource; /// -/// Configuration for the SQSEventSourceBackgroundService service. +/// Configuration for the service. /// public class SQSEventSourceBackgroundServiceConfig { /// /// The batch size to read and send to Lambda function. This is the upper bound of messages to read and send. - /// SQS will return with less then batch size if there are not enough messages in the queue. + /// SQS will return with less than batch size if there are not enough messages in the queue. /// public required int BatchSize { get; init; } = SQSEventSourceProcess.DefaultBatchSize; /// - /// If true the SQSEventSourceBackgroundService will skip deleting messages from the queue after the Lambda function returns. + /// If true the will skip deleting messages from the queue after the Lambda function returns. /// public required bool DisableMessageDelete { get; init; } diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceConfig.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceConfig.cs index 6a6fc7e61..471fdfcc8 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceConfig.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceConfig.cs @@ -10,12 +10,12 @@ internal class SQSEventSourceConfig { /// /// The batch size to read and send to Lambda function. This is the upper bound of messages to read and send. - /// SQS will return with less then batch size if there are not enough messages in the queue. + /// SQS will return with less than batch size if there are not enough messages in the queue. /// public int? BatchSize { get; set; } /// - /// If true the SQSEventSourceBackgroundService will skip deleting messages from the queue after the Lambda function returns. + /// If true the will skip deleting messages from the queue after the Lambda function returns. /// public bool? DisableMessageDelete { get; set; } @@ -30,6 +30,7 @@ internal class SQSEventSourceConfig /// If not set the current Test Tool instance will be used assuming it is running a Lambda runtime api emulator. /// public string? LambdaRuntimeApi { get; set; } + /// /// The AWS profile to use for credentials for fetching messages from the queue. /// diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs index 3848d4c42..2b7272cad 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceProcess.cs @@ -17,17 +17,17 @@ public class SQSEventSourceProcess internal const int DefaultVisiblityTimeout = 30; /// - /// The Parent task for all of the tasks started for each list SQS event source. + /// The Parent task for all the tasks started for each list SQS event source. /// public required Task RunningTask { get; init; } /// /// Startup SQS event sources /// - /// - /// - /// - /// + /// The settings to launch the tool. + /// CancellationToken to end + /// This instance of that was started. + /// Thrown when the config is invalid. public static SQSEventSourceProcess Startup(RunCommandSettings settings, CancellationToken cancellationToken = default) { if (string.IsNullOrEmpty(settings.SQSEventSourceConfig)) @@ -57,6 +57,7 @@ public static SQSEventSourceProcess Startup(RunCommandSettings settings, Cancell var sqsClient = new AmazonSQSClient(sqsConfig); builder.Services.AddSingleton(sqsClient); + builder.Services.AddSingleton(); var queueUrl = sqsEventSourceConfig.QueueUrl; if (string.IsNullOrEmpty(queueUrl)) @@ -105,15 +106,15 @@ public static SQSEventSourceProcess Startup(RunCommandSettings settings, Cancell /// With the JSON format it is possible to configure multiple event sources but special care is required /// escaping the quotes. The JSON format also provides consistency with the API Gateway configuration. /// - /// The comma delimited key pairs allows users to configure a single SQS event source without having + /// The comma-delimited key pairs allows users to configure a single SQS event source without having /// to deal with escaping quotes. /// /// If the value of sqsEventSourceConfigString points to a file that exists the contents of the file /// will be read and sued for the value for SQS event source config. /// - /// - /// - /// + /// The SQS event source config to load. + /// The list of SQS event source configs. + /// Thrown when the config is invalid. internal static List LoadSQSEventSourceConfig(string sqsEventSourceConfigString) { if (File.Exists(sqsEventSourceConfigString)) From eeee081fedf75b0e4b28e97a370334eba46fbe62 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 2 Apr 2025 17:18:33 -0700 Subject: [PATCH 15/42] Cancel polling for next invocations if server if request has been canceled. --- .../src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs index 3f9666896..0aa041b48 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs @@ -132,7 +132,7 @@ public async Task GetNextInvocation(HttpContext ctx, string functionName) } else { - while (!runtimeDataStore.TryActivateEvent(out activeEvent)) + while (!runtimeDataStore.TryActivateEvent(out activeEvent) && !ctx.RequestAborted.IsCancellationRequested) { await Task.Delay(TimeSpan.FromMilliseconds(100)); } From 2d37ce731ea70ee694b056a4fdf47d14537ba348 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 2 Apr 2025 18:08:50 -0700 Subject: [PATCH 16/42] When cleaning up test instances don't block waiting for the cancel to complete --- .../BaseApiGatewayTest.cs | 3 +-- .../BasicApiGatewayTests.cs | 6 +++--- .../BinaryResponseTests.cs | 2 +- .../EdgeCaseTests.cs | 4 ++-- .../LargePayloadTests.cs | 4 ++-- .../SQSEventSourceTests.cs | 12 ++++++------ 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index faffe1a84..4c9106905 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -47,8 +47,7 @@ protected async Task CleanupAsync() { if (CancellationTokenSource != null) { - await CancellationTokenSource.CancelAsync(); - CancellationTokenSource.Dispose(); + _ = CancellationTokenSource.CancelAsync(); CancellationTokenSource = new CancellationTokenSource(); } } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs index edd0f49fd..825dd0269 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs @@ -60,7 +60,7 @@ public async Task TestLambdaToUpperV2() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } @@ -105,7 +105,7 @@ public async Task TestLambdaToUpperRest() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } @@ -150,7 +150,7 @@ public async Task TestLambdaToUpperV1() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs index 23f46a472..de2dd021c 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs @@ -78,7 +78,7 @@ public async Task TestLambdaBinaryResponse() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs index 6c5fccfb6..29301680e 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs @@ -56,7 +56,7 @@ public async Task TestLambdaReturnString() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } @@ -101,7 +101,7 @@ public async Task TestLambdaWithNullEndpoint() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs index c7faf251d..eb8bd959c 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs @@ -63,7 +63,7 @@ public async Task TestLambdaWithLargeRequestPayload_RestAndV1(ApiGatewayEmulator } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } @@ -110,7 +110,7 @@ public async Task TestLambdaWithLargeRequestPayload_HttpV2() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 3bba55f21..a2be674cd 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -65,7 +65,7 @@ public async Task ProcessSingleMessage() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } @@ -114,7 +114,7 @@ public async Task SQSEventSourceComesFromEnvironmentVariable() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } @@ -180,7 +180,7 @@ public async Task ProcessMessagesFromMultipleEventSources() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl1); await sqsClient.DeleteQueueAsync(queueUrl2); Console.SetError(consoleError); @@ -230,7 +230,7 @@ public async Task MessageNotDeleted() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } @@ -280,7 +280,7 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } @@ -343,7 +343,7 @@ public async Task PartialFailureResponse() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } From e5121e454e33ae37629323a6829236786ba10560 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 2 Apr 2025 18:43:12 -0700 Subject: [PATCH 17/42] Rework the RunCommand block on running tasks to take into account the cancellation token --- .../src/Amazon.Lambda.TestTool/Commands/RunCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs index a046bad12..ac55613a4 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs @@ -81,7 +81,7 @@ public override async Task ExecuteAsync(CommandContext context, RunCommandS tasks.Add(sqsEventSourceProcess.RunningTask); } - await Task.WhenAny(tasks); + await Task.Run(() => Task.WaitAny(tasks.ToArray(), cancellationTokenSource.Token)); return CommandReturnCodes.Success; } From 484a88504a718b121e67cf43b598d63c5483f88c Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 2 Apr 2025 19:49:11 -0700 Subject: [PATCH 18/42] Shutdown cleanup --- .../src/Amazon.Lambda.TestTool/Commands/RunCommand.cs | 4 ++++ .../src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs index ac55613a4..9e0c70d19 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs @@ -85,6 +85,10 @@ public override async Task ExecuteAsync(CommandContext context, RunCommandS return CommandReturnCodes.Success; } + catch (OperationCanceledException) when (cancellationTokenSource.IsCancellationRequested) + { + return CommandReturnCodes.Success; + } catch (Exception e) when (e.IsExpectedException()) { toolInteractiveService.WriteErrorLine(string.Empty); diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs index 0aa041b48..f9d11aa54 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs @@ -119,7 +119,7 @@ public async Task GetNextInvocation(HttpContext ctx, string functionName) { var runtimeDataStore = _runtimeApiDataStoreManager.GetLambdaRuntimeDataStore(functionName); - EventContainer? activeEvent; + EventContainer? activeEvent = null; // A Lambda function should never call to get the next event till it was done // processing the active event and there is no more active event. If there @@ -132,7 +132,7 @@ public async Task GetNextInvocation(HttpContext ctx, string functionName) } else { - while (!runtimeDataStore.TryActivateEvent(out activeEvent) && !ctx.RequestAborted.IsCancellationRequested) + while (!ctx.RequestAborted.IsCancellationRequested && !runtimeDataStore.TryActivateEvent(out activeEvent)) { await Task.Delay(TimeSpan.FromMilliseconds(100)); } From f64f3a3dbd82f534900b77a621c3bc4f31867df6 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 2 Apr 2025 20:22:58 -0700 Subject: [PATCH 19/42] Try disabling SQS integ tests --- .../SQSEventSourceTests.cs | 734 +++++++++--------- 1 file changed, 367 insertions(+), 367 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index a2be674cd..2bc00138e 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -22,371 +22,371 @@ public SQSEventSourceTests(ITestOutputHelper testOutputHelper) { } - [RetryFact] - public async Task ProcessSingleMessage() - { - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task SQSEventSourceComesFromEnvironmentVariable() - { - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task ProcessMessagesFromMultipleEventSources() - { - var sqsClient = new AmazonSQSClient(); - var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; - var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; - - var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; - var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; - - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - - var sqsEventSourceConfig = """ -[ - { - "QueueUrl" : "queueUrl1", - "FunctionName" : "SQSProcessor" - }, - { - "QueueUrl" : "queueUrl2", - "FunctionName" : "SQSProcessor" - } -] -""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); - await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - Assert.Equal(2, listOfProcessedMessages.Count); - Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl1); - await sqsClient.DeleteQueueAsync(queueUrl2); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task MessageNotDeleted() - { - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task LambdaThrowsErrorAndMessageNotDeleted() - { - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - - throw new Exception("Failed to process message"); - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task PartialFailureResponse() - { - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - await sqsClient.SendMessageAsync(queueUrl, "Message1"); - - var lambdaPort = GetFreePort(); - - // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - - var sqsResponse = new SQSBatchResponse(); - sqsResponse.BatchItemFailures = new List - { - new SQSBatchResponse.BatchItemFailure - { - ItemIdentifier = evnt.Records[0].MessageId - } - }; - - return sqsResponse; - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - // Since the message was never deleted by the event source it should still be eligibl for reading. - var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); - Assert.Single(response.Messages); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - - private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) - { - // Add a delay to handle SQS eventual consistency. - await Task.Delay(5000); - var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); - return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; - } - - private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) - { - Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); - - var environmentVariables = new Dictionary { }; - - if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) - { - var tokens = sqsEventSourceConfig.Split('&'); - if (tokens.Length == 2) - { - sqsEventSourceConfig = tokens[0]; - var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); - var envValue = tokens[1]; - environmentVariables[envName] = envValue; - } - } - - var settings = new RunCommandSettings - { - LambdaEmulatorPort = lambdaPort, - NoLaunchWindow = true, - SQSEventSourceConfig = sqsEventSourceConfig - }; - - var command = new RunCommand(InteractiveService, new TestEnvironmentManager(environmentVariables)); - var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); - _ = command.ExecuteAsync(context, settings, cancellationTokenSource); - - await Task.Delay(2000, cancellationTokenSource.Token); - } +// [RetryFact] +// public async Task ProcessSingleMessage() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; +// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); + +// var lambdaPort = GetFreePort(); +// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// Assert.Single(listOfProcessedMessages); +// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); +// Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl); +// Console.SetError(consoleError); +// } +// } + +// [RetryFact] +// public async Task SQSEventSourceComesFromEnvironmentVariable() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; +// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); + +// var lambdaPort = GetFreePort(); +// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// Assert.Single(listOfProcessedMessages); +// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); +// Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl); +// Console.SetError(consoleError); +// } +// } + +// [RetryFact] +// public async Task ProcessMessagesFromMultipleEventSources() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; +// var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; + +// var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; +// var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; + +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); + +// var sqsEventSourceConfig = """ +//[ +// { +// "QueueUrl" : "queueUrl1", +// "FunctionName" : "SQSProcessor" +// }, +// { +// "QueueUrl" : "queueUrl2", +// "FunctionName" : "SQSProcessor" +// } +//] +//""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); + +// var lambdaPort = GetFreePort(); +// var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); +// await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// Assert.Equal(2, listOfProcessedMessages.Count); +// Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl1); +// await sqsClient.DeleteQueueAsync(queueUrl2); +// Console.SetError(consoleError); +// } +// } + +// [RetryFact] +// public async Task MessageNotDeleted() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; +// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); + +// var lambdaPort = GetFreePort(); +// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// Assert.Single(listOfProcessedMessages); +// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); +// Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl); +// Console.SetError(consoleError); +// } +// } + +// [RetryFact] +// public async Task LambdaThrowsErrorAndMessageNotDeleted() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; +// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); +// var lambdaPort = GetFreePort(); +// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } + +// throw new Exception("Failed to process message"); +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// Assert.Single(listOfProcessedMessages); +// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); +// Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl); +// Console.SetError(consoleError); +// } +// } + +// [RetryFact] +// public async Task PartialFailureResponse() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; +// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); +// await sqsClient.SendMessageAsync(queueUrl, "Message1"); + +// var lambdaPort = GetFreePort(); + +// // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. +// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } + +// var sqsResponse = new SQSBatchResponse(); +// sqsResponse.BatchItemFailures = new List +// { +// new SQSBatchResponse.BatchItemFailure +// { +// ItemIdentifier = evnt.Records[0].MessageId +// } +// }; + +// return sqsResponse; +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// // Since the message was never deleted by the event source it should still be eligibl for reading. +// var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); +// Assert.Single(response.Messages); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl); +// Console.SetError(consoleError); +// } +// } + + +// private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) +// { +// // Add a delay to handle SQS eventual consistency. +// await Task.Delay(5000); +// var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); +// return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; +// } + +// private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) +// { +// Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); + +// var environmentVariables = new Dictionary { }; + +// if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) +// { +// var tokens = sqsEventSourceConfig.Split('&'); +// if (tokens.Length == 2) +// { +// sqsEventSourceConfig = tokens[0]; +// var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); +// var envValue = tokens[1]; +// environmentVariables[envName] = envValue; +// } +// } + +// var settings = new RunCommandSettings +// { +// LambdaEmulatorPort = lambdaPort, +// NoLaunchWindow = true, +// SQSEventSourceConfig = sqsEventSourceConfig +// }; + +// var command = new RunCommand(InteractiveService, new TestEnvironmentManager(environmentVariables)); +// var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); +// _ = command.ExecuteAsync(context, settings, cancellationTokenSource); + +// await Task.Delay(2000, cancellationTokenSource.Token); +// } } From 6298a1a9087b2a1eb4ae9c2db9a64756b6b0d84e Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 2 Apr 2025 21:21:50 -0700 Subject: [PATCH 20/42] Try disabling more tests to track down CI issue --- .../EdgeCaseTests.cs | 144 +++++++++--------- .../TestOutputToolInteractiveService.cs | 4 +- 2 files changed, 74 insertions(+), 74 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs index 29301680e..2f37f8644 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs @@ -20,89 +20,89 @@ public EdgeCaseTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper { } - [RetryFact] - public async Task TestLambdaReturnString() - { - var ports = GetFreePorts(); - var lambdaPort = ports.lambdaPort; - var apiGatewayPort = ports.apiGatewayPort; + //[RetryFact] + //public async Task TestLambdaReturnString() + //{ + // var ports = GetFreePorts(); + // var lambdaPort = ports.lambdaPort; + // var apiGatewayPort = ports.apiGatewayPort; - CancellationTokenSource = new CancellationTokenSource(); - CancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(120)); - var consoleError = Console.Error; + // CancellationTokenSource = new CancellationTokenSource(); + // CancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(120)); + // var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - await StartTestToolProcessAsync(ApiGatewayEmulatorMode.HttpV2, "stringfunction", lambdaPort, apiGatewayPort, CancellationTokenSource); - await WaitForGatewayHealthCheck(apiGatewayPort); + // try + // { + // Console.SetError(TextWriter.Null); + // await StartTestToolProcessAsync(ApiGatewayEmulatorMode.HttpV2, "stringfunction", lambdaPort, apiGatewayPort, CancellationTokenSource); + // await WaitForGatewayHealthCheck(apiGatewayPort); - var handler = (APIGatewayHttpApiV2ProxyRequest request, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"TestLambdaReturnString"); - return request.Body.ToUpper(); - }; + // var handler = (APIGatewayHttpApiV2ProxyRequest request, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"TestLambdaReturnString"); + // return request.Body.ToUpper(); + // }; - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/stringfunction") - .Build() - .RunAsync(CancellationTokenSource.Token); + // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/stringfunction") + // .Build() + // .RunAsync(CancellationTokenSource.Token); - var response = await TestEndpoint("stringfunction", apiGatewayPort); - var responseContent = await response.Content.ReadAsStringAsync(); + // var response = await TestEndpoint("stringfunction", apiGatewayPort); + // var responseContent = await response.Content.ReadAsStringAsync(); - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.Equal("HELLO WORLD", responseContent); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - Console.SetError(consoleError); - } - } + // Assert.Equal(HttpStatusCode.OK, response.StatusCode); + // Assert.Equal("HELLO WORLD", responseContent); + // } + // finally + // { + // _ = CancellationTokenSource.CancelAsync(); + // Console.SetError(consoleError); + // } + //} - [RetryFact] - public async Task TestLambdaWithNullEndpoint() - { - var ports = GetFreePorts(); - var lambdaPort = ports.lambdaPort; - var apiGatewayPort = ports.apiGatewayPort; + //[RetryFact] + //public async Task TestLambdaWithNullEndpoint() + //{ + // var ports = GetFreePorts(); + // var lambdaPort = ports.lambdaPort; + // var apiGatewayPort = ports.apiGatewayPort; - CancellationTokenSource = new CancellationTokenSource(); - CancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(120)); - var consoleError = Console.Error; + // CancellationTokenSource = new CancellationTokenSource(); + // CancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(120)); + // var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - await StartTestToolProcessWithNullEndpoint(ApiGatewayEmulatorMode.HttpV2, "testfunction", lambdaPort, apiGatewayPort, CancellationTokenSource); - await WaitForGatewayHealthCheck(apiGatewayPort); + // try + // { + // Console.SetError(TextWriter.Null); + // await StartTestToolProcessWithNullEndpoint(ApiGatewayEmulatorMode.HttpV2, "testfunction", lambdaPort, apiGatewayPort, CancellationTokenSource); + // await WaitForGatewayHealthCheck(apiGatewayPort); - var handler = (APIGatewayHttpApiV2ProxyRequest request, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"TestLambdaWithNullEndpoint"); - return new APIGatewayHttpApiV2ProxyResponse - { - StatusCode = 200, - Body = request.Body.ToUpper() - }; - }; + // var handler = (APIGatewayHttpApiV2ProxyRequest request, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"TestLambdaWithNullEndpoint"); + // return new APIGatewayHttpApiV2ProxyResponse + // { + // StatusCode = 200, + // Body = request.Body.ToUpper() + // }; + // }; - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/testfunction") - .Build() - .RunAsync(CancellationTokenSource.Token); + // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/testfunction") + // .Build() + // .RunAsync(CancellationTokenSource.Token); - var response = await TestEndpoint("testfunction", apiGatewayPort); - var responseContent = await response.Content.ReadAsStringAsync(); + // var response = await TestEndpoint("testfunction", apiGatewayPort); + // var responseContent = await response.Content.ReadAsStringAsync(); - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.Equal("HELLO WORLD", responseContent); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - Console.SetError(consoleError); - } - } + // Assert.Equal(HttpStatusCode.OK, response.StatusCode); + // Assert.Equal("HELLO WORLD", responseContent); + // } + // finally + // { + // _ = CancellationTokenSource.CancelAsync(); + // Console.SetError(consoleError); + // } + //} } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs index a2da09af5..5053a630e 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs @@ -14,7 +14,7 @@ public void WriteErrorLine(string? message) } catch (Exception) { - // This can happen when Xunit thinks where is no active test + // This can happen when Xunit thinks there is no active test } } public void WriteLine(string? message) @@ -25,7 +25,7 @@ public void WriteLine(string? message) } catch(Exception) { - // This can happen when Xunit thinks where is no active test + // This can happen when Xunit thinks there is no active test } } } From 5e06508b82ce5213c66afc435e8dd0616132be16 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 2 Apr 2025 21:53:20 -0700 Subject: [PATCH 21/42] Try reverting the use of TestOutputToolInteractiveService --- .../src/Amazon.Lambda.TestTool/Commands/RunCommand.cs | 4 ++-- .../Processes/ApiGatewayEmulatorProcess.cs | 7 +------ .../Processes/TestToolProcess.cs | 4 +--- .../BaseApiGatewayTest.cs | 10 +++++----- .../TestOutputToolInteractiveService.cs | 2 +- .../Processes/ApiGatewayEmulatorProcessTests.cs | 2 +- .../RuntimeApiTests.cs | 4 ++-- 7 files changed, 13 insertions(+), 20 deletions(-) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs index 9e0c70d19..3cb4018c8 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs @@ -42,7 +42,7 @@ public override async Task ExecuteAsync(CommandContext context, RunCommandS if (settings.LambdaEmulatorPort.HasValue) { - var testToolProcess = TestToolProcess.Startup(settings, toolInteractiveService, cancellationTokenSource.Token); + var testToolProcess = TestToolProcess.Startup(settings, cancellationTokenSource.Token); tasks.Add(testToolProcess.RunningTask); if (!settings.NoLaunchWindow) @@ -71,7 +71,7 @@ public override async Task ExecuteAsync(CommandContext context, RunCommandS } var apiGatewayEmulatorProcess = - ApiGatewayEmulatorProcess.Startup(settings, toolInteractiveService, cancellationTokenSource.Token); + ApiGatewayEmulatorProcess.Startup(settings, cancellationTokenSource.Token); tasks.Add(apiGatewayEmulatorProcess.RunningTask); } diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/ApiGatewayEmulatorProcess.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/ApiGatewayEmulatorProcess.cs index 9694a5afd..0ddb448a1 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/ApiGatewayEmulatorProcess.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/ApiGatewayEmulatorProcess.cs @@ -40,7 +40,7 @@ public class ApiGatewayEmulatorProcess /// /// Creates the Web API and runs it in the background. /// - public static ApiGatewayEmulatorProcess Startup(RunCommandSettings settings, IToolInteractiveService toolInteractiveService, CancellationToken cancellationToken = default) + public static ApiGatewayEmulatorProcess Startup(RunCommandSettings settings, CancellationToken cancellationToken = default) { if (settings.ApiGatewayEmulatorMode is null) { @@ -62,11 +62,6 @@ public static ApiGatewayEmulatorProcess Startup(RunCommandSettings settings, ITo var app = builder.Build(); - if (!app.Environment.IsProduction()) - { - app.Services.GetRequiredService().AddProvider(new ToolInteractiveLoggerProvider(toolInteractiveService)); - } - app.MapHealthChecks("/__lambda_test_tool_apigateway_health__"); app.Lifetime.ApplicationStarted.Register(() => diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/TestToolProcess.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/TestToolProcess.cs index 38eb88870..5309315c4 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/TestToolProcess.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/TestToolProcess.cs @@ -37,7 +37,7 @@ public class TestToolProcess /// /// Creates the Web Application and runs it in the background. /// - public static TestToolProcess Startup(RunCommandSettings settings, IToolInteractiveService toolInteractiveService, CancellationToken cancellationToken = default) + public static TestToolProcess Startup(RunCommandSettings settings, CancellationToken cancellationToken = default) { var builder = WebApplication.CreateBuilder(); @@ -82,8 +82,6 @@ public static TestToolProcess Startup(RunCommandSettings settings, IToolInteract } else { - app.Services.GetRequiredService().AddProvider(new ToolInteractiveLoggerProvider(toolInteractiveService)); - // nosemgrep: csharp.lang.security.stacktrace-disclosure.stacktrace-disclosure app.UseDeveloperExceptionPage(); app.UseStaticFiles(); diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index 4c9106905..6d6b4c7d3 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -24,7 +24,7 @@ namespace Amazon.Lambda.TestTool.IntegrationTests; public abstract class BaseApiGatewayTest { protected readonly ITestOutputHelper TestOutputHelper; - protected readonly TestOutputToolInteractiveService InteractiveService; + protected readonly Mock MockInteractiveService; protected readonly Mock MockEnvironmentManager; protected readonly Mock MockRemainingArgs; protected CancellationTokenSource CancellationTokenSource; @@ -32,14 +32,14 @@ public abstract class BaseApiGatewayTest protected BaseApiGatewayTest(ITestOutputHelper testOutputHelper) { - // Enable the intneral logging of runtime support. + // Enable the internal logging of runtime support. Environment.SetEnvironmentVariable("LAMBDA_RUNTIMESUPPORT_DEBUG", "true"); TestOutputHelper = testOutputHelper; MockEnvironmentManager = new Mock(); + MockInteractiveService = new Mock(); MockRemainingArgs = new Mock(); - InteractiveService = new TestOutputToolInteractiveService(testOutputHelper); CancellationTokenSource = new CancellationTokenSource(); } @@ -70,7 +70,7 @@ protected async Task StartTestToolProcessAsync(ApiGatewayEmulatorMode apiGateway ApiGatewayEmulatorPort = apiGatewayPort }; - var command = new RunCommand(InteractiveService, MockEnvironmentManager.Object); + var command = new RunCommand(MockInteractiveService.Object, MockEnvironmentManager.Object); var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); _ = command.ExecuteAsync(context, settings, cancellationTokenSource); @@ -200,7 +200,7 @@ protected async Task StartTestToolProcessWithNullEndpoint(ApiGatewayEmulatorMode ApiGatewayEmulatorPort = apiGatewayPort }; - var command = new RunCommand(InteractiveService, MockEnvironmentManager.Object); + var command = new RunCommand(MockInteractiveService.Object, MockEnvironmentManager.Object); var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); _ = command.ExecuteAsync(context, settings, cancellationTokenSource); diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs index 5053a630e..46ab3e9e9 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs @@ -4,7 +4,7 @@ namespace Amazon.Lambda.TestTool.Tests.Common; -public class TestOutputToolInteractiveService(ITestOutputHelper testOutputHelper) : IToolInteractiveService +public class TestOutputToolInteractiveService(ITestOutputHelper testOutputHelper) //: IToolInteractiveService { public void WriteErrorLine(string? message) { diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/Processes/ApiGatewayEmulatorProcessTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/Processes/ApiGatewayEmulatorProcessTests.cs index e1ca45511..fe6a6d790 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/Processes/ApiGatewayEmulatorProcessTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/Processes/ApiGatewayEmulatorProcessTests.cs @@ -29,7 +29,7 @@ public async Task RouteNotFound(ApiGatewayEmulatorMode mode, HttpStatusCode stat var apiUrl = $"http://{settings.LambdaEmulatorHost}:{settings.ApiGatewayEmulatorPort}/__lambda_test_tool_apigateway_health__"; // Act - var process = ApiGatewayEmulatorProcess.Startup(settings, new TestOutputToolInteractiveService(testOutputHelper), cancellationSource.Token); + var process = ApiGatewayEmulatorProcess.Startup(settings, cancellationSource.Token); var isApiRunning = await TestHelpers.WaitForApiToStartAsync(apiUrl); var response = await TestHelpers.SendRequest($"{process.ServiceUrl}/invalid"); await cancellationSource.CancelAsync(); diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/RuntimeApiTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/RuntimeApiTests.cs index ca8434ec8..998460411 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/RuntimeApiTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.UnitTests/RuntimeApiTests.cs @@ -36,7 +36,7 @@ public async Task AddEventToDataStore() var options = new RunCommandSettings(); options.LambdaEmulatorPort = lambdaPort; Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); - var testToolProcess = TestToolProcess.Startup(options, new TestOutputToolInteractiveService(testOutputHelper), cancellationTokenSource.Token); + var testToolProcess = TestToolProcess.Startup(options, cancellationTokenSource.Token); try { var lambdaClient = ConstructLambdaServiceClient(testToolProcess.ServiceUrl); @@ -89,7 +89,7 @@ public async Task InvokeRequestResponse() var options = new RunCommandSettings(); options.LambdaEmulatorPort = lambdaPort; Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); - var testToolProcess = TestToolProcess.Startup(options, new TestOutputToolInteractiveService(testOutputHelper), cancellationTokenSource.Token); + var testToolProcess = TestToolProcess.Startup(options, cancellationTokenSource.Token); try { var handler = (string input, ILambdaContext context) => From fc0e128b0a255a7a939f43317bf0426c83ede751 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 2 Apr 2025 22:42:15 -0700 Subject: [PATCH 22/42] Revert more test changes --- .../ToolInteractiveLoggerProvider.cs | 31 ------------------- .../BaseApiGatewayTest.cs | 16 +++------- .../BasicApiGatewayTests.cs | 6 ++-- .../BinaryResponseTests.cs | 2 +- .../EdgeCaseTests.cs | 2 +- .../LargePayloadTests.cs | 4 +-- .../SQSEventSourceTests.cs | 10 +++--- .../Helpers/TestHelpers.cs | 23 -------------- .../TestOutputToolInteractiveService.cs | 31 ------------------- 9 files changed, 16 insertions(+), 109 deletions(-) delete mode 100644 Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Utilities/ToolInteractiveLoggerProvider.cs delete mode 100644 Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Utilities/ToolInteractiveLoggerProvider.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Utilities/ToolInteractiveLoggerProvider.cs deleted file mode 100644 index f29f73719..000000000 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Utilities/ToolInteractiveLoggerProvider.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -using Microsoft.Extensions.Logging.Abstractions; -using Amazon.Lambda.TestTool.Services; - -namespace Amazon.Lambda.TestTool.Utilities; - -internal class ToolInteractiveLoggerProvider(IToolInteractiveService toolInteractiveSerivce) : ILoggerProvider -{ - private Logger _logger = new Logger(toolInteractiveSerivce); - public ILogger CreateLogger(string categoryName) => _logger; - public void Dispose() { } - - class Logger(IToolInteractiveService toolInteractiveSerivce) : ILogger - { - public IDisposable? BeginScope(TState state) - where TState : notnull - { - return null; - } - - public bool IsEnabled(LogLevel logLevel) => true; - - public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) - { - var message = formatter(state, exception); - toolInteractiveSerivce.WriteLine(message); - } - } -} diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index 6d6b4c7d3..33eaf8189 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -32,14 +32,10 @@ public abstract class BaseApiGatewayTest protected BaseApiGatewayTest(ITestOutputHelper testOutputHelper) { - // Enable the internal logging of runtime support. - Environment.SetEnvironmentVariable("LAMBDA_RUNTIMESUPPORT_DEBUG", "true"); - TestOutputHelper = testOutputHelper; MockEnvironmentManager = new Mock(); MockInteractiveService = new Mock(); MockRemainingArgs = new Mock(); - CancellationTokenSource = new CancellationTokenSource(); } @@ -47,7 +43,7 @@ protected async Task CleanupAsync() { if (CancellationTokenSource != null) { - _ = CancellationTokenSource.CancelAsync(); + await CancellationTokenSource.CancelAsync(); CancellationTokenSource = new CancellationTokenSource(); } } @@ -161,16 +157,12 @@ protected async Task TestEndpoint(string routeName, int api protected int GetFreePort() { - var port = TestHelpers.GetRandomIntegerInRange(49152, 65535); - using var listener = new TcpListener(IPAddress.Loopback, port); + var random = new Random(); + var port = random.Next(49152, 65535); + var listener = new TcpListener(IPAddress.Loopback, port); try { listener.Start(); - using TcpClient client = new TcpClient("127.0.0.1", port); - using NetworkStream stream = client.GetStream(); - stream.WriteByte(0x01); - stream.Flush(); - return port; } catch (SocketException) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs index 825dd0269..edd0f49fd 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BasicApiGatewayTests.cs @@ -60,7 +60,7 @@ public async Task TestLambdaToUpperV2() } finally { - _ = CancellationTokenSource.CancelAsync(); + await CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } @@ -105,7 +105,7 @@ public async Task TestLambdaToUpperRest() } finally { - _ = CancellationTokenSource.CancelAsync(); + await CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } @@ -150,7 +150,7 @@ public async Task TestLambdaToUpperV1() } finally { - _ = CancellationTokenSource.CancelAsync(); + await CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs index de2dd021c..23f46a472 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BinaryResponseTests.cs @@ -78,7 +78,7 @@ public async Task TestLambdaBinaryResponse() } finally { - _ = CancellationTokenSource.CancelAsync(); + await CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs index 2f37f8644..5e35d8557 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs @@ -101,7 +101,7 @@ public EdgeCaseTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper // } // finally // { - // _ = CancellationTokenSource.CancelAsync(); + // await CancellationTokenSource.CancelAsync(); // Console.SetError(consoleError); // } //} diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs index eb8bd959c..c7faf251d 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/LargePayloadTests.cs @@ -63,7 +63,7 @@ public async Task TestLambdaWithLargeRequestPayload_RestAndV1(ApiGatewayEmulator } finally { - _ = CancellationTokenSource.CancelAsync(); + await CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } @@ -110,7 +110,7 @@ public async Task TestLambdaWithLargeRequestPayload_HttpV2() } finally { - _ = CancellationTokenSource.CancelAsync(); + await CancellationTokenSource.CancelAsync(); Console.SetError(consoleError); } } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 2bc00138e..cdc3390c5 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -114,7 +114,7 @@ public SQSEventSourceTests(ITestOutputHelper testOutputHelper) // } // finally // { -// _ = CancellationTokenSource.CancelAsync(); +// await CancellationTokenSource.CancelAsync(); // await sqsClient.DeleteQueueAsync(queueUrl); // Console.SetError(consoleError); // } @@ -180,7 +180,7 @@ public SQSEventSourceTests(ITestOutputHelper testOutputHelper) // } // finally // { -// _ = CancellationTokenSource.CancelAsync(); +// await CancellationTokenSource.CancelAsync(); // await sqsClient.DeleteQueueAsync(queueUrl1); // await sqsClient.DeleteQueueAsync(queueUrl2); // Console.SetError(consoleError); @@ -230,7 +230,7 @@ public SQSEventSourceTests(ITestOutputHelper testOutputHelper) // } // finally // { -// _ = CancellationTokenSource.CancelAsync(); +// await CancellationTokenSource.CancelAsync(); // await sqsClient.DeleteQueueAsync(queueUrl); // Console.SetError(consoleError); // } @@ -280,7 +280,7 @@ public SQSEventSourceTests(ITestOutputHelper testOutputHelper) // } // finally // { -// _ = CancellationTokenSource.CancelAsync(); +// await CancellationTokenSource.CancelAsync(); // await sqsClient.DeleteQueueAsync(queueUrl); // Console.SetError(consoleError); // } @@ -343,7 +343,7 @@ public SQSEventSourceTests(ITestOutputHelper testOutputHelper) // } // finally // { -// _ = CancellationTokenSource.CancelAsync(); +// await CancellationTokenSource.CancelAsync(); // await sqsClient.DeleteQueueAsync(queueUrl); // Console.SetError(consoleError); // } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/Helpers/TestHelpers.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/Helpers/TestHelpers.cs index 71c8b14a4..79c618d48 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/Helpers/TestHelpers.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/Helpers/TestHelpers.cs @@ -1,8 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -using System.Security.Cryptography; - namespace Amazon.Lambda.TestTool.Tests.Common.Helpers; public static class TestHelpers @@ -53,25 +51,4 @@ public static int GetNextApiGatewayPort() { return Interlocked.Increment(ref _maxApiGatewayPort); } - - private static RandomNumberGenerator rng = RandomNumberGenerator.Create(); - public static int GetRandomIntegerInRange(int minValue, int maxValue) - { - if (minValue >= maxValue) - throw new ArgumentOutOfRangeException(nameof(minValue), "minValue must be less than maxValue"); - - // Create a byte array to hold the random bytes - byte[] randomBytes = new byte[4]; - - // Fill the array with random bytes - rng.GetBytes(randomBytes); - - // Convert the bytes to an integer - int randomInteger = BitConverter.ToInt32(randomBytes, 0); - - // Make sure the random integer is within the desired range - // Apply modulus to get a number in the range [0, maxValue - minValue] - int range = maxValue - minValue; - return (Math.Abs(randomInteger) % range) + minValue; - } } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs deleted file mode 100644 index 46ab3e9e9..000000000 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.Tests.Common/TestOutputToolInteractiveService.cs +++ /dev/null @@ -1,31 +0,0 @@ -using Amazon.Lambda.TestTool.Services; -using Xunit.Abstractions; - - -namespace Amazon.Lambda.TestTool.Tests.Common; - -public class TestOutputToolInteractiveService(ITestOutputHelper testOutputHelper) //: IToolInteractiveService -{ - public void WriteErrorLine(string? message) - { - try - { - testOutputHelper.WriteLine("Error: " + message); - } - catch (Exception) - { - // This can happen when Xunit thinks there is no active test - } - } - public void WriteLine(string? message) - { - try - { - testOutputHelper.WriteLine(message); - } - catch(Exception) - { - // This can happen when Xunit thinks there is no active test - } - } -} From 7cb6f87a2dd8b8ef0fbfa7397265559e4774e831 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 00:13:49 -0700 Subject: [PATCH 23/42] Add back tests see if they now work in CI --- .../BaseApiGatewayTest.cs | 5 +- .../EdgeCaseTests.cs | 144 ++-- .../SQSEventSourceTests.cs | 737 +++++++++--------- 3 files changed, 443 insertions(+), 443 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index 33eaf8189..10abb57a2 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -14,10 +14,6 @@ using System.Text; using System.Net.Http; using System.Net.Http.Headers; -using System.Security.Cryptography; -using Amazon.Lambda.TestTool.Tests.Common.Helpers; -using Microsoft.AspNetCore.Hosting.Server; -using Amazon.Lambda.TestTool.Tests.Common; namespace Amazon.Lambda.TestTool.IntegrationTests; @@ -44,6 +40,7 @@ protected async Task CleanupAsync() if (CancellationTokenSource != null) { await CancellationTokenSource.CancelAsync(); + CancellationTokenSource.Dispose(); CancellationTokenSource = new CancellationTokenSource(); } } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs index 5e35d8557..6c5fccfb6 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/EdgeCaseTests.cs @@ -20,89 +20,89 @@ public EdgeCaseTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper { } - //[RetryFact] - //public async Task TestLambdaReturnString() - //{ - // var ports = GetFreePorts(); - // var lambdaPort = ports.lambdaPort; - // var apiGatewayPort = ports.apiGatewayPort; + [RetryFact] + public async Task TestLambdaReturnString() + { + var ports = GetFreePorts(); + var lambdaPort = ports.lambdaPort; + var apiGatewayPort = ports.apiGatewayPort; - // CancellationTokenSource = new CancellationTokenSource(); - // CancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(120)); - // var consoleError = Console.Error; + CancellationTokenSource = new CancellationTokenSource(); + CancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(120)); + var consoleError = Console.Error; - // try - // { - // Console.SetError(TextWriter.Null); - // await StartTestToolProcessAsync(ApiGatewayEmulatorMode.HttpV2, "stringfunction", lambdaPort, apiGatewayPort, CancellationTokenSource); - // await WaitForGatewayHealthCheck(apiGatewayPort); + try + { + Console.SetError(TextWriter.Null); + await StartTestToolProcessAsync(ApiGatewayEmulatorMode.HttpV2, "stringfunction", lambdaPort, apiGatewayPort, CancellationTokenSource); + await WaitForGatewayHealthCheck(apiGatewayPort); - // var handler = (APIGatewayHttpApiV2ProxyRequest request, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"TestLambdaReturnString"); - // return request.Body.ToUpper(); - // }; + var handler = (APIGatewayHttpApiV2ProxyRequest request, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"TestLambdaReturnString"); + return request.Body.ToUpper(); + }; - // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/stringfunction") - // .Build() - // .RunAsync(CancellationTokenSource.Token); + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/stringfunction") + .Build() + .RunAsync(CancellationTokenSource.Token); - // var response = await TestEndpoint("stringfunction", apiGatewayPort); - // var responseContent = await response.Content.ReadAsStringAsync(); + var response = await TestEndpoint("stringfunction", apiGatewayPort); + var responseContent = await response.Content.ReadAsStringAsync(); - // Assert.Equal(HttpStatusCode.OK, response.StatusCode); - // Assert.Equal("HELLO WORLD", responseContent); - // } - // finally - // { - // _ = CancellationTokenSource.CancelAsync(); - // Console.SetError(consoleError); - // } - //} + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("HELLO WORLD", responseContent); + } + finally + { + await CancellationTokenSource.CancelAsync(); + Console.SetError(consoleError); + } + } - //[RetryFact] - //public async Task TestLambdaWithNullEndpoint() - //{ - // var ports = GetFreePorts(); - // var lambdaPort = ports.lambdaPort; - // var apiGatewayPort = ports.apiGatewayPort; + [RetryFact] + public async Task TestLambdaWithNullEndpoint() + { + var ports = GetFreePorts(); + var lambdaPort = ports.lambdaPort; + var apiGatewayPort = ports.apiGatewayPort; - // CancellationTokenSource = new CancellationTokenSource(); - // CancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(120)); - // var consoleError = Console.Error; + CancellationTokenSource = new CancellationTokenSource(); + CancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(120)); + var consoleError = Console.Error; - // try - // { - // Console.SetError(TextWriter.Null); - // await StartTestToolProcessWithNullEndpoint(ApiGatewayEmulatorMode.HttpV2, "testfunction", lambdaPort, apiGatewayPort, CancellationTokenSource); - // await WaitForGatewayHealthCheck(apiGatewayPort); + try + { + Console.SetError(TextWriter.Null); + await StartTestToolProcessWithNullEndpoint(ApiGatewayEmulatorMode.HttpV2, "testfunction", lambdaPort, apiGatewayPort, CancellationTokenSource); + await WaitForGatewayHealthCheck(apiGatewayPort); - // var handler = (APIGatewayHttpApiV2ProxyRequest request, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"TestLambdaWithNullEndpoint"); - // return new APIGatewayHttpApiV2ProxyResponse - // { - // StatusCode = 200, - // Body = request.Body.ToUpper() - // }; - // }; + var handler = (APIGatewayHttpApiV2ProxyRequest request, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"TestLambdaWithNullEndpoint"); + return new APIGatewayHttpApiV2ProxyResponse + { + StatusCode = 200, + Body = request.Body.ToUpper() + }; + }; - // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/testfunction") - // .Build() - // .RunAsync(CancellationTokenSource.Token); + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/testfunction") + .Build() + .RunAsync(CancellationTokenSource.Token); - // var response = await TestEndpoint("testfunction", apiGatewayPort); - // var responseContent = await response.Content.ReadAsStringAsync(); + var response = await TestEndpoint("testfunction", apiGatewayPort); + var responseContent = await response.Content.ReadAsStringAsync(); - // Assert.Equal(HttpStatusCode.OK, response.StatusCode); - // Assert.Equal("HELLO WORLD", responseContent); - // } - // finally - // { - // await CancellationTokenSource.CancelAsync(); - // Console.SetError(consoleError); - // } - //} + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("HELLO WORLD", responseContent); + } + finally + { + await CancellationTokenSource.CancelAsync(); + Console.SetError(consoleError); + } + } } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index cdc3390c5..d7a7f7e1b 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -11,6 +11,8 @@ using Amazon.SQS.Model; using Amazon.Lambda.TestTool.Tests.Common; using Amazon.Lambda.TestTool.Tests.Common.Retries; +using Amazon.Lambda.TestTool.Services; +using Moq; namespace Amazon.Lambda.TestTool.IntegrationTests; @@ -22,371 +24,372 @@ public SQSEventSourceTests(ITestOutputHelper testOutputHelper) { } -// [RetryFact] -// public async Task ProcessSingleMessage() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; -// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); - -// var lambdaPort = GetFreePort(); -// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// Assert.Single(listOfProcessedMessages); -// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); -// Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); -// } -// finally -// { -// _ = CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl); -// Console.SetError(consoleError); -// } -// } - -// [RetryFact] -// public async Task SQSEventSourceComesFromEnvironmentVariable() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; -// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); - -// var lambdaPort = GetFreePort(); -// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// Assert.Single(listOfProcessedMessages); -// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); -// Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); -// } -// finally -// { -// await CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl); -// Console.SetError(consoleError); -// } -// } - -// [RetryFact] -// public async Task ProcessMessagesFromMultipleEventSources() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; -// var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; - -// var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; -// var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; - -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); - -// var sqsEventSourceConfig = """ -//[ -// { -// "QueueUrl" : "queueUrl1", -// "FunctionName" : "SQSProcessor" -// }, -// { -// "QueueUrl" : "queueUrl2", -// "FunctionName" : "SQSProcessor" -// } -//] -//""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); - -// var lambdaPort = GetFreePort(); -// var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); -// await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// Assert.Equal(2, listOfProcessedMessages.Count); -// Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); -// } -// finally -// { -// await CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl1); -// await sqsClient.DeleteQueueAsync(queueUrl2); -// Console.SetError(consoleError); -// } -// } - -// [RetryFact] -// public async Task MessageNotDeleted() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; -// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); - -// var lambdaPort = GetFreePort(); -// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// Assert.Single(listOfProcessedMessages); -// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); -// Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); -// } -// finally -// { -// await CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl); -// Console.SetError(consoleError); -// } -// } - -// [RetryFact] -// public async Task LambdaThrowsErrorAndMessageNotDeleted() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; -// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); -// var lambdaPort = GetFreePort(); -// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } - -// throw new Exception("Failed to process message"); -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// Assert.Single(listOfProcessedMessages); -// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); -// Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); -// } -// finally -// { -// await CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl); -// Console.SetError(consoleError); -// } -// } - -// [RetryFact] -// public async Task PartialFailureResponse() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; -// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); -// await sqsClient.SendMessageAsync(queueUrl, "Message1"); - -// var lambdaPort = GetFreePort(); - -// // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. -// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } - -// var sqsResponse = new SQSBatchResponse(); -// sqsResponse.BatchItemFailures = new List -// { -// new SQSBatchResponse.BatchItemFailure -// { -// ItemIdentifier = evnt.Records[0].MessageId -// } -// }; - -// return sqsResponse; -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// // Since the message was never deleted by the event source it should still be eligibl for reading. -// var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); -// Assert.Single(response.Messages); -// } -// finally -// { -// await CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl); -// Console.SetError(consoleError); -// } -// } - - -// private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) -// { -// // Add a delay to handle SQS eventual consistency. -// await Task.Delay(5000); -// var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); -// return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; -// } - -// private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) -// { -// Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); - -// var environmentVariables = new Dictionary { }; - -// if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) -// { -// var tokens = sqsEventSourceConfig.Split('&'); -// if (tokens.Length == 2) -// { -// sqsEventSourceConfig = tokens[0]; -// var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); -// var envValue = tokens[1]; -// environmentVariables[envName] = envValue; -// } -// } - -// var settings = new RunCommandSettings -// { -// LambdaEmulatorPort = lambdaPort, -// NoLaunchWindow = true, -// SQSEventSourceConfig = sqsEventSourceConfig -// }; - -// var command = new RunCommand(InteractiveService, new TestEnvironmentManager(environmentVariables)); -// var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); -// _ = command.ExecuteAsync(context, settings, cancellationTokenSource); - -// await Task.Delay(2000, cancellationTokenSource.Token); -// } + [RetryFact] + public async Task ProcessSingleMessage() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + _ = CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task SQSEventSourceComesFromEnvironmentVariable() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + await CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task ProcessMessagesFromMultipleEventSources() + { + var sqsClient = new AmazonSQSClient(); + var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; + var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; + + var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; + var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; + + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + + var sqsEventSourceConfig = """ +[ + { + "QueueUrl" : "queueUrl1", + "FunctionName" : "SQSProcessor" + }, + { + "QueueUrl" : "queueUrl2", + "FunctionName" : "SQSProcessor" + } +] +""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); + await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Equal(2, listOfProcessedMessages.Count); + Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); + } + finally + { + await CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl1); + await sqsClient.DeleteQueueAsync(queueUrl2); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task MessageNotDeleted() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + await CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task LambdaThrowsErrorAndMessageNotDeleted() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + + throw new Exception("Failed to process message"); + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + await CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task PartialFailureResponse() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + await sqsClient.SendMessageAsync(queueUrl, "Message1"); + + var lambdaPort = GetFreePort(); + + // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + + var sqsResponse = new SQSBatchResponse(); + sqsResponse.BatchItemFailures = new List + { + new SQSBatchResponse.BatchItemFailure + { + ItemIdentifier = evnt.Records[0].MessageId + } + }; + + return sqsResponse; + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + // Since the message was never deleted by the event source it should still be eligibl for reading. + var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); + Assert.Single(response.Messages); + } + finally + { + await CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + + private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) + { + // Add a delay to handle SQS eventual consistency. + await Task.Delay(5000); + var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); + return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; + } + + private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) + { + Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); + + var environmentVariables = new Dictionary { }; + + if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) + { + var tokens = sqsEventSourceConfig.Split('&'); + if (tokens.Length == 2) + { + sqsEventSourceConfig = tokens[0]; + var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); + var envValue = tokens[1]; + environmentVariables[envName] = envValue; + } + } + + var settings = new RunCommandSettings + { + LambdaEmulatorPort = lambdaPort, + NoLaunchWindow = true, + SQSEventSourceConfig = sqsEventSourceConfig + }; + + + var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); + var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); + _ = command.ExecuteAsync(context, settings, cancellationTokenSource); + + await Task.Delay(2000, cancellationTokenSource.Token); + } } From a38f6d72135c605be3531aaaf7805d69437b995d Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 10:00:00 -0700 Subject: [PATCH 24/42] Tweak SQS tests --- .../SQSEventSourceTests.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index d7a7f7e1b..87d3271f1 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -116,7 +116,7 @@ public async Task SQSEventSourceComesFromEnvironmentVariable() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } @@ -182,7 +182,7 @@ public async Task ProcessMessagesFromMultipleEventSources() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl1); await sqsClient.DeleteQueueAsync(queueUrl2); Console.SetError(consoleError); @@ -232,7 +232,7 @@ public async Task MessageNotDeleted() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } @@ -282,7 +282,7 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } @@ -339,13 +339,16 @@ public async Task PartialFailureResponse() await Task.Delay(500); } + // Wait for message to be deleted. + await Task.Delay(2000); + // Since the message was never deleted by the event source it should still be eligibl for reading. var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); Assert.Single(response.Messages); } finally { - await CancellationTokenSource.CancelAsync(); + _ = CancellationTokenSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } From 3c6e8d579530ecd0ac64ec4138a30ed2f1aaa352 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 10:43:24 -0700 Subject: [PATCH 25/42] Try turning off the SQS tests --- .../SQSEventSourceTests.cs | 748 +++++++++--------- 1 file changed, 374 insertions(+), 374 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 87d3271f1..6c0e99906 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -17,382 +17,382 @@ namespace Amazon.Lambda.TestTool.IntegrationTests; [Collection("SQSEventSourceTests")] -public class SQSEventSourceTests : BaseApiGatewayTest +public class SQSEventSourceTests //: BaseApiGatewayTest { - public SQSEventSourceTests(ITestOutputHelper testOutputHelper) - : base(testOutputHelper) - { - } - - [RetryFact] - public async Task ProcessSingleMessage() - { - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task SQSEventSourceComesFromEnvironmentVariable() - { - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task ProcessMessagesFromMultipleEventSources() - { - var sqsClient = new AmazonSQSClient(); - var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; - var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; - - var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; - var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; - - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - - var sqsEventSourceConfig = """ -[ - { - "QueueUrl" : "queueUrl1", - "FunctionName" : "SQSProcessor" - }, - { - "QueueUrl" : "queueUrl2", - "FunctionName" : "SQSProcessor" - } -] -""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); - await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - Assert.Equal(2, listOfProcessedMessages.Count); - Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl1); - await sqsClient.DeleteQueueAsync(queueUrl2); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task MessageNotDeleted() - { - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task LambdaThrowsErrorAndMessageNotDeleted() - { - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - - throw new Exception("Failed to process message"); - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task PartialFailureResponse() - { - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - var consoleError = Console.Error; - try - { - Console.SetError(TextWriter.Null); - await sqsClient.SendMessageAsync(queueUrl, "Message1"); - - var lambdaPort = GetFreePort(); - - // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - - var sqsResponse = new SQSBatchResponse(); - sqsResponse.BatchItemFailures = new List - { - new SQSBatchResponse.BatchItemFailure - { - ItemIdentifier = evnt.Records[0].MessageId - } - }; - - return sqsResponse; - }; - - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(CancellationTokenSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - await Task.Delay(500); - } - - // Wait for message to be deleted. - await Task.Delay(2000); - - // Since the message was never deleted by the event source it should still be eligibl for reading. - var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); - Assert.Single(response.Messages); - } - finally - { - _ = CancellationTokenSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - - private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) - { - // Add a delay to handle SQS eventual consistency. - await Task.Delay(5000); - var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); - return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; - } - - private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) - { - Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); - - var environmentVariables = new Dictionary { }; - - if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) - { - var tokens = sqsEventSourceConfig.Split('&'); - if (tokens.Length == 2) - { - sqsEventSourceConfig = tokens[0]; - var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); - var envValue = tokens[1]; - environmentVariables[envName] = envValue; - } - } - - var settings = new RunCommandSettings - { - LambdaEmulatorPort = lambdaPort, - NoLaunchWindow = true, - SQSEventSourceConfig = sqsEventSourceConfig - }; +// public SQSEventSourceTests(ITestOutputHelper testOutputHelper) +// : base(testOutputHelper) +// { +// } + +// [RetryFact] +// public async Task ProcessSingleMessage() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; +// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); + +// var lambdaPort = GetFreePort(); +// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// Assert.Single(listOfProcessedMessages); +// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); +// Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl); +// Console.SetError(consoleError); +// } +// } + +// [RetryFact] +// public async Task SQSEventSourceComesFromEnvironmentVariable() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; +// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); + +// var lambdaPort = GetFreePort(); +// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// Assert.Single(listOfProcessedMessages); +// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); +// Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl); +// Console.SetError(consoleError); +// } +// } + +// [RetryFact] +// public async Task ProcessMessagesFromMultipleEventSources() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; +// var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; + +// var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; +// var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; + +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); + +// var sqsEventSourceConfig = """ +//[ +// { +// "QueueUrl" : "queueUrl1", +// "FunctionName" : "SQSProcessor" +// }, +// { +// "QueueUrl" : "queueUrl2", +// "FunctionName" : "SQSProcessor" +// } +//] +//""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); + +// var lambdaPort = GetFreePort(); +// var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); +// await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// Assert.Equal(2, listOfProcessedMessages.Count); +// Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl1); +// await sqsClient.DeleteQueueAsync(queueUrl2); +// Console.SetError(consoleError); +// } +// } + +// [RetryFact] +// public async Task MessageNotDeleted() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; +// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); + +// var lambdaPort = GetFreePort(); +// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// Assert.Single(listOfProcessedMessages); +// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); +// Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl); +// Console.SetError(consoleError); +// } +// } + +// [RetryFact] +// public async Task LambdaThrowsErrorAndMessageNotDeleted() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; +// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); +// var lambdaPort = GetFreePort(); +// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } + +// throw new Exception("Failed to process message"); +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// Assert.Single(listOfProcessedMessages); +// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); +// Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl); +// Console.SetError(consoleError); +// } +// } + +// [RetryFact] +// public async Task PartialFailureResponse() +// { +// var sqsClient = new AmazonSQSClient(); +// var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; +// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; +// var consoleError = Console.Error; +// try +// { +// Console.SetError(TextWriter.Null); +// await sqsClient.SendMessageAsync(queueUrl, "Message1"); + +// var lambdaPort = GetFreePort(); + +// // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. +// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); + +// var listOfProcessedMessages = new List(); +// var handler = (SQSEvent evnt, ILambdaContext context) => +// { +// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); +// foreach (var message in evnt.Records) +// { +// listOfProcessedMessages.Add(message); +// } + +// var sqsResponse = new SQSBatchResponse(); +// sqsResponse.BatchItemFailures = new List +// { +// new SQSBatchResponse.BatchItemFailure +// { +// ItemIdentifier = evnt.Records[0].MessageId +// } +// }; + +// return sqsResponse; +// }; + +// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) +// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") +// .Build() +// .RunAsync(CancellationTokenSource.Token); + +// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + +// var startTime = DateTime.UtcNow; +// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) +// { +// await Task.Delay(500); +// } + +// // Wait for message to be deleted. +// await Task.Delay(2000); + +// // Since the message was never deleted by the event source it should still be eligibl for reading. +// var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); +// Assert.Single(response.Messages); +// } +// finally +// { +// _ = CancellationTokenSource.CancelAsync(); +// await sqsClient.DeleteQueueAsync(queueUrl); +// Console.SetError(consoleError); +// } +// } + + +// private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) +// { +// // Add a delay to handle SQS eventual consistency. +// await Task.Delay(5000); +// var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); +// return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; +// } + +// private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) +// { +// Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); + +// var environmentVariables = new Dictionary { }; + +// if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) +// { +// var tokens = sqsEventSourceConfig.Split('&'); +// if (tokens.Length == 2) +// { +// sqsEventSourceConfig = tokens[0]; +// var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); +// var envValue = tokens[1]; +// environmentVariables[envName] = envValue; +// } +// } + +// var settings = new RunCommandSettings +// { +// LambdaEmulatorPort = lambdaPort, +// NoLaunchWindow = true, +// SQSEventSourceConfig = sqsEventSourceConfig +// }; - var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); - var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); - _ = command.ExecuteAsync(context, settings, cancellationTokenSource); +// var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); +// var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); +// _ = command.ExecuteAsync(context, settings, cancellationTokenSource); - await Task.Delay(2000, cancellationTokenSource.Token); - } +// await Task.Delay(2000, cancellationTokenSource.Token); +// } } From 1a70ee30f2debea34412a05b752b680e196f3481 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 11:09:56 -0700 Subject: [PATCH 26/42] Add back one SQS test --- .../SQSEventSourceTests.cs | 754 +++++++++--------- 1 file changed, 377 insertions(+), 377 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 6c0e99906..0270bc126 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -17,382 +17,382 @@ namespace Amazon.Lambda.TestTool.IntegrationTests; [Collection("SQSEventSourceTests")] -public class SQSEventSourceTests //: BaseApiGatewayTest +public class SQSEventSourceTests : BaseApiGatewayTest { -// public SQSEventSourceTests(ITestOutputHelper testOutputHelper) -// : base(testOutputHelper) -// { -// } - -// [RetryFact] -// public async Task ProcessSingleMessage() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; -// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); - -// var lambdaPort = GetFreePort(); -// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// Assert.Single(listOfProcessedMessages); -// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); -// Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); -// } -// finally -// { -// _ = CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl); -// Console.SetError(consoleError); -// } -// } - -// [RetryFact] -// public async Task SQSEventSourceComesFromEnvironmentVariable() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; -// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); - -// var lambdaPort = GetFreePort(); -// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// Assert.Single(listOfProcessedMessages); -// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); -// Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); -// } -// finally -// { -// _ = CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl); -// Console.SetError(consoleError); -// } -// } - -// [RetryFact] -// public async Task ProcessMessagesFromMultipleEventSources() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; -// var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; - -// var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; -// var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; - -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); - -// var sqsEventSourceConfig = """ -//[ -// { -// "QueueUrl" : "queueUrl1", -// "FunctionName" : "SQSProcessor" -// }, -// { -// "QueueUrl" : "queueUrl2", -// "FunctionName" : "SQSProcessor" -// } -//] -//""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); - -// var lambdaPort = GetFreePort(); -// var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); -// await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// Assert.Equal(2, listOfProcessedMessages.Count); -// Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); -// } -// finally -// { -// _ = CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl1); -// await sqsClient.DeleteQueueAsync(queueUrl2); -// Console.SetError(consoleError); -// } -// } - -// [RetryFact] -// public async Task MessageNotDeleted() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; -// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); - -// var lambdaPort = GetFreePort(); -// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// Assert.Single(listOfProcessedMessages); -// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); -// Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); -// } -// finally -// { -// _ = CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl); -// Console.SetError(consoleError); -// } -// } - -// [RetryFact] -// public async Task LambdaThrowsErrorAndMessageNotDeleted() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; -// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); -// var lambdaPort = GetFreePort(); -// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } - -// throw new Exception("Failed to process message"); -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// Assert.Single(listOfProcessedMessages); -// Assert.Equal("TheBody", listOfProcessedMessages[0].Body); -// Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); -// } -// finally -// { -// _ = CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl); -// Console.SetError(consoleError); -// } -// } - -// [RetryFact] -// public async Task PartialFailureResponse() -// { -// var sqsClient = new AmazonSQSClient(); -// var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; -// var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; -// var consoleError = Console.Error; -// try -// { -// Console.SetError(TextWriter.Null); -// await sqsClient.SendMessageAsync(queueUrl, "Message1"); - -// var lambdaPort = GetFreePort(); - -// // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. -// var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); - -// var listOfProcessedMessages = new List(); -// var handler = (SQSEvent evnt, ILambdaContext context) => -// { -// TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); -// foreach (var message in evnt.Records) -// { -// listOfProcessedMessages.Add(message); -// } - -// var sqsResponse = new SQSBatchResponse(); -// sqsResponse.BatchItemFailures = new List -// { -// new SQSBatchResponse.BatchItemFailure -// { -// ItemIdentifier = evnt.Records[0].MessageId -// } -// }; - -// return sqsResponse; -// }; - -// _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) -// .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") -// .Build() -// .RunAsync(CancellationTokenSource.Token); - -// await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - -// var startTime = DateTime.UtcNow; -// while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) -// { -// await Task.Delay(500); -// } - -// // Wait for message to be deleted. -// await Task.Delay(2000); - -// // Since the message was never deleted by the event source it should still be eligibl for reading. -// var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); -// Assert.Single(response.Messages); -// } -// finally -// { -// _ = CancellationTokenSource.CancelAsync(); -// await sqsClient.DeleteQueueAsync(queueUrl); -// Console.SetError(consoleError); -// } -// } - - -// private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) -// { -// // Add a delay to handle SQS eventual consistency. -// await Task.Delay(5000); -// var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); -// return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; -// } - -// private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) -// { -// Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); - -// var environmentVariables = new Dictionary { }; - -// if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) -// { -// var tokens = sqsEventSourceConfig.Split('&'); -// if (tokens.Length == 2) -// { -// sqsEventSourceConfig = tokens[0]; -// var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); -// var envValue = tokens[1]; -// environmentVariables[envName] = envValue; -// } -// } - -// var settings = new RunCommandSettings -// { -// LambdaEmulatorPort = lambdaPort, -// NoLaunchWindow = true, -// SQSEventSourceConfig = sqsEventSourceConfig -// }; - - -// var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); -// var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); -// _ = command.ExecuteAsync(context, settings, cancellationTokenSource); - -// await Task.Delay(2000, cancellationTokenSource.Token); -// } + public SQSEventSourceTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper) + { + } + + [RetryFact] + public async Task ProcessSingleMessage() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + _ = CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + // [RetryFact] + // public async Task SQSEventSourceComesFromEnvironmentVariable() + // { + // var sqsClient = new AmazonSQSClient(); + // var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; + // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + // var consoleError = Console.Error; + // try + // { + // Console.SetError(TextWriter.Null); + + // var lambdaPort = GetFreePort(); + // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + + // var listOfProcessedMessages = new List(); + // var handler = (SQSEvent evnt, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + // foreach (var message in evnt.Records) + // { + // listOfProcessedMessages.Add(message); + // } + // }; + + // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + // .Build() + // .RunAsync(CancellationTokenSource.Token); + + // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + // var startTime = DateTime.UtcNow; + // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + // { + // await Task.Delay(500); + // } + + // Assert.Single(listOfProcessedMessages); + // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + // Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + // } + // finally + // { + // _ = CancellationTokenSource.CancelAsync(); + // await sqsClient.DeleteQueueAsync(queueUrl); + // Console.SetError(consoleError); + // } + // } + + // [RetryFact] + // public async Task ProcessMessagesFromMultipleEventSources() + // { + // var sqsClient = new AmazonSQSClient(); + // var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; + // var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; + + // var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; + // var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; + + // var consoleError = Console.Error; + // try + // { + // Console.SetError(TextWriter.Null); + + // var sqsEventSourceConfig = """ + //[ + // { + // "QueueUrl" : "queueUrl1", + // "FunctionName" : "SQSProcessor" + // }, + // { + // "QueueUrl" : "queueUrl2", + // "FunctionName" : "SQSProcessor" + // } + //] + //""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); + + // var lambdaPort = GetFreePort(); + // var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, CancellationTokenSource); + + // var listOfProcessedMessages = new List(); + // var handler = (SQSEvent evnt, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + // foreach (var message in evnt.Records) + // { + // listOfProcessedMessages.Add(message); + // } + // }; + + // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + // .Build() + // .RunAsync(CancellationTokenSource.Token); + + // await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); + // await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); + + // var startTime = DateTime.UtcNow; + // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + // { + // await Task.Delay(500); + // } + + // Assert.Equal(2, listOfProcessedMessages.Count); + // Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); + // } + // finally + // { + // _ = CancellationTokenSource.CancelAsync(); + // await sqsClient.DeleteQueueAsync(queueUrl1); + // await sqsClient.DeleteQueueAsync(queueUrl2); + // Console.SetError(consoleError); + // } + // } + + // [RetryFact] + // public async Task MessageNotDeleted() + // { + // var sqsClient = new AmazonSQSClient(); + // var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; + // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + // var consoleError = Console.Error; + // try + // { + // Console.SetError(TextWriter.Null); + + // var lambdaPort = GetFreePort(); + // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); + + // var listOfProcessedMessages = new List(); + // var handler = (SQSEvent evnt, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + // foreach (var message in evnt.Records) + // { + // listOfProcessedMessages.Add(message); + // } + // }; + + // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + // .Build() + // .RunAsync(CancellationTokenSource.Token); + + // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + // var startTime = DateTime.UtcNow; + // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + // { + // await Task.Delay(500); + // } + + // Assert.Single(listOfProcessedMessages); + // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + // Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + // } + // finally + // { + // _ = CancellationTokenSource.CancelAsync(); + // await sqsClient.DeleteQueueAsync(queueUrl); + // Console.SetError(consoleError); + // } + // } + + // [RetryFact] + // public async Task LambdaThrowsErrorAndMessageNotDeleted() + // { + // var sqsClient = new AmazonSQSClient(); + // var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; + // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + // var consoleError = Console.Error; + // try + // { + // Console.SetError(TextWriter.Null); + // var lambdaPort = GetFreePort(); + // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + + // var listOfProcessedMessages = new List(); + // var handler = (SQSEvent evnt, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + // foreach (var message in evnt.Records) + // { + // listOfProcessedMessages.Add(message); + // } + + // throw new Exception("Failed to process message"); + // }; + + // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + // .Build() + // .RunAsync(CancellationTokenSource.Token); + + // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + // var startTime = DateTime.UtcNow; + // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + // { + // await Task.Delay(500); + // } + + // Assert.Single(listOfProcessedMessages); + // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + // Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + // } + // finally + // { + // _ = CancellationTokenSource.CancelAsync(); + // await sqsClient.DeleteQueueAsync(queueUrl); + // Console.SetError(consoleError); + // } + // } + + // [RetryFact] + // public async Task PartialFailureResponse() + // { + // var sqsClient = new AmazonSQSClient(); + // var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; + // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + // var consoleError = Console.Error; + // try + // { + // Console.SetError(TextWriter.Null); + // await sqsClient.SendMessageAsync(queueUrl, "Message1"); + + // var lambdaPort = GetFreePort(); + + // // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. + // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); + + // var listOfProcessedMessages = new List(); + // var handler = (SQSEvent evnt, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + // foreach (var message in evnt.Records) + // { + // listOfProcessedMessages.Add(message); + // } + + // var sqsResponse = new SQSBatchResponse(); + // sqsResponse.BatchItemFailures = new List + // { + // new SQSBatchResponse.BatchItemFailure + // { + // ItemIdentifier = evnt.Records[0].MessageId + // } + // }; + + // return sqsResponse; + // }; + + // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + // .Build() + // .RunAsync(CancellationTokenSource.Token); + + // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + // var startTime = DateTime.UtcNow; + // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + // { + // await Task.Delay(500); + // } + + // // Wait for message to be deleted. + // await Task.Delay(2000); + + // // Since the message was never deleted by the event source it should still be eligibl for reading. + // var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); + // Assert.Single(response.Messages); + // } + // finally + // { + // _ = CancellationTokenSource.CancelAsync(); + // await sqsClient.DeleteQueueAsync(queueUrl); + // Console.SetError(consoleError); + // } + // } + + + private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) + { + // Add a delay to handle SQS eventual consistency. + await Task.Delay(5000); + var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); + return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; + } + + private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) + { + Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); + + var environmentVariables = new Dictionary { }; + + if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) + { + var tokens = sqsEventSourceConfig.Split('&'); + if (tokens.Length == 2) + { + sqsEventSourceConfig = tokens[0]; + var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); + var envValue = tokens[1]; + environmentVariables[envName] = envValue; + } + } + + var settings = new RunCommandSettings + { + LambdaEmulatorPort = lambdaPort, + NoLaunchWindow = true, + SQSEventSourceConfig = sqsEventSourceConfig + }; + + + var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); + var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); + _ = command.ExecuteAsync(context, settings, cancellationTokenSource); + + await Task.Delay(2000, cancellationTokenSource.Token); + } } From 8aa7b80056efe999d6e3dfd8a7dec85b1fa0f9ea Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 12:06:52 -0700 Subject: [PATCH 27/42] Add back another test --- .../SQSEventSourceTests.cs | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 0270bc126..a6f9a2a27 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -73,54 +73,54 @@ public async Task ProcessSingleMessage() } } - // [RetryFact] - // public async Task SQSEventSourceComesFromEnvironmentVariable() - // { - // var sqsClient = new AmazonSQSClient(); - // var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; - // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - // var consoleError = Console.Error; - // try - // { - // Console.SetError(TextWriter.Null); + [RetryFact] + public async Task SQSEventSourceComesFromEnvironmentVariable() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); - // var lambdaPort = GetFreePort(); - // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - // var listOfProcessedMessages = new List(); - // var handler = (SQSEvent evnt, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - // foreach (var message in evnt.Records) - // { - // listOfProcessedMessages.Add(message); - // } - // }; + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; - // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - // .Build() - // .RunAsync(CancellationTokenSource.Token); + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); - // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - // var startTime = DateTime.UtcNow; - // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - // { - // await Task.Delay(500); - // } + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } - // Assert.Single(listOfProcessedMessages); - // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - // Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - // } - // finally - // { - // _ = CancellationTokenSource.CancelAsync(); - // await sqsClient.DeleteQueueAsync(queueUrl); - // Console.SetError(consoleError); - // } - // } + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + _ = CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } // [RetryFact] // public async Task ProcessMessagesFromMultipleEventSources() From 81e866779f5202bfab6a685e7926e7bf13100134 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 12:42:15 -0700 Subject: [PATCH 28/42] Add another test --- .../SQSEventSourceTests.cs | 134 +++++++++--------- 1 file changed, 70 insertions(+), 64 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index a6f9a2a27..f52edec6c 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -27,6 +27,8 @@ public SQSEventSourceTests(ITestOutputHelper testOutputHelper) [RetryFact] public async Task ProcessSingleMessage() { + var cancellationSource = new CancellationTokenSource(); + var sqsClient = new AmazonSQSClient(); var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; @@ -36,7 +38,7 @@ public async Task ProcessSingleMessage() Console.SetError(TextWriter.Null); var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); var listOfProcessedMessages = new List(); var handler = (SQSEvent evnt, ILambdaContext context) => @@ -51,7 +53,7 @@ public async Task ProcessSingleMessage() _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") .Build() - .RunAsync(CancellationTokenSource.Token); + .RunAsync(cancellationSource.Token); await sqsClient.SendMessageAsync(queueUrl, "TheBody"); @@ -67,7 +69,7 @@ public async Task ProcessSingleMessage() } finally { - _ = CancellationTokenSource.CancelAsync(); + _ = cancellationSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } @@ -76,8 +78,10 @@ public async Task ProcessSingleMessage() [RetryFact] public async Task SQSEventSourceComesFromEnvironmentVariable() { + var cancellationSource = new CancellationTokenSource(); + var sqsClient = new AmazonSQSClient(); - var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; + var queueName = nameof(SQSEventSourceComesFromEnvironmentVariable) + DateTime.Now.Ticks; var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; var consoleError = Console.Error; try @@ -85,7 +89,7 @@ public async Task SQSEventSourceComesFromEnvironmentVariable() Console.SetError(TextWriter.Null); var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); var listOfProcessedMessages = new List(); var handler = (SQSEvent evnt, ILambdaContext context) => @@ -100,7 +104,7 @@ public async Task SQSEventSourceComesFromEnvironmentVariable() _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") .Build() - .RunAsync(CancellationTokenSource.Token); + .RunAsync(cancellationSource.Token); await sqsClient.SendMessageAsync(queueUrl, "TheBody"); @@ -116,78 +120,80 @@ public async Task SQSEventSourceComesFromEnvironmentVariable() } finally { - _ = CancellationTokenSource.CancelAsync(); + _ = cancellationSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } } - // [RetryFact] - // public async Task ProcessMessagesFromMultipleEventSources() - // { - // var sqsClient = new AmazonSQSClient(); - // var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; - // var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; + [RetryFact] + public async Task ProcessMessagesFromMultipleEventSources() + { + var cancellationSource = new CancellationTokenSource(); - // var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; - // var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; + var sqsClient = new AmazonSQSClient(); + var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; + var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; - // var consoleError = Console.Error; - // try - // { - // Console.SetError(TextWriter.Null); + var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; + var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; - // var sqsEventSourceConfig = """ - //[ - // { - // "QueueUrl" : "queueUrl1", - // "FunctionName" : "SQSProcessor" - // }, - // { - // "QueueUrl" : "queueUrl2", - // "FunctionName" : "SQSProcessor" - // } - //] - //""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); - // var lambdaPort = GetFreePort(); - // var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, CancellationTokenSource); + var sqsEventSourceConfig = """ + [ + { + "QueueUrl" : "queueUrl1", + "FunctionName" : "SQSProcessor" + }, + { + "QueueUrl" : "queueUrl2", + "FunctionName" : "SQSProcessor" + } + ] + """.Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); - // var listOfProcessedMessages = new List(); - // var handler = (SQSEvent evnt, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - // foreach (var message in evnt.Records) - // { - // listOfProcessedMessages.Add(message); - // } - // }; + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, cancellationSource); - // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - // .Build() - // .RunAsync(CancellationTokenSource.Token); + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; - // await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); - // await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(cancellationSource.Token); - // var startTime = DateTime.UtcNow; - // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - // { - // await Task.Delay(500); - // } + await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); + await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); - // Assert.Equal(2, listOfProcessedMessages.Count); - // Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); - // } - // finally - // { - // _ = CancellationTokenSource.CancelAsync(); - // await sqsClient.DeleteQueueAsync(queueUrl1); - // await sqsClient.DeleteQueueAsync(queueUrl2); - // Console.SetError(consoleError); - // } - // } + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Equal(2, listOfProcessedMessages.Count); + Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); + } + finally + { + _ = cancellationSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl1); + await sqsClient.DeleteQueueAsync(queueUrl2); + Console.SetError(consoleError); + } + } // [RetryFact] // public async Task MessageNotDeleted() From bf19b4b4ee964a5332857ca4077f0948f42b5fbf Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 13:16:39 -0700 Subject: [PATCH 29/42] Add cancellation tokens on HTTP requests. --- .../ApiGatewayResponseExtensionsAdditionalTests.cs | 4 ++-- .../BaseApiGatewayTest.cs | 6 +++--- .../Helpers/ApiGatewayHelper.cs | 2 -- .../Helpers/ApiGatewayTestHelper.cs | 4 ++-- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/ApiGatewayResponseExtensionsAdditionalTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/ApiGatewayResponseExtensionsAdditionalTests.cs index d3019fdc5..22d95b857 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/ApiGatewayResponseExtensionsAdditionalTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/ApiGatewayResponseExtensionsAdditionalTests.cs @@ -39,7 +39,7 @@ public async Task ToHttpResponse_RestAPIGatewayV1DecodesBase64() var baseUrl = _fixture.GetAppropriateBaseUrl(ApiGatewayType.RestWithBinarySupport); var url = _fixture.GetRouteUrl(baseUrl, TestRoutes.Ids.DecodeParseBinary); - var actualResponse = await _httpClient.PostAsync(url, new StringContent(JsonSerializer.Serialize(testResponse))); + var actualResponse = await _httpClient.PostAsync(url, new StringContent(JsonSerializer.Serialize(testResponse)), new CancellationTokenSource(5000).Token); await _fixture.ApiGatewayTestHelper.AssertResponsesEqual(actualResponse, httpContext.Response); Assert.Equal(200, (int)actualResponse.StatusCode); var content = await actualResponse.Content.ReadAsStringAsync(); @@ -62,7 +62,7 @@ public async Task ToHttpResponse_HttpV1APIGatewayV1DecodesBase64() var baseUrl = _fixture.GetAppropriateBaseUrl(ApiGatewayType.HttpV1); var url = _fixture.GetRouteUrl(baseUrl, TestRoutes.Ids.ParseAndReturnBody); - var actualResponse = await _httpClient.PostAsync(url, new StringContent(JsonSerializer.Serialize(testResponse))); + var actualResponse = await _httpClient.PostAsync(url, new StringContent(JsonSerializer.Serialize(testResponse)), new CancellationTokenSource(5000).Token); await _fixture.ApiGatewayTestHelper.AssertResponsesEqual(actualResponse, httpContext.Response); Assert.Equal(200, (int)actualResponse.StatusCode); diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index 10abb57a2..60eef5896 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -83,7 +83,7 @@ protected async Task WaitForGatewayHealthCheck(int apiGatewayPort) { try { - var response = await client.GetAsync(healthUrl); + var response = await client.GetAsync(healthUrl, new CancellationTokenSource(10000).Token); if (response.IsSuccessStatusCode) { TestOutputHelper.WriteLine("API Gateway health check succeeded"); @@ -122,8 +122,8 @@ protected async Task TestEndpoint(string routeName, int api { "POST" => await client.PostAsync( $"http://localhost:{apiGatewayPort}/{routeName}", - new StringContent(payload ?? "hello world", Encoding.UTF8, new MediaTypeHeaderValue("text/plain"))), - "GET" => await client.GetAsync($"http://localhost:{apiGatewayPort}/{routeName}"), + new StringContent(payload ?? "hello world", Encoding.UTF8, new MediaTypeHeaderValue("text/plain")), new CancellationTokenSource(5000).Token), + "GET" => await client.GetAsync($"http://localhost:{apiGatewayPort}/{routeName}", new CancellationTokenSource(5000).Token), _ => throw new ArgumentException($"Unsupported HTTP method: {httpMethod}") }; } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Helpers/ApiGatewayHelper.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Helpers/ApiGatewayHelper.cs index 2f1036951..5818bf3e0 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Helpers/ApiGatewayHelper.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Helpers/ApiGatewayHelper.cs @@ -15,13 +15,11 @@ public class ApiGatewayHelper { private readonly IAmazonAPIGateway _apiGatewayV1Client; private readonly IAmazonApiGatewayV2 _apiGatewayV2Client; - private readonly HttpClient _httpClient; public ApiGatewayHelper(IAmazonAPIGateway apiGatewayV1Client, IAmazonApiGatewayV2 apiGatewayV2Client) { _apiGatewayV1Client = apiGatewayV1Client; _apiGatewayV2Client = apiGatewayV2Client; - _httpClient = new HttpClient(); } public async Task WaitForApiAvailability(string apiId, string apiUrl, bool isHttpApi, int maxWaitTimeSeconds = 60) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Helpers/ApiGatewayTestHelper.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Helpers/ApiGatewayTestHelper.cs index 3237a5345..b4584e198 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Helpers/ApiGatewayTestHelper.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/Helpers/ApiGatewayTestHelper.cs @@ -24,7 +24,7 @@ public ApiGatewayTestHelper() httpContext.Response.Body = new MemoryStream(); await testResponse.ToHttpResponseAsync(httpContext, emulatorMode); var serialized = JsonSerializer.Serialize(testResponse); - var actualResponse = await _httpClient.PostAsync(apiUrl, new StringContent(serialized)); + var actualResponse = await _httpClient.PostAsync(apiUrl, new StringContent(serialized), new CancellationTokenSource(5000).Token); return (actualResponse, httpContext.Response); } @@ -34,7 +34,7 @@ public ApiGatewayTestHelper() testResponseHttpContext.Response.Body = new MemoryStream(); await testResponse.ToHttpResponseAsync(testResponseHttpContext); var serialized = JsonSerializer.Serialize(testResponse); - var actualResponse = await _httpClient.PostAsync(apiUrl, new StringContent(serialized)); + var actualResponse = await _httpClient.PostAsync(apiUrl, new StringContent(serialized), new CancellationTokenSource(5000).Token); return (actualResponse, testResponseHttpContext.Response); } From 54e8318a32dcca9296eb6dbab7c60de6750426d1 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 15:07:02 -0700 Subject: [PATCH 30/42] Try rewriting GetFreePort --- .../BaseApiGatewayTest.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index 60eef5896..562655482 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -14,6 +14,10 @@ using System.Text; using System.Net.Http; using System.Net.Http.Headers; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using System.Threading; +using Microsoft.Extensions.Hosting; namespace Amazon.Lambda.TestTool.IntegrationTests; @@ -153,6 +157,23 @@ protected async Task TestEndpoint(string routeName, int api } protected int GetFreePort() + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + var app = builder.Build(); + app.MapGet("/", () => "test"); + + CancellationTokenSource tokenSource = new CancellationTokenSource(); + var runTask = app.RunAsync(tokenSource.Token); + var uri = new Uri(app.Urls.First()); + + tokenSource.Cancel(); + Task.Delay(1000); + + return uri.Port; + } + + protected int GetFreePortOldVersion() { var random = new Random(); var port = random.Next(49152, 65535); From 329c38de47d07a23a5bf829a74ec29360c83f561 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 15:49:24 -0700 Subject: [PATCH 31/42] Add some checks that the Lambda Runtime API started correct if not Assert an error for the failure to start --- .../Commands/RunCommand.cs | 6 ++++++ .../SQSEventSourceTests.cs | 19 ++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs index 3cb4018c8..f366c2f83 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Commands/RunCommand.cs @@ -23,6 +23,11 @@ public sealed class RunCommand( public const string LAMBDA_RUNTIME_API_PORT = "LAMBDA_RUNTIME_API_PORT"; public const string API_GATEWAY_EMULATOR_PORT = "API_GATEWAY_EMULATOR_PORT"; + /// + /// Task for the Lambda Runtime API. + /// + public Task LambdRuntimeApiTask { get; private set; } + /// /// The method responsible for executing the . /// @@ -43,6 +48,7 @@ public override async Task ExecuteAsync(CommandContext context, RunCommandS if (settings.LambdaEmulatorPort.HasValue) { var testToolProcess = TestToolProcess.Startup(settings, cancellationTokenSource.Token); + LambdRuntimeApiTask = testToolProcess.RunningTask; tasks.Add(testToolProcess.RunningTask); if (!settings.NoLaunchWindow) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index f52edec6c..78eb18fe6 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -369,7 +369,8 @@ private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, st return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; } - private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) + // Do not use async/await so we can be sure to hand back the Task that created by RunCommand back to caller. + private Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) { Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); @@ -397,8 +398,20 @@ private async Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSour var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); - _ = command.ExecuteAsync(context, settings, cancellationTokenSource); + Task testToolTask = command.ExecuteAsync(context, settings, cancellationTokenSource); - await Task.Delay(2000, cancellationTokenSource.Token); + var timeout = DateTime.UtcNow.AddMinutes(1); + while (DateTime.UtcNow < timeout && command.LambdRuntimeApiTask == null) + { + Thread.Sleep(100); + } + + Thread.Sleep(2000); + + Assert.NotNull(command.LambdRuntimeApiTask); + Assert.False(command.LambdRuntimeApiTask.IsFaulted, "Task to start Lambda Runtime API failed: " + command.LambdRuntimeApiTask.Exception?.ToString()); + + Thread.Sleep(2000); + return testToolTask; } } From f85380f987bf73fd66668a43e08616e598296621 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 16:15:02 -0700 Subject: [PATCH 32/42] Add back all the SQS tests. --- .../SQSEventSourceBackgroundService.cs | 2 +- .../BaseApiGatewayTest.cs | 20 -- .../SQSEventSourceTests.cs | 328 +++++++++--------- 3 files changed, 165 insertions(+), 185 deletions(-) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs index 0d8f79ece..2125b6f9a 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Processes/SQSEventSource/SQSEventSourceBackgroundService.cs @@ -88,7 +88,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { _logger.LogDebug("No messages received from while polling SQS"); // Since there are no messages, sleep a bit to wait for messages to come. - await Task.Delay(1000); + await Task.Delay(200); continue; } diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index 562655482..4ac09f284 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -173,26 +173,6 @@ protected int GetFreePort() return uri.Port; } - protected int GetFreePortOldVersion() - { - var random = new Random(); - var port = random.Next(49152, 65535); - var listener = new TcpListener(IPAddress.Loopback, port); - try - { - listener.Start(); - return port; - } - catch (SocketException) - { - return GetFreePort(); - } - finally - { - listener.Stop(); - } - } - protected async Task StartTestToolProcessWithNullEndpoint(ApiGatewayEmulatorMode apiGatewayMode, string routeName, int lambdaPort, int apiGatewayPort, CancellationTokenSource cancellationTokenSource) { Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 78eb18fe6..a6fe3b8bd 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -195,170 +195,170 @@ public async Task ProcessMessagesFromMultipleEventSources() } } - // [RetryFact] - // public async Task MessageNotDeleted() - // { - // var sqsClient = new AmazonSQSClient(); - // var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; - // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - // var consoleError = Console.Error; - // try - // { - // Console.SetError(TextWriter.Null); - - // var lambdaPort = GetFreePort(); - // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); - - // var listOfProcessedMessages = new List(); - // var handler = (SQSEvent evnt, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - // foreach (var message in evnt.Records) - // { - // listOfProcessedMessages.Add(message); - // } - // }; - - // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - // .Build() - // .RunAsync(CancellationTokenSource.Token); - - // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - // var startTime = DateTime.UtcNow; - // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - // { - // await Task.Delay(500); - // } - - // Assert.Single(listOfProcessedMessages); - // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - // Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - // } - // finally - // { - // _ = CancellationTokenSource.CancelAsync(); - // await sqsClient.DeleteQueueAsync(queueUrl); - // Console.SetError(consoleError); - // } - // } - - // [RetryFact] - // public async Task LambdaThrowsErrorAndMessageNotDeleted() - // { - // var sqsClient = new AmazonSQSClient(); - // var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; - // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - // var consoleError = Console.Error; - // try - // { - // Console.SetError(TextWriter.Null); - // var lambdaPort = GetFreePort(); - // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); - - // var listOfProcessedMessages = new List(); - // var handler = (SQSEvent evnt, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - // foreach (var message in evnt.Records) - // { - // listOfProcessedMessages.Add(message); - // } - - // throw new Exception("Failed to process message"); - // }; - - // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - // .Build() - // .RunAsync(CancellationTokenSource.Token); - - // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - // var startTime = DateTime.UtcNow; - // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - // { - // await Task.Delay(500); - // } - - // Assert.Single(listOfProcessedMessages); - // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - // Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - // } - // finally - // { - // _ = CancellationTokenSource.CancelAsync(); - // await sqsClient.DeleteQueueAsync(queueUrl); - // Console.SetError(consoleError); - // } - // } - - // [RetryFact] - // public async Task PartialFailureResponse() - // { - // var sqsClient = new AmazonSQSClient(); - // var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; - // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - // var consoleError = Console.Error; - // try - // { - // Console.SetError(TextWriter.Null); - // await sqsClient.SendMessageAsync(queueUrl, "Message1"); - - // var lambdaPort = GetFreePort(); - - // // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. - // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); - - // var listOfProcessedMessages = new List(); - // var handler = (SQSEvent evnt, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - // foreach (var message in evnt.Records) - // { - // listOfProcessedMessages.Add(message); - // } - - // var sqsResponse = new SQSBatchResponse(); - // sqsResponse.BatchItemFailures = new List - // { - // new SQSBatchResponse.BatchItemFailure - // { - // ItemIdentifier = evnt.Records[0].MessageId - // } - // }; - - // return sqsResponse; - // }; - - // _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - // .Build() - // .RunAsync(CancellationTokenSource.Token); - - // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - // var startTime = DateTime.UtcNow; - // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - // { - // await Task.Delay(500); - // } - - // // Wait for message to be deleted. - // await Task.Delay(2000); - - // // Since the message was never deleted by the event source it should still be eligibl for reading. - // var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); - // Assert.Single(response.Messages); - // } - // finally - // { - // _ = CancellationTokenSource.CancelAsync(); - // await sqsClient.DeleteQueueAsync(queueUrl); - // Console.SetError(consoleError); - // } - // } + [RetryFact] + public async Task MessageNotDeleted() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + _ = CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task LambdaThrowsErrorAndMessageNotDeleted() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + + throw new Exception("Failed to process message"); + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + _ = CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task PartialFailureResponse() + { + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + var consoleError = Console.Error; + try + { + Console.SetError(TextWriter.Null); + await sqsClient.SendMessageAsync(queueUrl, "Message1"); + + var lambdaPort = GetFreePort(); + + // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + + var sqsResponse = new SQSBatchResponse(); + sqsResponse.BatchItemFailures = new List + { + new SQSBatchResponse.BatchItemFailure + { + ItemIdentifier = evnt.Records[0].MessageId + } + }; + + return sqsResponse; + }; + + _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(CancellationTokenSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + await Task.Delay(500); + } + + // Wait for message to be deleted. + await Task.Delay(2000); + + // Since the message was never deleted by the event source it should still be eligibl for reading. + var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); + Assert.Single(response.Messages); + } + finally + { + _ = CancellationTokenSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) From e3be00b234dd826dfd0bf47bc0e1a786709c1373 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 16:26:42 -0700 Subject: [PATCH 33/42] Add delay using the SQS queue before using it --- .../SQSEventSourceTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index a6fe3b8bd..1659a8bf7 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -32,6 +32,7 @@ public async Task ProcessSingleMessage() var sqsClient = new AmazonSQSClient(); var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + await Task.Delay(2000); var consoleError = Console.Error; try { @@ -83,6 +84,7 @@ public async Task SQSEventSourceComesFromEnvironmentVariable() var sqsClient = new AmazonSQSClient(); var queueName = nameof(SQSEventSourceComesFromEnvironmentVariable) + DateTime.Now.Ticks; var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + await Task.Delay(2000); var consoleError = Console.Error; try { @@ -137,6 +139,7 @@ public async Task ProcessMessagesFromMultipleEventSources() var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; + await Task.Delay(2000); var consoleError = Console.Error; try @@ -201,6 +204,7 @@ public async Task MessageNotDeleted() var sqsClient = new AmazonSQSClient(); var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + await Task.Delay(2000); var consoleError = Console.Error; try { @@ -250,6 +254,7 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() var sqsClient = new AmazonSQSClient(); var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + await Task.Delay(2000); var consoleError = Console.Error; try { @@ -300,6 +305,7 @@ public async Task PartialFailureResponse() var sqsClient = new AmazonSQSClient(); var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + await Task.Delay(2000); var consoleError = Console.Error; try { From 0daf44df4e551f10156bf4ebaf87fa2811a57ae9 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 16:39:24 -0700 Subject: [PATCH 34/42] Check if Lambda function task is failed --- .../SQSEventSourceTests.cs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 1659a8bf7..0105724ec 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -51,7 +51,7 @@ public async Task ProcessSingleMessage() } }; - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") .Build() .RunAsync(cancellationSource.Token); @@ -61,6 +61,7 @@ public async Task ProcessSingleMessage() var startTime = DateTime.UtcNow; while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); await Task.Delay(500); } @@ -103,7 +104,7 @@ public async Task SQSEventSourceComesFromEnvironmentVariable() } }; - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") .Build() .RunAsync(cancellationSource.Token); @@ -113,6 +114,7 @@ public async Task SQSEventSourceComesFromEnvironmentVariable() var startTime = DateTime.UtcNow; while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); await Task.Delay(500); } @@ -172,7 +174,7 @@ public async Task ProcessMessagesFromMultipleEventSources() } }; - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") .Build() .RunAsync(cancellationSource.Token); @@ -183,6 +185,7 @@ public async Task ProcessMessagesFromMultipleEventSources() var startTime = DateTime.UtcNow; while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); await Task.Delay(500); } @@ -223,7 +226,7 @@ public async Task MessageNotDeleted() } }; - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") .Build() .RunAsync(CancellationTokenSource.Token); @@ -233,6 +236,7 @@ public async Task MessageNotDeleted() var startTime = DateTime.UtcNow; while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); await Task.Delay(500); } @@ -274,7 +278,7 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() throw new Exception("Failed to process message"); }; - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") .Build() .RunAsync(CancellationTokenSource.Token); @@ -284,6 +288,7 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() var startTime = DateTime.UtcNow; while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); await Task.Delay(500); } @@ -338,7 +343,7 @@ public async Task PartialFailureResponse() return sqsResponse; }; - _ = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") .Build() .RunAsync(CancellationTokenSource.Token); @@ -348,6 +353,7 @@ public async Task PartialFailureResponse() var startTime = DateTime.UtcNow; while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); await Task.Delay(500); } From 073f1decd140f8a041fcf488f12847affc5aa04f Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 17:05:47 -0700 Subject: [PATCH 35/42] Tweak tests --- .../SQSEventSourceTests.cs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 0105724ec..4537d1db8 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -204,6 +204,7 @@ public async Task ProcessMessagesFromMultipleEventSources() [RetryFact] public async Task MessageNotDeleted() { + var cancellationSource = new CancellationTokenSource(); var sqsClient = new AmazonSQSClient(); var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; @@ -214,7 +215,7 @@ public async Task MessageNotDeleted() Console.SetError(TextWriter.Null); var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", CancellationTokenSource); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", cancellationSource); var listOfProcessedMessages = new List(); var handler = (SQSEvent evnt, ILambdaContext context) => @@ -229,7 +230,7 @@ public async Task MessageNotDeleted() var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") .Build() - .RunAsync(CancellationTokenSource.Token); + .RunAsync(cancellationSource.Token); await sqsClient.SendMessageAsync(queueUrl, "TheBody"); @@ -246,7 +247,7 @@ public async Task MessageNotDeleted() } finally { - _ = CancellationTokenSource.CancelAsync(); + _ = cancellationSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } @@ -255,6 +256,7 @@ public async Task MessageNotDeleted() [RetryFact] public async Task LambdaThrowsErrorAndMessageNotDeleted() { + var cancellationSource = new CancellationTokenSource(); var sqsClient = new AmazonSQSClient(); var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; @@ -264,7 +266,7 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() { Console.SetError(TextWriter.Null); var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", CancellationTokenSource); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); var listOfProcessedMessages = new List(); var handler = (SQSEvent evnt, ILambdaContext context) => @@ -281,7 +283,7 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") .Build() - .RunAsync(CancellationTokenSource.Token); + .RunAsync(cancellationSource.Token); await sqsClient.SendMessageAsync(queueUrl, "TheBody"); @@ -298,7 +300,7 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() } finally { - _ = CancellationTokenSource.CancelAsync(); + _ = cancellationSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } @@ -307,6 +309,7 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() [RetryFact] public async Task PartialFailureResponse() { + var cancellationSource = new CancellationTokenSource(); var sqsClient = new AmazonSQSClient(); var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; @@ -320,7 +323,7 @@ public async Task PartialFailureResponse() var lambdaPort = GetFreePort(); // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", CancellationTokenSource); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", cancellationSource); var listOfProcessedMessages = new List(); var handler = (SQSEvent evnt, ILambdaContext context) => @@ -346,7 +349,7 @@ public async Task PartialFailureResponse() var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") .Build() - .RunAsync(CancellationTokenSource.Token); + .RunAsync(cancellationSource.Token); await sqsClient.SendMessageAsync(queueUrl, "TheBody"); @@ -366,7 +369,7 @@ public async Task PartialFailureResponse() } finally { - _ = CancellationTokenSource.CancelAsync(); + _ = cancellationSource.CancelAsync(); await sqsClient.DeleteQueueAsync(queueUrl); Console.SetError(consoleError); } From 98b10c010b0265110a576e95d33581e688ee540e Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 17:17:30 -0700 Subject: [PATCH 36/42] Add health check for lambda runtime api --- .../Services/LambdaRuntimeAPI.cs | 2 ++ .../SQSEventSourceTests.cs | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs index f9d11aa54..b05bb85ea 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/Services/LambdaRuntimeAPI.cs @@ -20,6 +20,8 @@ internal LambdaRuntimeApi(WebApplication app) { _runtimeApiDataStoreManager = app.Services.GetRequiredService(); + app.MapGet("/lambda-runtime-api/healthcheck", () => "health"); + app.MapPost("/2015-03-31/functions/function/invocations", (Delegate)PostEventDefaultFunction); app.MapPost("/2015-03-31/functions/{functionName}/invocations", PostEvent); diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 4537d1db8..5b6ada2ff 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -426,6 +426,21 @@ private Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConf Assert.NotNull(command.LambdRuntimeApiTask); Assert.False(command.LambdRuntimeApiTask.IsFaulted, "Task to start Lambda Runtime API failed: " + command.LambdRuntimeApiTask.Exception?.ToString()); + using var httpClient = new HttpClient(); + + var healthCheckUrl = $"http://localhost:{lambdaPort}/lambda-runtime-api/healthcheck"; + TestOutputHelper.WriteLine($"Attempting health check url for Lambda runtime api: {healthCheckUrl}"); + + try + { + var health = httpClient.GetStringAsync(healthCheckUrl).GetAwaiter().GetResult(); + TestOutputHelper.WriteLine($"Get successful health check: {health}"); + } + catch(Exception ex) + { + Assert.Fail($"Failed to make healthcheck: {ex}"); + } + Thread.Sleep(2000); return testToolTask; } From 7ac01d4e4fc756104369a5f3ebcad8577b8f3fe5 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 20:17:43 -0700 Subject: [PATCH 37/42] Try turning on trace logging --- .../src/Amazon.Lambda.TestTool/appsettings.Development.json | 2 +- .../src/Amazon.Lambda.TestTool/appsettings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.Development.json b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.Development.json index 0b8026458..6ae6ebe48 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.Development.json +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.Development.json @@ -1,7 +1,7 @@ { "Logging": { "LogLevel": { - "Default": "Information" + "Default": "Trace" } }, "DetailedErrors": true diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.json b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.json index 8a024f3bd..613e5f6ee 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.json +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.json @@ -1,7 +1,7 @@ { "Logging": { "LogLevel": { - "Default": "Error" + "Default": "Trace" } }, "AllowedHosts": "*" From 33386a5415622b53c3d6b4afaa1853790d9b23e9 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 20:43:23 -0700 Subject: [PATCH 38/42] Try redirecting stderr to stdout in tests --- .../SQSEventSourceTests.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 5b6ada2ff..fc4aa1882 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -36,7 +36,7 @@ public async Task ProcessSingleMessage() var consoleError = Console.Error; try { - Console.SetError(TextWriter.Null); + Console.SetError(Console.Out); var lambdaPort = GetFreePort(); var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); @@ -89,7 +89,7 @@ public async Task SQSEventSourceComesFromEnvironmentVariable() var consoleError = Console.Error; try { - Console.SetError(TextWriter.Null); + Console.SetError(Console.Out); var lambdaPort = GetFreePort(); var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); @@ -146,7 +146,7 @@ public async Task ProcessMessagesFromMultipleEventSources() var consoleError = Console.Error; try { - Console.SetError(TextWriter.Null); + Console.SetError(Console.Out); var sqsEventSourceConfig = """ [ @@ -212,7 +212,7 @@ public async Task MessageNotDeleted() var consoleError = Console.Error; try { - Console.SetError(TextWriter.Null); + Console.SetError(Console.Out); var lambdaPort = GetFreePort(); var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", cancellationSource); @@ -264,7 +264,7 @@ public async Task LambdaThrowsErrorAndMessageNotDeleted() var consoleError = Console.Error; try { - Console.SetError(TextWriter.Null); + Console.SetError(Console.Out); var lambdaPort = GetFreePort(); var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); @@ -317,7 +317,7 @@ public async Task PartialFailureResponse() var consoleError = Console.Error; try { - Console.SetError(TextWriter.Null); + Console.SetError(Console.Out); await sqsClient.SendMessageAsync(queueUrl, "Message1"); var lambdaPort = GetFreePort(); From 9f717785ace006d0da1e172c7e4c02c012852ffb Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 21:15:35 -0700 Subject: [PATCH 39/42] More tweaks to code looking for a free port. --- .../BaseApiGatewayTest.cs | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs index 4ac09f284..f277ffa2a 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/BaseApiGatewayTest.cs @@ -18,6 +18,7 @@ using Microsoft.AspNetCore.Hosting; using System.Threading; using Microsoft.Extensions.Hosting; +using Amazon.SecurityToken; namespace Amazon.Lambda.TestTool.IntegrationTests; @@ -158,17 +159,46 @@ protected async Task TestEndpoint(string routeName, int api protected int GetFreePort() { + Console.WriteLine("Looking for free port"); var builder = WebApplication.CreateBuilder(); builder.WebHost.UseUrls("http://127.0.0.1:0"); var app = builder.Build(); app.MapGet("/", () => "test"); CancellationTokenSource tokenSource = new CancellationTokenSource(); + tokenSource.CancelAfter(5000); var runTask = app.RunAsync(tokenSource.Token); var uri = new Uri(app.Urls.First()); + using var client = new HttpClient(); + string? content = null; + + Console.WriteLine($"Testing port: {uri.Port}"); + var timeout = DateTime.UtcNow.AddSeconds(30); + while(DateTime.UtcNow < timeout) + { + try + { + content = client.GetStringAsync(uri, tokenSource.Token).GetAwaiter().GetResult(); + Console.WriteLine("Port was successful"); + break; + } + catch + { + Thread.Sleep(100); + } + } + + if (!string.Equals(content, "test")) + { + Console.WriteLine("Port test failed trying again"); + var recursivePort = GetFreePort(); + tokenSource.Cancel(); + return recursivePort; + } + tokenSource.Cancel(); - Task.Delay(1000); + Task.Delay(2000); return uri.Port; } From 768286f274bbe661e724d134e6fb677c69d5f68a Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 22:42:13 -0700 Subject: [PATCH 40/42] Try again without SQS integ tests --- .../SQSEventSourceTests.cs | 840 +++++++++--------- 1 file changed, 420 insertions(+), 420 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index fc4aa1882..09f169d6c 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -24,424 +24,424 @@ public SQSEventSourceTests(ITestOutputHelper testOutputHelper) { } - [RetryFact] - public async Task ProcessSingleMessage() - { - var cancellationSource = new CancellationTokenSource(); - - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - await Task.Delay(2000); - var consoleError = Console.Error; - try - { - Console.SetError(Console.Out); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(cancellationSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = cancellationSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task SQSEventSourceComesFromEnvironmentVariable() - { - var cancellationSource = new CancellationTokenSource(); - - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(SQSEventSourceComesFromEnvironmentVariable) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - await Task.Delay(2000); - var consoleError = Console.Error; - try - { - Console.SetError(Console.Out); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(cancellationSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = cancellationSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task ProcessMessagesFromMultipleEventSources() - { - var cancellationSource = new CancellationTokenSource(); - - var sqsClient = new AmazonSQSClient(); - var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; - var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; - - var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; - var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; - await Task.Delay(2000); - - var consoleError = Console.Error; - try - { - Console.SetError(Console.Out); - - var sqsEventSourceConfig = """ - [ - { - "QueueUrl" : "queueUrl1", - "FunctionName" : "SQSProcessor" - }, - { - "QueueUrl" : "queueUrl2", - "FunctionName" : "SQSProcessor" - } - ] - """.Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, cancellationSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(cancellationSource.Token); - - await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); - await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - await Task.Delay(500); - } - - Assert.Equal(2, listOfProcessedMessages.Count); - Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); - } - finally - { - _ = cancellationSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl1); - await sqsClient.DeleteQueueAsync(queueUrl2); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task MessageNotDeleted() - { - var cancellationSource = new CancellationTokenSource(); - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - await Task.Delay(2000); - var consoleError = Console.Error; - try - { - Console.SetError(Console.Out); - - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", cancellationSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - }; - - var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(cancellationSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = cancellationSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task LambdaThrowsErrorAndMessageNotDeleted() - { - var cancellationSource = new CancellationTokenSource(); - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - await Task.Delay(2000); - var consoleError = Console.Error; - try - { - Console.SetError(Console.Out); - var lambdaPort = GetFreePort(); - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - - throw new Exception("Failed to process message"); - }; - - var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(cancellationSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - await Task.Delay(500); - } - - Assert.Single(listOfProcessedMessages); - Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - } - finally - { - _ = cancellationSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - [RetryFact] - public async Task PartialFailureResponse() - { - var cancellationSource = new CancellationTokenSource(); - var sqsClient = new AmazonSQSClient(); - var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; - var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - await Task.Delay(2000); - var consoleError = Console.Error; - try - { - Console.SetError(Console.Out); - await sqsClient.SendMessageAsync(queueUrl, "Message1"); - - var lambdaPort = GetFreePort(); - - // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. - var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", cancellationSource); - - var listOfProcessedMessages = new List(); - var handler = (SQSEvent evnt, ILambdaContext context) => - { - TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - foreach (var message in evnt.Records) - { - listOfProcessedMessages.Add(message); - } - - var sqsResponse = new SQSBatchResponse(); - sqsResponse.BatchItemFailures = new List - { - new SQSBatchResponse.BatchItemFailure - { - ItemIdentifier = evnt.Records[0].MessageId - } - }; - - return sqsResponse; - }; - - var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - .Build() - .RunAsync(cancellationSource.Token); - - await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - var startTime = DateTime.UtcNow; - while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - { - Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - await Task.Delay(500); - } - - // Wait for message to be deleted. - await Task.Delay(2000); - - // Since the message was never deleted by the event source it should still be eligibl for reading. - var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); - Assert.Single(response.Messages); - } - finally - { - _ = cancellationSource.CancelAsync(); - await sqsClient.DeleteQueueAsync(queueUrl); - Console.SetError(consoleError); - } - } - - - private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) - { - // Add a delay to handle SQS eventual consistency. - await Task.Delay(5000); - var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); - return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; - } - - // Do not use async/await so we can be sure to hand back the Task that created by RunCommand back to caller. - private Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) - { - Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); - - var environmentVariables = new Dictionary { }; - - if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) - { - var tokens = sqsEventSourceConfig.Split('&'); - if (tokens.Length == 2) - { - sqsEventSourceConfig = tokens[0]; - var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); - var envValue = tokens[1]; - environmentVariables[envName] = envValue; - } - } - - var settings = new RunCommandSettings - { - LambdaEmulatorPort = lambdaPort, - NoLaunchWindow = true, - SQSEventSourceConfig = sqsEventSourceConfig - }; - - - var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); - var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); - Task testToolTask = command.ExecuteAsync(context, settings, cancellationTokenSource); - - var timeout = DateTime.UtcNow.AddMinutes(1); - while (DateTime.UtcNow < timeout && command.LambdRuntimeApiTask == null) - { - Thread.Sleep(100); - } - - Thread.Sleep(2000); - - Assert.NotNull(command.LambdRuntimeApiTask); - Assert.False(command.LambdRuntimeApiTask.IsFaulted, "Task to start Lambda Runtime API failed: " + command.LambdRuntimeApiTask.Exception?.ToString()); - - using var httpClient = new HttpClient(); - - var healthCheckUrl = $"http://localhost:{lambdaPort}/lambda-runtime-api/healthcheck"; - TestOutputHelper.WriteLine($"Attempting health check url for Lambda runtime api: {healthCheckUrl}"); - - try - { - var health = httpClient.GetStringAsync(healthCheckUrl).GetAwaiter().GetResult(); - TestOutputHelper.WriteLine($"Get successful health check: {health}"); - } - catch(Exception ex) - { - Assert.Fail($"Failed to make healthcheck: {ex}"); - } - - Thread.Sleep(2000); - return testToolTask; - } + //[RetryFact] + //public async Task ProcessSingleMessage() + //{ + // var cancellationSource = new CancellationTokenSource(); + + // var sqsClient = new AmazonSQSClient(); + // var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; + // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + // await Task.Delay(2000); + // var consoleError = Console.Error; + // try + // { + // Console.SetError(Console.Out); + + // var lambdaPort = GetFreePort(); + // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); + + // var listOfProcessedMessages = new List(); + // var handler = (SQSEvent evnt, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + // foreach (var message in evnt.Records) + // { + // listOfProcessedMessages.Add(message); + // } + // }; + + // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + // .Build() + // .RunAsync(cancellationSource.Token); + + // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + // var startTime = DateTime.UtcNow; + // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + // { + // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + // await Task.Delay(500); + // } + + // Assert.Single(listOfProcessedMessages); + // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + // Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + // } + // finally + // { + // _ = cancellationSource.CancelAsync(); + // await sqsClient.DeleteQueueAsync(queueUrl); + // Console.SetError(consoleError); + // } + //} + + //[RetryFact] + //public async Task SQSEventSourceComesFromEnvironmentVariable() + //{ + // var cancellationSource = new CancellationTokenSource(); + + // var sqsClient = new AmazonSQSClient(); + // var queueName = nameof(SQSEventSourceComesFromEnvironmentVariable) + DateTime.Now.Ticks; + // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + // await Task.Delay(2000); + // var consoleError = Console.Error; + // try + // { + // Console.SetError(Console.Out); + + // var lambdaPort = GetFreePort(); + // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); + + // var listOfProcessedMessages = new List(); + // var handler = (SQSEvent evnt, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + // foreach (var message in evnt.Records) + // { + // listOfProcessedMessages.Add(message); + // } + // }; + + // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + // .Build() + // .RunAsync(cancellationSource.Token); + + // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + // var startTime = DateTime.UtcNow; + // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + // { + // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + // await Task.Delay(500); + // } + + // Assert.Single(listOfProcessedMessages); + // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + // Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + // } + // finally + // { + // _ = cancellationSource.CancelAsync(); + // await sqsClient.DeleteQueueAsync(queueUrl); + // Console.SetError(consoleError); + // } + //} + + //[RetryFact] + //public async Task ProcessMessagesFromMultipleEventSources() + //{ + // var cancellationSource = new CancellationTokenSource(); + + // var sqsClient = new AmazonSQSClient(); + // var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; + // var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; + + // var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; + // var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; + // await Task.Delay(2000); + + // var consoleError = Console.Error; + // try + // { + // Console.SetError(Console.Out); + + // var sqsEventSourceConfig = """ + //[ + // { + // "QueueUrl" : "queueUrl1", + // "FunctionName" : "SQSProcessor" + // }, + // { + // "QueueUrl" : "queueUrl2", + // "FunctionName" : "SQSProcessor" + // } + //] + //""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); + + // var lambdaPort = GetFreePort(); + // var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, cancellationSource); + + // var listOfProcessedMessages = new List(); + // var handler = (SQSEvent evnt, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + // foreach (var message in evnt.Records) + // { + // listOfProcessedMessages.Add(message); + // } + // }; + + // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + // .Build() + // .RunAsync(cancellationSource.Token); + + // await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); + // await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); + + // var startTime = DateTime.UtcNow; + // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + // { + // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + // await Task.Delay(500); + // } + + // Assert.Equal(2, listOfProcessedMessages.Count); + // Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); + // } + // finally + // { + // _ = cancellationSource.CancelAsync(); + // await sqsClient.DeleteQueueAsync(queueUrl1); + // await sqsClient.DeleteQueueAsync(queueUrl2); + // Console.SetError(consoleError); + // } + //} + + //[RetryFact] + //public async Task MessageNotDeleted() + //{ + // var cancellationSource = new CancellationTokenSource(); + // var sqsClient = new AmazonSQSClient(); + // var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; + // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + // await Task.Delay(2000); + // var consoleError = Console.Error; + // try + // { + // Console.SetError(Console.Out); + + // var lambdaPort = GetFreePort(); + // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", cancellationSource); + + // var listOfProcessedMessages = new List(); + // var handler = (SQSEvent evnt, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + // foreach (var message in evnt.Records) + // { + // listOfProcessedMessages.Add(message); + // } + // }; + + // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + // .Build() + // .RunAsync(cancellationSource.Token); + + // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + // var startTime = DateTime.UtcNow; + // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + // { + // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + // await Task.Delay(500); + // } + + // Assert.Single(listOfProcessedMessages); + // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + // Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + // } + // finally + // { + // _ = cancellationSource.CancelAsync(); + // await sqsClient.DeleteQueueAsync(queueUrl); + // Console.SetError(consoleError); + // } + //} + + //[RetryFact] + //public async Task LambdaThrowsErrorAndMessageNotDeleted() + //{ + // var cancellationSource = new CancellationTokenSource(); + // var sqsClient = new AmazonSQSClient(); + // var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; + // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + // await Task.Delay(2000); + // var consoleError = Console.Error; + // try + // { + // Console.SetError(Console.Out); + // var lambdaPort = GetFreePort(); + // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); + + // var listOfProcessedMessages = new List(); + // var handler = (SQSEvent evnt, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + // foreach (var message in evnt.Records) + // { + // listOfProcessedMessages.Add(message); + // } + + // throw new Exception("Failed to process message"); + // }; + + // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + // .Build() + // .RunAsync(cancellationSource.Token); + + // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + // var startTime = DateTime.UtcNow; + // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + // { + // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + // await Task.Delay(500); + // } + + // Assert.Single(listOfProcessedMessages); + // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + // Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + // } + // finally + // { + // _ = cancellationSource.CancelAsync(); + // await sqsClient.DeleteQueueAsync(queueUrl); + // Console.SetError(consoleError); + // } + //} + + //[RetryFact] + //public async Task PartialFailureResponse() + //{ + // var cancellationSource = new CancellationTokenSource(); + // var sqsClient = new AmazonSQSClient(); + // var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; + // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + // await Task.Delay(2000); + // var consoleError = Console.Error; + // try + // { + // Console.SetError(Console.Out); + // await sqsClient.SendMessageAsync(queueUrl, "Message1"); + + // var lambdaPort = GetFreePort(); + + // // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. + // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", cancellationSource); + + // var listOfProcessedMessages = new List(); + // var handler = (SQSEvent evnt, ILambdaContext context) => + // { + // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + // foreach (var message in evnt.Records) + // { + // listOfProcessedMessages.Add(message); + // } + + // var sqsResponse = new SQSBatchResponse(); + // sqsResponse.BatchItemFailures = new List + // { + // new SQSBatchResponse.BatchItemFailure + // { + // ItemIdentifier = evnt.Records[0].MessageId + // } + // }; + + // return sqsResponse; + // }; + + // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + // .Build() + // .RunAsync(cancellationSource.Token); + + // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + // var startTime = DateTime.UtcNow; + // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + // { + // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + // await Task.Delay(500); + // } + + // // Wait for message to be deleted. + // await Task.Delay(2000); + + // // Since the message was never deleted by the event source it should still be eligibl for reading. + // var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); + // Assert.Single(response.Messages); + // } + // finally + // { + // _ = cancellationSource.CancelAsync(); + // await sqsClient.DeleteQueueAsync(queueUrl); + // Console.SetError(consoleError); + // } + //} + + + //private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) + //{ + // // Add a delay to handle SQS eventual consistency. + // await Task.Delay(5000); + // var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); + // return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; + //} + + //// Do not use async/await so we can be sure to hand back the Task that created by RunCommand back to caller. + //private Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) + //{ + // Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); + + // var environmentVariables = new Dictionary { }; + + // if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) + // { + // var tokens = sqsEventSourceConfig.Split('&'); + // if (tokens.Length == 2) + // { + // sqsEventSourceConfig = tokens[0]; + // var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); + // var envValue = tokens[1]; + // environmentVariables[envName] = envValue; + // } + // } + + // var settings = new RunCommandSettings + // { + // LambdaEmulatorPort = lambdaPort, + // NoLaunchWindow = true, + // SQSEventSourceConfig = sqsEventSourceConfig + // }; + + + // var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); + // var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); + // Task testToolTask = command.ExecuteAsync(context, settings, cancellationTokenSource); + + // var timeout = DateTime.UtcNow.AddMinutes(1); + // while (DateTime.UtcNow < timeout && command.LambdRuntimeApiTask == null) + // { + // Thread.Sleep(100); + // } + + // Thread.Sleep(2000); + + // Assert.NotNull(command.LambdRuntimeApiTask); + // Assert.False(command.LambdRuntimeApiTask.IsFaulted, "Task to start Lambda Runtime API failed: " + command.LambdRuntimeApiTask.Exception?.ToString()); + + // using var httpClient = new HttpClient(); + + // var healthCheckUrl = $"http://localhost:{lambdaPort}/lambda-runtime-api/healthcheck"; + // TestOutputHelper.WriteLine($"Attempting health check url for Lambda runtime api: {healthCheckUrl}"); + + // try + // { + // var health = httpClient.GetStringAsync(healthCheckUrl).GetAwaiter().GetResult(); + // TestOutputHelper.WriteLine($"Get successful health check: {health}"); + // } + // catch(Exception ex) + // { + // Assert.Fail($"Failed to make healthcheck: {ex}"); + // } + + // Thread.Sleep(2000); + // return testToolTask; + //} } From 6c561674dfb83abee96693977f71b3f5ee5fae0a Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 23:00:00 -0700 Subject: [PATCH 41/42] Switch logging back to Error --- .../src/Amazon.Lambda.TestTool/appsettings.Development.json | 2 +- .../src/Amazon.Lambda.TestTool/appsettings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.Development.json b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.Development.json index 6ae6ebe48..c95571f6b 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.Development.json +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.Development.json @@ -1,7 +1,7 @@ { "Logging": { "LogLevel": { - "Default": "Trace" + "Default": "Error" } }, "DetailedErrors": true diff --git a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.json b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.json index 613e5f6ee..8a024f3bd 100644 --- a/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.json +++ b/Tools/LambdaTestTool-v2/src/Amazon.Lambda.TestTool/appsettings.json @@ -1,7 +1,7 @@ { "Logging": { "LogLevel": { - "Default": "Trace" + "Default": "Error" } }, "AllowedHosts": "*" From b7bcbd9f26177daea3115bd0a9f30124046fbd51 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Thu, 3 Apr 2025 23:44:47 -0700 Subject: [PATCH 42/42] Add Tests --- .../SQSEventSourceTests.cs | 840 +++++++++--------- 1 file changed, 420 insertions(+), 420 deletions(-) diff --git a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs index 09f169d6c..fc4aa1882 100644 --- a/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs +++ b/Tools/LambdaTestTool-v2/tests/Amazon.Lambda.TestTool.IntegrationTests/SQSEventSourceTests.cs @@ -24,424 +24,424 @@ public SQSEventSourceTests(ITestOutputHelper testOutputHelper) { } - //[RetryFact] - //public async Task ProcessSingleMessage() - //{ - // var cancellationSource = new CancellationTokenSource(); - - // var sqsClient = new AmazonSQSClient(); - // var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; - // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - // await Task.Delay(2000); - // var consoleError = Console.Error; - // try - // { - // Console.SetError(Console.Out); - - // var lambdaPort = GetFreePort(); - // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); - - // var listOfProcessedMessages = new List(); - // var handler = (SQSEvent evnt, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - // foreach (var message in evnt.Records) - // { - // listOfProcessedMessages.Add(message); - // } - // }; - - // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - // .Build() - // .RunAsync(cancellationSource.Token); - - // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - // var startTime = DateTime.UtcNow; - // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - // { - // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - // await Task.Delay(500); - // } - - // Assert.Single(listOfProcessedMessages); - // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - // Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - // } - // finally - // { - // _ = cancellationSource.CancelAsync(); - // await sqsClient.DeleteQueueAsync(queueUrl); - // Console.SetError(consoleError); - // } - //} - - //[RetryFact] - //public async Task SQSEventSourceComesFromEnvironmentVariable() - //{ - // var cancellationSource = new CancellationTokenSource(); - - // var sqsClient = new AmazonSQSClient(); - // var queueName = nameof(SQSEventSourceComesFromEnvironmentVariable) + DateTime.Now.Ticks; - // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - // await Task.Delay(2000); - // var consoleError = Console.Error; - // try - // { - // Console.SetError(Console.Out); - - // var lambdaPort = GetFreePort(); - // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); - - // var listOfProcessedMessages = new List(); - // var handler = (SQSEvent evnt, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - // foreach (var message in evnt.Records) - // { - // listOfProcessedMessages.Add(message); - // } - // }; - - // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - // .Build() - // .RunAsync(cancellationSource.Token); - - // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - // var startTime = DateTime.UtcNow; - // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - // { - // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - // await Task.Delay(500); - // } - - // Assert.Single(listOfProcessedMessages); - // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - // Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - // } - // finally - // { - // _ = cancellationSource.CancelAsync(); - // await sqsClient.DeleteQueueAsync(queueUrl); - // Console.SetError(consoleError); - // } - //} - - //[RetryFact] - //public async Task ProcessMessagesFromMultipleEventSources() - //{ - // var cancellationSource = new CancellationTokenSource(); - - // var sqsClient = new AmazonSQSClient(); - // var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; - // var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; - - // var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; - // var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; - // await Task.Delay(2000); - - // var consoleError = Console.Error; - // try - // { - // Console.SetError(Console.Out); - - // var sqsEventSourceConfig = """ - //[ - // { - // "QueueUrl" : "queueUrl1", - // "FunctionName" : "SQSProcessor" - // }, - // { - // "QueueUrl" : "queueUrl2", - // "FunctionName" : "SQSProcessor" - // } - //] - //""".Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); - - // var lambdaPort = GetFreePort(); - // var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, cancellationSource); - - // var listOfProcessedMessages = new List(); - // var handler = (SQSEvent evnt, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - // foreach (var message in evnt.Records) - // { - // listOfProcessedMessages.Add(message); - // } - // }; - - // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - // .Build() - // .RunAsync(cancellationSource.Token); - - // await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); - // await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); - - // var startTime = DateTime.UtcNow; - // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - // { - // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - // await Task.Delay(500); - // } - - // Assert.Equal(2, listOfProcessedMessages.Count); - // Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); - // } - // finally - // { - // _ = cancellationSource.CancelAsync(); - // await sqsClient.DeleteQueueAsync(queueUrl1); - // await sqsClient.DeleteQueueAsync(queueUrl2); - // Console.SetError(consoleError); - // } - //} - - //[RetryFact] - //public async Task MessageNotDeleted() - //{ - // var cancellationSource = new CancellationTokenSource(); - // var sqsClient = new AmazonSQSClient(); - // var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; - // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - // await Task.Delay(2000); - // var consoleError = Console.Error; - // try - // { - // Console.SetError(Console.Out); - - // var lambdaPort = GetFreePort(); - // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", cancellationSource); - - // var listOfProcessedMessages = new List(); - // var handler = (SQSEvent evnt, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - // foreach (var message in evnt.Records) - // { - // listOfProcessedMessages.Add(message); - // } - // }; - - // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - // .Build() - // .RunAsync(cancellationSource.Token); - - // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - // var startTime = DateTime.UtcNow; - // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - // { - // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - // await Task.Delay(500); - // } - - // Assert.Single(listOfProcessedMessages); - // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - // Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - // } - // finally - // { - // _ = cancellationSource.CancelAsync(); - // await sqsClient.DeleteQueueAsync(queueUrl); - // Console.SetError(consoleError); - // } - //} - - //[RetryFact] - //public async Task LambdaThrowsErrorAndMessageNotDeleted() - //{ - // var cancellationSource = new CancellationTokenSource(); - // var sqsClient = new AmazonSQSClient(); - // var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; - // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - // await Task.Delay(2000); - // var consoleError = Console.Error; - // try - // { - // Console.SetError(Console.Out); - // var lambdaPort = GetFreePort(); - // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); - - // var listOfProcessedMessages = new List(); - // var handler = (SQSEvent evnt, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - // foreach (var message in evnt.Records) - // { - // listOfProcessedMessages.Add(message); - // } - - // throw new Exception("Failed to process message"); - // }; - - // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - // .Build() - // .RunAsync(cancellationSource.Token); - - // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - // var startTime = DateTime.UtcNow; - // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - // { - // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - // await Task.Delay(500); - // } - - // Assert.Single(listOfProcessedMessages); - // Assert.Equal("TheBody", listOfProcessedMessages[0].Body); - // Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); - // } - // finally - // { - // _ = cancellationSource.CancelAsync(); - // await sqsClient.DeleteQueueAsync(queueUrl); - // Console.SetError(consoleError); - // } - //} - - //[RetryFact] - //public async Task PartialFailureResponse() - //{ - // var cancellationSource = new CancellationTokenSource(); - // var sqsClient = new AmazonSQSClient(); - // var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; - // var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; - // await Task.Delay(2000); - // var consoleError = Console.Error; - // try - // { - // Console.SetError(Console.Out); - // await sqsClient.SendMessageAsync(queueUrl, "Message1"); - - // var lambdaPort = GetFreePort(); - - // // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. - // var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", cancellationSource); - - // var listOfProcessedMessages = new List(); - // var handler = (SQSEvent evnt, ILambdaContext context) => - // { - // TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); - // foreach (var message in evnt.Records) - // { - // listOfProcessedMessages.Add(message); - // } - - // var sqsResponse = new SQSBatchResponse(); - // sqsResponse.BatchItemFailures = new List - // { - // new SQSBatchResponse.BatchItemFailure - // { - // ItemIdentifier = evnt.Records[0].MessageId - // } - // }; - - // return sqsResponse; - // }; - - // var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) - // .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") - // .Build() - // .RunAsync(cancellationSource.Token); - - // await sqsClient.SendMessageAsync(queueUrl, "TheBody"); - - // var startTime = DateTime.UtcNow; - // while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) - // { - // Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); - // await Task.Delay(500); - // } - - // // Wait for message to be deleted. - // await Task.Delay(2000); - - // // Since the message was never deleted by the event source it should still be eligibl for reading. - // var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); - // Assert.Single(response.Messages); - // } - // finally - // { - // _ = cancellationSource.CancelAsync(); - // await sqsClient.DeleteQueueAsync(queueUrl); - // Console.SetError(consoleError); - // } - //} - - - //private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) - //{ - // // Add a delay to handle SQS eventual consistency. - // await Task.Delay(5000); - // var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); - // return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; - //} - - //// Do not use async/await so we can be sure to hand back the Task that created by RunCommand back to caller. - //private Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) - //{ - // Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); - - // var environmentVariables = new Dictionary { }; - - // if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) - // { - // var tokens = sqsEventSourceConfig.Split('&'); - // if (tokens.Length == 2) - // { - // sqsEventSourceConfig = tokens[0]; - // var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); - // var envValue = tokens[1]; - // environmentVariables[envName] = envValue; - // } - // } - - // var settings = new RunCommandSettings - // { - // LambdaEmulatorPort = lambdaPort, - // NoLaunchWindow = true, - // SQSEventSourceConfig = sqsEventSourceConfig - // }; - - - // var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); - // var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); - // Task testToolTask = command.ExecuteAsync(context, settings, cancellationTokenSource); - - // var timeout = DateTime.UtcNow.AddMinutes(1); - // while (DateTime.UtcNow < timeout && command.LambdRuntimeApiTask == null) - // { - // Thread.Sleep(100); - // } - - // Thread.Sleep(2000); - - // Assert.NotNull(command.LambdRuntimeApiTask); - // Assert.False(command.LambdRuntimeApiTask.IsFaulted, "Task to start Lambda Runtime API failed: " + command.LambdRuntimeApiTask.Exception?.ToString()); - - // using var httpClient = new HttpClient(); - - // var healthCheckUrl = $"http://localhost:{lambdaPort}/lambda-runtime-api/healthcheck"; - // TestOutputHelper.WriteLine($"Attempting health check url for Lambda runtime api: {healthCheckUrl}"); - - // try - // { - // var health = httpClient.GetStringAsync(healthCheckUrl).GetAwaiter().GetResult(); - // TestOutputHelper.WriteLine($"Get successful health check: {health}"); - // } - // catch(Exception ex) - // { - // Assert.Fail($"Failed to make healthcheck: {ex}"); - // } - - // Thread.Sleep(2000); - // return testToolTask; - //} + [RetryFact] + public async Task ProcessSingleMessage() + { + var cancellationSource = new CancellationTokenSource(); + + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(ProcessSingleMessage) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + await Task.Delay(2000); + var consoleError = Console.Error; + try + { + Console.SetError(Console.Out); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(cancellationSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + _ = cancellationSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task SQSEventSourceComesFromEnvironmentVariable() + { + var cancellationSource = new CancellationTokenSource(); + + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(SQSEventSourceComesFromEnvironmentVariable) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + await Task.Delay(2000); + var consoleError = Console.Error; + try + { + Console.SetError(Console.Out); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"env:SQS_CONFIG&QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(cancellationSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(0, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + _ = cancellationSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task ProcessMessagesFromMultipleEventSources() + { + var cancellationSource = new CancellationTokenSource(); + + var sqsClient = new AmazonSQSClient(); + var queueName1 = nameof(ProcessMessagesFromMultipleEventSources) + "-1-" + DateTime.Now.Ticks; + var queueUrl1 = (await sqsClient.CreateQueueAsync(queueName1)).QueueUrl; + + var queueName2 = nameof(ProcessMessagesFromMultipleEventSources) + "-2-" + DateTime.Now.Ticks; + var queueUrl2 = (await sqsClient.CreateQueueAsync(queueName2)).QueueUrl; + await Task.Delay(2000); + + var consoleError = Console.Error; + try + { + Console.SetError(Console.Out); + + var sqsEventSourceConfig = """ + [ + { + "QueueUrl" : "queueUrl1", + "FunctionName" : "SQSProcessor" + }, + { + "QueueUrl" : "queueUrl2", + "FunctionName" : "SQSProcessor" + } + ] + """.Replace("queueUrl1", queueUrl1).Replace("queueUrl2", queueUrl2); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, sqsEventSourceConfig, cancellationSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(cancellationSource.Token); + + await sqsClient.SendMessageAsync(queueUrl1, "MessageFromQueue1"); + await sqsClient.SendMessageAsync(queueUrl2, "MessageFromQueue2"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + await Task.Delay(500); + } + + Assert.Equal(2, listOfProcessedMessages.Count); + Assert.NotEqual(listOfProcessedMessages[0].EventSourceArn, listOfProcessedMessages[1].EventSourceArn); + } + finally + { + _ = cancellationSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl1); + await sqsClient.DeleteQueueAsync(queueUrl2); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task MessageNotDeleted() + { + var cancellationSource = new CancellationTokenSource(); + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(MessageNotDeleted) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + await Task.Delay(2000); + var consoleError = Console.Error; + try + { + Console.SetError(Console.Out); + + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,DisableMessageDelete=true", cancellationSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + }; + + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(cancellationSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + _ = cancellationSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task LambdaThrowsErrorAndMessageNotDeleted() + { + var cancellationSource = new CancellationTokenSource(); + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(LambdaThrowsErrorAndMessageNotDeleted) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + await Task.Delay(2000); + var consoleError = Console.Error; + try + { + Console.SetError(Console.Out); + var lambdaPort = GetFreePort(); + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor", cancellationSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + + throw new Exception("Failed to process message"); + }; + + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(cancellationSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + await Task.Delay(500); + } + + Assert.Single(listOfProcessedMessages); + Assert.Equal("TheBody", listOfProcessedMessages[0].Body); + Assert.Equal(1, await GetNumberOfMessagesInQueueAsync(sqsClient, queueUrl)); + } + finally + { + _ = cancellationSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + [RetryFact] + public async Task PartialFailureResponse() + { + var cancellationSource = new CancellationTokenSource(); + var sqsClient = new AmazonSQSClient(); + var queueName = nameof(PartialFailureResponse) + DateTime.Now.Ticks; + var queueUrl = (await sqsClient.CreateQueueAsync(queueName)).QueueUrl; + await Task.Delay(2000); + var consoleError = Console.Error; + try + { + Console.SetError(Console.Out); + await sqsClient.SendMessageAsync(queueUrl, "Message1"); + + var lambdaPort = GetFreePort(); + + // Lower VisibilityTimeout to speed up receiving the message at the end to prove the message wasn't deleted. + var testToolTask = StartTestToolProcessAsync(lambdaPort, $"QueueUrl={queueUrl},FunctionName=SQSProcessor,VisibilityTimeout=3", cancellationSource); + + var listOfProcessedMessages = new List(); + var handler = (SQSEvent evnt, ILambdaContext context) => + { + TestOutputHelper.WriteLine($"Lambda handler called with {evnt.Records.Count} messages"); + foreach (var message in evnt.Records) + { + listOfProcessedMessages.Add(message); + } + + var sqsResponse = new SQSBatchResponse(); + sqsResponse.BatchItemFailures = new List + { + new SQSBatchResponse.BatchItemFailure + { + ItemIdentifier = evnt.Records[0].MessageId + } + }; + + return sqsResponse; + }; + + var lambdaTask = LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer()) + .ConfigureOptions(x => x.RuntimeApiEndpoint = $"localhost:{lambdaPort}/SQSProcessor") + .Build() + .RunAsync(cancellationSource.Token); + + await sqsClient.SendMessageAsync(queueUrl, "TheBody"); + + var startTime = DateTime.UtcNow; + while (listOfProcessedMessages.Count == 0 && DateTime.UtcNow < startTime.AddMinutes(2)) + { + Assert.False(lambdaTask.IsFaulted, "Lambda function failed: " + lambdaTask.Exception?.ToString()); + await Task.Delay(500); + } + + // Wait for message to be deleted. + await Task.Delay(2000); + + // Since the message was never deleted by the event source it should still be eligibl for reading. + var response = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = queueUrl, WaitTimeSeconds = 20 }); + Assert.Single(response.Messages); + } + finally + { + _ = cancellationSource.CancelAsync(); + await sqsClient.DeleteQueueAsync(queueUrl); + Console.SetError(consoleError); + } + } + + + private async Task GetNumberOfMessagesInQueueAsync(IAmazonSQS sqsClient, string queueUrl) + { + // Add a delay to handle SQS eventual consistency. + await Task.Delay(5000); + var response = await sqsClient.GetQueueAttributesAsync(queueUrl, new List { "All" }); + return response.ApproximateNumberOfMessages + response.ApproximateNumberOfMessagesNotVisible; + } + + // Do not use async/await so we can be sure to hand back the Task that created by RunCommand back to caller. + private Task StartTestToolProcessAsync(int lambdaPort, string sqsEventSourceConfig, CancellationTokenSource cancellationTokenSource) + { + Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); + + var environmentVariables = new Dictionary { }; + + if (sqsEventSourceConfig.StartsWith(Constants.ArgumentEnvironmentVariablePrefix)) + { + var tokens = sqsEventSourceConfig.Split('&'); + if (tokens.Length == 2) + { + sqsEventSourceConfig = tokens[0]; + var envName = tokens[0].Replace(Constants.ArgumentEnvironmentVariablePrefix, ""); + var envValue = tokens[1]; + environmentVariables[envName] = envValue; + } + } + + var settings = new RunCommandSettings + { + LambdaEmulatorPort = lambdaPort, + NoLaunchWindow = true, + SQSEventSourceConfig = sqsEventSourceConfig + }; + + + var command = new RunCommand(MockInteractiveService.Object, new TestEnvironmentManager(environmentVariables)); + var context = new CommandContext(new List(), MockRemainingArgs.Object, "run", null); + Task testToolTask = command.ExecuteAsync(context, settings, cancellationTokenSource); + + var timeout = DateTime.UtcNow.AddMinutes(1); + while (DateTime.UtcNow < timeout && command.LambdRuntimeApiTask == null) + { + Thread.Sleep(100); + } + + Thread.Sleep(2000); + + Assert.NotNull(command.LambdRuntimeApiTask); + Assert.False(command.LambdRuntimeApiTask.IsFaulted, "Task to start Lambda Runtime API failed: " + command.LambdRuntimeApiTask.Exception?.ToString()); + + using var httpClient = new HttpClient(); + + var healthCheckUrl = $"http://localhost:{lambdaPort}/lambda-runtime-api/healthcheck"; + TestOutputHelper.WriteLine($"Attempting health check url for Lambda runtime api: {healthCheckUrl}"); + + try + { + var health = httpClient.GetStringAsync(healthCheckUrl).GetAwaiter().GetResult(); + TestOutputHelper.WriteLine($"Get successful health check: {health}"); + } + catch(Exception ex) + { + Assert.Fail($"Failed to make healthcheck: {ex}"); + } + + Thread.Sleep(2000); + return testToolTask; + } }