diff --git a/.autover/changes/4133a18f-b6f9-4f24-886e-217756fbb670.json b/.autover/changes/4133a18f-b6f9-4f24-886e-217756fbb670.json new file mode 100644 index 000000000..93522a7b9 --- /dev/null +++ b/.autover/changes/4133a18f-b6f9-4f24-886e-217756fbb670.json @@ -0,0 +1,11 @@ +{ + "Projects": [ + { + "Name": "Amazon.Lambda.RuntimeSupport", + "Type": "Minor", + "ChangelogMessages": [ + "Add support for the Lambda-Runtime-Invocation-Id header for cross-wiring invoke protection. The runtime echoes the header back on the response and error calls when the Runtime API provides it, and treats an HTTP 410 Gone (invoke timeout) response as a non-fatal condition, logging it and continuing to the next invocation." + ] + } + ] +} diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs index 9b8186dda..87445c768 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs @@ -380,11 +380,17 @@ internal async Task InvokeOnceAsync(CancellationToken cancellationToken = defaul Func processingFunc = async () => { + // Per-invocation id echoed back on /response and /error for cross-wiring protection. + // Null when the Runtime API did not send the Lambda-Runtime-Invocation-Id header, in + // which case nothing is echoed. Captured as a local so concurrent invocations in + // multi-concurrency mode each carry their own id. + string invocationId = null; if (invocation.LambdaContext is LambdaContext impl) { Client.ConsoleLogger.SetRuntimeHeaders(impl.RuntimeApiHeaders); SetInvocationTraceId(impl.RuntimeApiHeaders.TraceId); SetSerializerOnContext(impl); + invocationId = impl.InvocationId; } // Initialize ResponseStreamFactory — includes RuntimeApiClient reference @@ -395,7 +401,8 @@ internal async Task InvokeOnceAsync(CancellationToken cancellationToken = defaul invocation.LambdaContext.AwsRequestId, isMultiConcurrency, runtimeApiClient, - cancellationToken); + cancellationToken, + invocationId); } try @@ -420,7 +427,7 @@ internal async Task InvokeOnceAsync(CancellationToken cancellationToken = defaul } else { - await Client.ReportInvocationErrorAsync(invocation.LambdaContext.AwsRequestId, exception, cancellationToken); + await Client.ReportInvocationErrorAsync(invocation.LambdaContext.AwsRequestId, invocationId, exception, cancellationToken); } } finally @@ -450,7 +457,7 @@ internal async Task InvokeOnceAsync(CancellationToken cancellationToken = defaul _logger.LogInformation("Starting sending response"); try { - await Client.SendResponseAsync(invocation.LambdaContext.AwsRequestId, response?.OutputStream, cancellationToken); + await Client.SendResponseAsync(invocation.LambdaContext.AwsRequestId, invocationId, response?.OutputStream, cancellationToken); } finally { @@ -465,6 +472,15 @@ internal async Task InvokeOnceAsync(CancellationToken cancellationToken = defaul _logger.LogInformation("Finished InvokeOnceAsync"); } + catch (RuntimeApiInvokeTimeoutException timeout) + { + // The Runtime API rejected the response or error with HTTP 410 Gone because the + // invocation had already timed out (cross-wiring protection). This is expected and + // not fatal — the response would have been discarded anyway. Log via the internal + // logger (not the customer log) and loop back to /next in both on-demand and + // multi-concurrency modes rather than crashing the process. + _logger.LogInformation($"Invocation {timeout.AwsRequestId} timed out before its response was submitted; the Runtime API rejected it. Continuing to the next invocation."); + } catch(Exception ex) { // Only capture and continue for multi concurrency because we do not want to change diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs index 11f2f1a49..6df4336fd 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs @@ -62,11 +62,13 @@ public RawStreamingHttpClient(string hostAndPort) /// for error reporting. /// /// The Lambda request ID. + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// The response stream that provides data and error state. /// The User-Agent header value. /// Cancellation token. public async Task SendStreamingResponseAsync( string awsRequestId, + string invocationId, ResponseStream responseStream, string userAgent, CancellationToken cancellationToken = default) @@ -86,6 +88,10 @@ public async Task SendStreamingResponseAsync( headers.Append($"{StreamingConstants.ResponseModeHeader}: {StreamingConstants.StreamingResponseMode}\r\n"); headers.Append("Transfer-Encoding: chunked\r\n"); headers.Append($"Trailer: {StreamingConstants.ErrorTypeTrailer}, {StreamingConstants.ErrorBodyTrailer}\r\n"); + // Echo the per-invocation id back for cross-wiring protection. Only sent when the + // Runtime API provided it on /next. + if (!string.IsNullOrEmpty(invocationId)) + headers.Append($"{RuntimeApiHeaders.HeaderInvocationId}: {invocationId}\r\n"); headers.Append("\r\n"); var headerBytes = Encoding.ASCII.GetBytes(headers.ToString()); diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamContext.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamContext.cs index 970c43138..b7cc3d16c 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamContext.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamContext.cs @@ -30,6 +30,12 @@ internal class ResponseStreamContext /// public string AwsRequestId { get; set; } + /// + /// The unique-per-invocation id echoed back on the streaming response for cross-wiring + /// protection. Null when the Runtime API did not provide it, in which case nothing is echoed. + /// + public string InvocationId { get; set; } + /// /// Whether CreateStream() has been called for this invocation. /// diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamFactory.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamFactory.cs index 0170ddb27..534ed8fe6 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamFactory.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamFactory.cs @@ -62,7 +62,7 @@ public static ResponseStream CreateStream(byte[] prelude) // This runs concurrently — SerializeToStreamAsync will block // until the handler finishes writing or reports an error. context.SendTask = context.RuntimeApiClient.StartStreamingResponseAsync( - context.AwsRequestId, lambdaStream, context.CancellationToken); + context.AwsRequestId, context.InvocationId, lambdaStream, context.CancellationToken); return lambdaStream; } @@ -71,11 +71,13 @@ public static ResponseStream CreateStream(byte[] prelude) internal static void InitializeInvocation( string awsRequestId, bool isMultiConcurrency, - RuntimeApiClient runtimeApiClient, CancellationToken cancellationToken) + RuntimeApiClient runtimeApiClient, CancellationToken cancellationToken, + string invocationId = null) { var context = new ResponseStreamContext { AwsRequestId = awsRequestId, + InvocationId = invocationId, StreamCreated = false, Stream = null, RuntimeApiClient = runtimeApiClient, diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/IRuntimeApiClient.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/IRuntimeApiClient.cs index c43a949c9..3ac521875 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/IRuntimeApiClient.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/IRuntimeApiClient.cs @@ -65,7 +65,19 @@ public interface IRuntimeApiClient /// The optional cancellation token to use. /// A Task representing the asynchronous operation. Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, CancellationToken cancellationToken = default); - + + /// + /// Report an invocation error as an asynchronous operation, echoing the invocation id for + /// cross-wiring protection. + /// + /// The ID of the function request that caused the error. + /// The unique-per-invocation id to echo back to the Runtime API. When null, the header is not sent. + /// The exception to report. + /// The optional cancellation token to use. + /// A Task representing the asynchronous operation. + /// The invocation timed out before the error was submitted (cross-wiring protection). + Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default); + /// /// Triggers the snapshot to be taken, and then after resume, restores the lambda /// context from the Runtime API as an asynchronous operation when SnapStart is enabled. @@ -91,5 +103,17 @@ public interface IRuntimeApiClient /// The optional cancellation token to use. /// Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default); + + /// + /// Send a response to a function invocation to the Runtime API as an asynchronous operation, + /// echoing the invocation id for cross-wiring protection. + /// + /// The ID of the function request being responded to. + /// The unique-per-invocation id to echo back to the Runtime API. When null, the header is not sent. + /// The content of the response to the function invocation. + /// The optional cancellation token to use. + /// + /// The invocation timed out before the response was submitted (cross-wiring protection). + Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default); } } diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs index aeeb7ac4b..1036d6642 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs @@ -59,10 +59,12 @@ Task> RestoreErrorAsync(string lambda_Runtime_Fu /// Runtime makes this request in order to submit a response. /// Accepted /// A server side error occurred. + /// The invocation timed out before the response was submitted (cross-wiring protection). /// /// + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - System.Threading.Tasks.Task> ResponseAsync(string awsRequestId, System.IO.Stream outputStream, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.Task> ResponseAsync(string awsRequestId, System.IO.Stream outputStream, string invocationId, System.Threading.CancellationToken cancellationToken); /// /// Runtime makes this request in order to submit an error response. It can be either a function error, or a runtime error. Error will be served in response to the invoke. @@ -71,9 +73,11 @@ Task> RestoreErrorAsync(string lambda_Runtime_Fu /// /// /// + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// /// - System.Threading.Tasks.Task> ErrorWithXRayCauseAsync(string awsRequestId, string lambda_Runtime_Function_Error_Type, string errorJson, string xrayCause, System.Threading.CancellationToken cancellationToken); + /// The invocation timed out before the error was submitted (cross-wiring protection). + System.Threading.Tasks.Task> ErrorWithXRayCauseAsync(string awsRequestId, string lambda_Runtime_Function_Error_Type, string errorJson, string xrayCause, string invocationId, System.Threading.CancellationToken cancellationToken); } @@ -311,16 +315,18 @@ public async Task> RestoreErrorAsync(string lamb /// A server side error occurred. public System.Threading.Tasks.Task> ResponseAsync(string awsRequestId, System.IO.Stream outputStream) { - return ResponseAsync(awsRequestId, outputStream, System.Threading.CancellationToken.None); + return ResponseAsync(awsRequestId, outputStream, null, System.Threading.CancellationToken.None); } /// Runtime makes this request in order to submit a response. /// Accepted /// A server side error occurred. + /// The invocation timed out before the response was submitted (cross-wiring protection). /// /// + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - public async System.Threading.Tasks.Task> ResponseAsync(string awsRequestId, System.IO.Stream outputStream, System.Threading.CancellationToken cancellationToken) + public async System.Threading.Tasks.Task> ResponseAsync(string awsRequestId, System.IO.Stream outputStream, string invocationId, System.Threading.CancellationToken cancellationToken) { _logger.LogInformation("Starting InternalClient.ResponseAsync"); @@ -343,6 +349,12 @@ public async System.Threading.Tasks.Task> Respon request_.Method = new System.Net.Http.HttpMethod("POST"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + // Echo the per-invocation id back for cross-wiring protection. Only sent when the + // Runtime API provided it on /next; added to this request message (never the shared + // HttpClient) so concurrent invocations cannot echo each other's id. + if (!string.IsNullOrEmpty(invocationId)) + request_.Headers.TryAddWithoutValidation(RuntimeApiHeaders.HeaderInvocationId, invocationId); + var url_ = $"{BaseUrl.TrimEnd('/')}/runtime/invocation/{awsRequestId}/response"; request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); @@ -362,6 +374,16 @@ public async System.Threading.Tasks.Task> Respon return new SwaggerResponse((int)response_.StatusCode, headers_, new StatusResponse()); } else + if (response_.StatusCode == HttpStatusCode.Gone) + { + // 410 Gone means the invocation timed out before this response was submitted. + // The Runtime API rejects the response to prevent it being cross-wired to a + // different invocation reusing the same request id. This is not a fatal error; + // the caller logs it and moves on to the next invocation. + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new RuntimeApiInvokeTimeoutException(awsRequestId, responseData_); + } + else if (response_.StatusCode == HttpStatusCode.BadRequest) { var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -446,9 +468,10 @@ public async System.Threading.Tasks.Task> Respon /// /// /// + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// /// - public async System.Threading.Tasks.Task> ErrorWithXRayCauseAsync(string awsRequestId, string lambda_Runtime_Function_Error_Type, string errorJson, string xrayCause, System.Threading.CancellationToken cancellationToken) + public async System.Threading.Tasks.Task> ErrorWithXRayCauseAsync(string awsRequestId, string lambda_Runtime_Function_Error_Type, string errorJson, string xrayCause, string invocationId, System.Threading.CancellationToken cancellationToken) { if (awsRequestId == null) throw new System.ArgumentNullException("awsRequestId"); @@ -465,6 +488,12 @@ public async System.Threading.Tasks.Task> ErrorW if (lambda_Runtime_Function_Error_Type != null) request_.Headers.TryAddWithoutValidation("Lambda-Runtime-Function-Error-Type", ConvertToString(lambda_Runtime_Function_Error_Type, System.Globalization.CultureInfo.InvariantCulture)); + // Echo the per-invocation id back for cross-wiring protection. Only sent when the + // Runtime API provided it on /next; added to this request message (never the shared + // HttpClient) so concurrent invocations cannot echo each other's id. + if (!string.IsNullOrEmpty(invocationId)) + request_.Headers.TryAddWithoutValidation(RuntimeApiHeaders.HeaderInvocationId, invocationId); + // This is the unmodeled X-Ray header to report back the cause of errors. if (xrayCause != null && System.Text.Encoding.UTF8.GetByteCount(xrayCause) < MAX_HEADER_SIZE_BYTES) { @@ -516,6 +545,15 @@ public async System.Threading.Tasks.Task> ErrorW } } else + if (response_.StatusCode == HttpStatusCode.Gone) + { + // 410 Gone means the invocation timed out before this error was submitted. + // The Runtime API rejects it to prevent cross-wiring to a different invocation + // reusing the same request id. Not fatal; the caller logs it and moves on. + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new RuntimeApiInvokeTimeoutException(awsRequestId, responseData_); + } + else if (response_.StatusCode == HttpStatusCode.BadRequest) { var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -686,4 +724,40 @@ public RuntimeApiClientException(string message, int statusCode, string response } } + /// + /// Thrown when the Runtime API responds to a /response or /error call with HTTP 410 Gone, + /// indicating the invocation had already timed out. This is part of the cross-wiring protection: + /// the Runtime API rejects the late response so it cannot be delivered to a different invocation + /// that reused the same request id. It is not a fatal error — the invoke loop logs it and + /// continues to the next invocation. + /// + /// + /// Intentionally derives directly from rather than + /// , whose constructor dereferences the response body. + /// + public class RuntimeApiInvokeTimeoutException : System.Exception + { + /// + /// The AWS request id of the invocation that timed out. + /// + public string AwsRequestId { get; private set; } + + /// + /// The raw response body returned by the Runtime API, if any. + /// + public string Response { get; private set; } + + /// + /// Constructs a new . + /// + /// The AWS request id of the invocation that timed out. + /// The raw response body returned by the Runtime API, if any. + public RuntimeApiInvokeTimeoutException(string awsRequestId, string response) + : base($"The invocation with request id '{awsRequestId}' timed out before its response was submitted. The Runtime API rejected the response with HTTP 410 Gone to prevent cross-wiring to another invocation.") + { + AwsRequestId = awsRequestId; + Response = response; + } + } + } diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs index 39cc7d055..75c2feaba 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs @@ -125,6 +125,27 @@ public async Task GetNextInvocationAsync(CancellationToken ca /// The optional cancellation token to use. /// A Task representing the asynchronous operation. public Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, CancellationToken cancellationToken = default) + { + if (awsRequestId == null) + throw new ArgumentNullException(nameof(awsRequestId)); + + if (exception == null) + throw new ArgumentNullException(nameof(exception)); + + return ReportInvocationErrorAsync(awsRequestId, null, exception, cancellationToken); + } + + /// + /// Report an invocation error as an asynchronous operation, echoing the invocation id for + /// cross-wiring protection. + /// + /// The ID of the function request that caused the error. + /// The unique-per-invocation id to echo back to the Runtime API. When null, the header is not sent. + /// The exception to report. + /// The optional cancellation token to use. + /// A Task representing the asynchronous operation. + /// The invocation timed out before the error was submitted (cross-wiring protection). + public Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default) { if (awsRequestId == null) throw new ArgumentNullException(nameof(awsRequestId)); @@ -137,7 +158,7 @@ public Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, var exceptionInfoJson = LambdaJsonExceptionWriter.WriteJson(exceptionInfo); var exceptionInfoXRayJson = LambdaXRayExceptionWriter.WriteJson(exceptionInfo); - return _internalClient.ErrorWithXRayCauseAsync(awsRequestId, exceptionInfo.ErrorType, exceptionInfoJson, exceptionInfoXRayJson, cancellationToken); + return _internalClient.ErrorWithXRayCauseAsync(awsRequestId, exceptionInfo.ErrorType, exceptionInfoJson, exceptionInfoXRayJson, invocationId, cancellationToken); } /// @@ -175,11 +196,12 @@ public Task ReportRestoreErrorAsync(Exception exception, String errorType = null /// This Task completes when the stream is finalized (MarkCompleted or error). /// /// The ID of the function request being responded to. + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// The ResponseStream that will provide the streaming data. /// The optional cancellation token to use. /// A Task representing the in-flight HTTP POST. The returned IDisposable is the RawStreamingHttpClient that owns the TCP connection. internal virtual async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { if (awsRequestId == null) throw new ArgumentNullException(nameof(awsRequestId)); if (responseStream == null) throw new ArgumentNullException(nameof(responseStream)); @@ -187,7 +209,7 @@ internal virtual async Task StartStreamingResponseAsync( var userAgent = _httpClient.DefaultRequestHeaders.UserAgent.ToString(); var rawClient = new RawStreamingHttpClient(LambdaEnvironment.RuntimeServerHostAndPort); - await rawClient.SendStreamingResponseAsync(awsRequestId, responseStream, userAgent, cancellationToken); + await rawClient.SendStreamingResponseAsync(awsRequestId, invocationId, responseStream, userAgent, cancellationToken); return rawClient; } @@ -201,7 +223,22 @@ internal virtual async Task StartStreamingResponseAsync( /// public async Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) { - await _internalClient.ResponseAsync(awsRequestId, outputStream, cancellationToken); + await SendResponseAsync(awsRequestId, null, outputStream, cancellationToken); + } + + /// + /// Send a response to a function invocation to the Runtime API as an asynchronous operation, + /// echoing the invocation id for cross-wiring protection. + /// + /// The ID of the function request being responded to. + /// The unique-per-invocation id to echo back to the Runtime API. When null, the header is not sent. + /// The content of the response to the function invocation. + /// The optional cancellation token to use. + /// + /// The invocation timed out before the response was submitted (cross-wiring protection). + public async Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default) + { + await _internalClient.ResponseAsync(awsRequestId, outputStream, invocationId, cancellationToken); } } } diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiHeaders.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiHeaders.cs index b02dc45ab..6dd922914 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiHeaders.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiHeaders.cs @@ -39,6 +39,15 @@ public interface IRuntimeApiHeaders /// Gets the tenant id for the Lambda function. /// string TenantId { get; } + + /// + /// A unique-per-invocation identifier used for cross-wiring protection. + /// Unlike , this value is never reused across retries. + /// It is echoed back on the response and error calls so the Runtime API can reject + /// responses that belong to a timed-out invocation. May be null when the Runtime API + /// does not provide the header, in which case nothing is echoed back. + /// + string InvocationId { get; } } internal class RuntimeApiHeaders : IRuntimeApiHeaders @@ -50,6 +59,7 @@ internal class RuntimeApiHeaders : IRuntimeApiHeaders internal const string HeaderDeadlineMs = "Lambda-Runtime-Deadline-Ms"; internal const string HeaderInvokedFunctionArn = "Lambda-Runtime-Invoked-Function-Arn"; internal const string HeaderAwsTenantId = "Lambda-Runtime-Aws-Tenant-Id"; + internal const string HeaderInvocationId = "Lambda-Runtime-Invocation-Id"; public RuntimeApiHeaders(Dictionary> headers) { @@ -62,6 +72,7 @@ public RuntimeApiHeaders(Dictionary> headers) InvokedFunctionArn = GetHeaderValueOrNull(caseInsensitiveHeaders, HeaderInvokedFunctionArn); TraceId = GetHeaderValueOrNull(caseInsensitiveHeaders, HeaderTraceId); TenantId = GetHeaderValueOrNull(caseInsensitiveHeaders, HeaderAwsTenantId); + InvocationId = GetHeaderValueOrNull(caseInsensitiveHeaders, HeaderInvocationId); } public string AwsRequestId { get; private set; } @@ -71,6 +82,7 @@ public RuntimeApiHeaders(Dictionary> headers) public string CognitoIdentityJson { get; private set; } public string DeadlineMs { get; private set; } public string TenantId { get; private set; } + public string InvocationId { get; private set; } private string GetHeaderValueRequired(Dictionary> headers, string header) { diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Context/LambdaContext.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Context/LambdaContext.cs index fea4a6bd2..943a61d91 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Context/LambdaContext.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Context/LambdaContext.cs @@ -85,5 +85,12 @@ public LambdaContext(RuntimeApiHeaders runtimeApiHeaders, LambdaEnvironment lamb public ILambdaSerializer Serializer { get; internal set; } internal IRuntimeApiHeaders RuntimeApiHeaders => _runtimeApiHeaders; + + /// + /// The unique-per-invocation identifier echoed back to the Runtime API on the response + /// and error calls for cross-wiring protection. Null when the Runtime API did not provide + /// the Lambda-Runtime-Invocation-Id header, in which case nothing is echoed. + /// + internal string InvocationId => _runtimeApiHeaders.InvocationId; } } diff --git a/Libraries/test/Amazon.Lambda.Annotations.SourceGenerators.Tests/SourceGeneratorTests.cs b/Libraries/test/Amazon.Lambda.Annotations.SourceGenerators.Tests/SourceGeneratorTests.cs index d60483457..7e757f152 100644 --- a/Libraries/test/Amazon.Lambda.Annotations.SourceGenerators.Tests/SourceGeneratorTests.cs +++ b/Libraries/test/Amazon.Lambda.Annotations.SourceGenerators.Tests/SourceGeneratorTests.cs @@ -2389,18 +2389,18 @@ private static DiagnosticResult[] GetExpectedRuntimeSupportDiagnostics() return new[] { // These are here because the System.Text.Json source generator isn't included in test compilations, so these members aren't generated. - DiagnosticResult.CompilerError("CS0534").WithSpan(clientFile, 85, 30, 85, 60).WithArguments(runtimeApiContext, "System.Text.Json.Serialization.JsonSerializerContext.GeneratedSerializerOptions.get"), - DiagnosticResult.CompilerError("CS0534").WithSpan(clientFile, 85, 30, 85, 60).WithArguments(runtimeApiContext, "System.Text.Json.Serialization.JsonSerializerContext.GetTypeInfo(System.Type)"), - DiagnosticResult.CompilerError("CS7036").WithSpan(clientFile, 85, 30, 85, 60).WithArguments("options", "System.Text.Json.Serialization.JsonSerializerContext.JsonSerializerContext(System.Text.Json.JsonSerializerOptions?)"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 157, 136, 157, 143).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 172, 135, 172, 142).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 275, 131, 275, 138).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 371, 135, 371, 142).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 386, 135, 386, 142).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 401, 135, 401, 142).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 510, 136, 510, 143).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 525, 135, 525, 142).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 540, 135, 540, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0534").WithSpan(clientFile, 89, 30, 89, 60).WithArguments(runtimeApiContext, "System.Text.Json.Serialization.JsonSerializerContext.GeneratedSerializerOptions.get"), + DiagnosticResult.CompilerError("CS0534").WithSpan(clientFile, 89, 30, 89, 60).WithArguments(runtimeApiContext, "System.Text.Json.Serialization.JsonSerializerContext.GetTypeInfo(System.Type)"), + DiagnosticResult.CompilerError("CS7036").WithSpan(clientFile, 89, 30, 89, 60).WithArguments("options", "System.Text.Json.Serialization.JsonSerializerContext.JsonSerializerContext(System.Text.Json.JsonSerializerOptions?)"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 161, 136, 161, 143).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 176, 135, 176, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 279, 131, 279, 138).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 393, 135, 393, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 408, 135, 408, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 423, 135, 423, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 539, 136, 539, 143).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 563, 135, 563, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 578, 135, 578, 142).WithArguments(runtimeApiContext, "Default"), // These are here because the internalvisibleto attribute isn't included in test compilations, so these types are inaccessible. diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaBootstrapTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaBootstrapTests.cs index d2b1a1556..2038ac68c 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaBootstrapTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaBootstrapTests.cs @@ -181,6 +181,60 @@ public async Task HandlerThrowsException() Assert.True(_testFunction.HandlerWasCalled); } + [Fact] + public async Task InvocationIdIsEchoedOnResponse() + { + var headers = new Dictionary> + { + { RuntimeApiHeaders.HeaderAwsRequestId, new List { "request_id" } }, + { RuntimeApiHeaders.HeaderInvocationId, new List { "invocation_id" } } + }; + var client = new TestRuntimeApiClient(_environmentVariables, headers); + + using (var bootstrap = new LambdaBootstrap(_testFunction.BaseHandlerAsync, null, null, _environmentVariables)) + { + bootstrap.Client = client; + await bootstrap.InvokeOnceAsync(); + } + + Assert.True(client.SendResponseAsyncCalled); + Assert.Equal("invocation_id", client.LastInvocationId); + } + + [Fact] + public async Task InvokeTimeoutOnResponseDoesNotThrow() + { + _testRuntimeApiClient.ThrowInvokeTimeoutOnResponse = true; + + using (var bootstrap = new LambdaBootstrap(_testFunction.BaseHandlerAsync, null, null, _environmentVariables)) + { + bootstrap.Client = _testRuntimeApiClient; + + // A 410 Gone on the response POST must be swallowed (logged and continue), not thrown. + await bootstrap.InvokeOnceAsync(); + } + + Assert.True(_testRuntimeApiClient.SendResponseAsyncCalled); + Assert.True(_testFunction.HandlerWasCalled); + } + + [Fact] + public async Task InvokeTimeoutOnErrorDoesNotThrow() + { + _testRuntimeApiClient.ThrowInvokeTimeoutOnError = true; + + using (var bootstrap = new LambdaBootstrap(_testFunction.BaseHandlerThrowsAsync, null, null, _environmentVariables)) + { + bootstrap.Client = _testRuntimeApiClient; + + // A 410 Gone on the error POST must be swallowed (logged and continue), not thrown. + await bootstrap.InvokeOnceAsync(); + } + + Assert.True(_testRuntimeApiClient.ReportInvocationErrorAsyncExceptionCalled); + Assert.True(_testFunction.HandlerWasCalled); + } + [Fact] public async Task HandlerInputAndOutputWork() { diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaContextTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaContextTests.cs index 56cf83819..cb6c64e13 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaContextTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaContextTests.cs @@ -55,5 +55,40 @@ public void RuntimeApiHeadersAddedToContext() Assert.Equal("my-function-arn", context.InvokedFunctionArn); Assert.Equal("tenant-generated-id", context.TenantId); } + + [Fact] + public void InvocationIdIsReadWhenHeaderPresent() + { + var headers = new Dictionary> + { + ["Lambda-Runtime-Aws-Request-Id"] = new[] { "request-generated-id" }, + ["Lambda-Runtime-Invocation-Id"] = new[] { "invocation-generated-id" } + }; + + var runtimeApiHeaders = new RuntimeApiHeaders(headers); + var lambdaEnvironment = new LambdaEnvironment(_environmentVariables); + + var context = new LambdaContext(runtimeApiHeaders, lambdaEnvironment, new Helpers.LogLevelLoggerWriter(new SystemEnvironmentVariables())); + + Assert.Equal("invocation-generated-id", runtimeApiHeaders.InvocationId); + Assert.Equal("invocation-generated-id", context.InvocationId); + } + + [Fact] + public void InvocationIdIsNullWhenHeaderAbsent() + { + var headers = new Dictionary> + { + ["Lambda-Runtime-Aws-Request-Id"] = new[] { "request-generated-id" } + }; + + var runtimeApiHeaders = new RuntimeApiHeaders(headers); + var lambdaEnvironment = new LambdaEnvironment(_environmentVariables); + + var context = new LambdaContext(runtimeApiHeaders, lambdaEnvironment, new Helpers.LogLevelLoggerWriter(new SystemEnvironmentVariables())); + + Assert.Null(runtimeApiHeaders.InvocationId); + Assert.Null(context.InvocationId); + } } } diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaResponseStreamingCoreTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaResponseStreamingCoreTests.cs index 99e655fa8..0e790965e 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaResponseStreamingCoreTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaResponseStreamingCoreTests.cs @@ -429,7 +429,7 @@ public NoOpStreamingRuntimeApiClient(IEnvironmentVariables envVars) : base(envVars, new TestHelpers.NoOpInternalRuntimeApiClient()) { } internal override async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { // Provide the HTTP output stream so writes don't block await responseStream.SetHttpOutputStreamAsync(new MemoryStream(), cancellationToken); diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/ResponseStreamFactoryTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/ResponseStreamFactoryTests.cs index 52c6cfd92..d57433000 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/ResponseStreamFactoryTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/ResponseStreamFactoryTests.cs @@ -49,11 +49,14 @@ public MockStreamingRuntimeApiClient() { } + public string LastInvocationId { get; private set; } + internal override async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { StartStreamingCalled = true; LastAwsRequestId = awsRequestId; + LastInvocationId = invocationId; LastResponseStream = responseStream; await SendTaskCompletion.Task; return new NoOpDisposable(); @@ -149,6 +152,24 @@ public void CreateStream_CallsStartStreamingResponseAsync() Assert.NotNull(mock.LastResponseStream); } + /// + /// Validates that the invocation id stored on the context is passed through to + /// StartStreamingResponseAsync so the streaming response echoes it for cross-wiring protection. + /// + [Fact] + public void CreateStream_PassesInvocationIdToStartStreaming() + { + var mock = new MockStreamingRuntimeApiClient(); + ResponseStreamFactory.InitializeInvocation( + "req-inv", isMultiConcurrency: false, + mock, CancellationToken.None, "invocation_id"); + + ResponseStreamFactory.CreateStream(Array.Empty()); + + Assert.True(mock.StartStreamingCalled); + Assert.Equal("invocation_id", mock.LastInvocationId); + } + // --- GetSendTask --- /// diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs index a6ce7d892..f78e8f700 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs @@ -84,7 +84,7 @@ public async Task StartStreamingResponseAsync_IncludesStreamingResponseModeHeade var stream = new ResponseStream(Array.Empty()); var client = CreateClientWithMockHandler(stream, out var handler); - await client.StartStreamingResponseAsync("req-1", stream, CancellationToken.None); + await client.StartStreamingResponseAsync("req-1", null, stream, CancellationToken.None); Assert.NotNull(handler.CapturedRequest); Assert.True(handler.CapturedRequest.Headers.Contains(StreamingConstants.ResponseModeHeader)); @@ -107,7 +107,7 @@ public async Task StartStreamingResponseAsync_IncludesChunkedTransferEncodingHea var stream = new ResponseStream(Array.Empty()); var client = CreateClientWithMockHandler(stream, out var handler); - await client.StartStreamingResponseAsync("req-2", stream, CancellationToken.None); + await client.StartStreamingResponseAsync("req-2", null, stream, CancellationToken.None); Assert.NotNull(handler.CapturedRequest); Assert.True(handler.CapturedRequest.Headers.TransferEncodingChunked); @@ -128,7 +128,7 @@ public async Task StartStreamingResponseAsync_DeclaresTrailerHeaderUpfront() var stream = new ResponseStream(Array.Empty()); var client = CreateClientWithMockHandler(stream, out var handler); - await client.StartStreamingResponseAsync("req-3", stream, CancellationToken.None); + await client.StartStreamingResponseAsync("req-3", null, stream, CancellationToken.None); Assert.NotNull(handler.CapturedRequest); Assert.True(handler.CapturedRequest.Headers.Contains("Trailer")); @@ -186,6 +186,114 @@ public async Task SendResponseAsync_BufferedResponse_ExcludesStreamingHeaders() // --- Argument validation --- + // --- Cross-wiring: Lambda-Runtime-Invocation-Id echo + 410 handling --- + + /// + /// Mock handler that captures the request and returns a configurable status code. + /// + private class ConfigurableMockHttpMessageHandler : HttpMessageHandler + { + public HttpRequestMessage CapturedRequest { get; private set; } + private readonly HttpStatusCode _statusCode; + private readonly string _body; + + public ConfigurableMockHttpMessageHandler(HttpStatusCode statusCode, string body = null) + { + _statusCode = statusCode; + _body = body; + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + CapturedRequest = request; + var response = new HttpResponseMessage(_statusCode); + if (_body != null) + { + response.Content = new StringContent(_body); + } + return Task.FromResult(response); + } + } + + private static RuntimeApiClient CreateClientWith(ConfigurableMockHttpMessageHandler handler) + { + var httpClient = new HttpClient(handler); + var envVars = new TestEnvironmentVariables(); + envVars.SetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API", "localhost:9001"); + return new RuntimeApiClient(envVars, httpClient); + } + + [Fact] + public async Task SendResponseAsync_WithInvocationId_EchoesHeader() + { + var handler = new ConfigurableMockHttpMessageHandler(HttpStatusCode.Accepted); + var client = CreateClientWith(handler); + + var outputStream = new MemoryStream(new byte[] { 1, 2, 3 }); + await client.SendResponseAsync("req-1", "inv-uuid-abc", outputStream, CancellationToken.None); + + Assert.NotNull(handler.CapturedRequest); + Assert.True(handler.CapturedRequest.Headers.Contains("Lambda-Runtime-Invocation-Id")); + Assert.Equal("inv-uuid-abc", + handler.CapturedRequest.Headers.GetValues("Lambda-Runtime-Invocation-Id").Single()); + } + + [Fact] + public async Task SendResponseAsync_WithoutInvocationId_OmitsHeader() + { + var handler = new ConfigurableMockHttpMessageHandler(HttpStatusCode.Accepted); + var client = CreateClientWith(handler); + + var outputStream = new MemoryStream(new byte[] { 1, 2, 3 }); + await client.SendResponseAsync("req-1", null, outputStream, CancellationToken.None); + + Assert.NotNull(handler.CapturedRequest); + Assert.False(handler.CapturedRequest.Headers.Contains("Lambda-Runtime-Invocation-Id"), + "Response should not include the invocation id header when none was provided."); + } + + [Fact] + public async Task SendResponseAsync_410Gone_ThrowsRuntimeApiInvokeTimeoutException() + { + var handler = new ConfigurableMockHttpMessageHandler(HttpStatusCode.Gone, + "{\"errorMessage\":\"Invoke timeout\",\"errorType\":\"InvokeTimeout\"}"); + var client = CreateClientWith(handler); + + var outputStream = new MemoryStream(new byte[] { 1, 2, 3 }); + var ex = await Assert.ThrowsAsync( + () => client.SendResponseAsync("req-timeout", "inv-uuid-abc", outputStream, CancellationToken.None)); + + Assert.Equal("req-timeout", ex.AwsRequestId); + } + + [Fact] + public async Task ReportInvocationErrorAsync_WithInvocationId_EchoesHeader() + { + var handler = new ConfigurableMockHttpMessageHandler(HttpStatusCode.Accepted, "{}"); + var client = CreateClientWith(handler); + + await client.ReportInvocationErrorAsync("req-1", "inv-uuid-abc", new Exception("boom"), CancellationToken.None); + + Assert.NotNull(handler.CapturedRequest); + Assert.True(handler.CapturedRequest.Headers.Contains("Lambda-Runtime-Invocation-Id")); + Assert.Equal("inv-uuid-abc", + handler.CapturedRequest.Headers.GetValues("Lambda-Runtime-Invocation-Id").Single()); + } + + [Fact] + public async Task ReportInvocationErrorAsync_410Gone_ThrowsRuntimeApiInvokeTimeoutException() + { + var handler = new ConfigurableMockHttpMessageHandler(HttpStatusCode.Gone, + "{\"errorMessage\":\"Invoke timeout\",\"errorType\":\"InvokeTimeout\"}"); + var client = CreateClientWith(handler); + + var ex = await Assert.ThrowsAsync( + () => client.ReportInvocationErrorAsync("req-timeout", "inv-uuid-abc", new Exception("boom"), CancellationToken.None)); + + Assert.Equal("req-timeout", ex.AwsRequestId); + } + [Fact] public async Task StartStreamingResponseAsync_NullRequestId_ThrowsArgumentNullException() { @@ -193,7 +301,7 @@ public async Task StartStreamingResponseAsync_NullRequestId_ThrowsArgumentNullEx var client = CreateClientWithMockHandler(stream, out _); await Assert.ThrowsAsync( - () => client.StartStreamingResponseAsync(null, stream, CancellationToken.None)); + () => client.StartStreamingResponseAsync(null, null, stream, CancellationToken.None)); } [Fact] @@ -203,7 +311,7 @@ public async Task StartStreamingResponseAsync_NullResponseStream_ThrowsArgumentN var client = CreateClientWithMockHandler(stream, out _); await Assert.ThrowsAsync( - () => client.StartStreamingResponseAsync("req-5", null, CancellationToken.None)); + () => client.StartStreamingResponseAsync("req-5", null, null, CancellationToken.None)); } } } diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs index d2f448d2a..c93b49430 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs @@ -105,10 +105,13 @@ public CapturingStreamingRuntimeApiClient( }; } + public string LastInvocationId { get; private set; } + internal override async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { StartStreamingCalled = true; + LastInvocationId = invocationId; LastResponseStream = responseStream; // Use a real MemoryStream as the HTTP output stream so we capture actual bytes @@ -123,7 +126,12 @@ internal override async Task StartStreamingResponseAsync( return new NoOpDisposable(); } - public new async Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) + public new Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) + { + return SendResponseAsync(awsRequestId, null, outputStream, cancellationToken); + } + + public new async Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default) { SendResponseCalled = true; if (outputStream != null) @@ -136,6 +144,11 @@ internal override async Task StartStreamingResponseAsync( } public new Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, CancellationToken cancellationToken = default) + { + return ReportInvocationErrorAsync(awsRequestId, null, exception, cancellationToken); + } + + public new Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default) { ReportInvocationErrorCalled = true; return Task.CompletedTask; @@ -394,7 +407,7 @@ public MockMultiConcurrencyStreamingClient() : base(new TestEnvironmentVariables(), new NoOpInternalRuntimeApiClient()) { } internal override async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { // Provide the HTTP output stream so writes don't block await responseStream.SetHttpOutputStreamAsync(new MemoryStream()); diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/NoOpInternalRuntimeApiClient.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/NoOpInternalRuntimeApiClient.cs index c73a0382c..37f35bf38 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/NoOpInternalRuntimeApiClient.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/NoOpInternalRuntimeApiClient.cs @@ -40,12 +40,12 @@ public Task> ResponseAsync(string awsRequestId, => Task.FromResult(EmptyStatusResponse); public Task> ResponseAsync( - string awsRequestId, Stream outputStream, CancellationToken cancellationToken) + string awsRequestId, Stream outputStream, string invocationId, CancellationToken cancellationToken) => Task.FromResult(EmptyStatusResponse); public Task> ErrorWithXRayCauseAsync( string awsRequestId, string lambda_Runtime_Function_Error_Type, - string errorJson, string xrayCause, CancellationToken cancellationToken) + string errorJson, string xrayCause, string invocationId, CancellationToken cancellationToken) => Task.FromResult(EmptyStatusResponse); public Task> RestoreNextAsync(CancellationToken cancellationToken) diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestMultiConcurrencyRuntimeApiClient.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestMultiConcurrencyRuntimeApiClient.cs index 754bc1536..4d7f4493c 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestMultiConcurrencyRuntimeApiClient.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestMultiConcurrencyRuntimeApiClient.cs @@ -99,6 +99,11 @@ public async Task GetNextInvocationAsync(CancellationToken ca } public Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) + { + return SendResponseAsync(awsRequestId, null, outputStream, cancellationToken); + } + + public Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default) { if (ProcessInvocationEvents.TryGetValue(awsRequestId, out var data)) { @@ -141,6 +146,11 @@ public Task ReportInvocationErrorAsync(string awsRequestId, string errorType, Ca return Task.Run(() => { }); } + public Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default) + { + return Task.Run(() => { }); + } + public Task ReportRestoreErrorAsync(Exception exception, String errorType = null, CancellationToken cancellationToken = default) { return Task.Run(() => { }); diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs index b2ef549bc..155f5df1b 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs @@ -48,6 +48,12 @@ public TestRuntimeApiClient(IEnvironmentVariables environmentVariables, Dictiona public bool ReportInvocationErrorAsyncTypeCalled { get; private set; } public bool SendResponseAsyncCalled { get; private set; } + /// When set, SendResponseAsync throws RuntimeApiInvokeTimeoutException to simulate a 410 Gone. + public bool ThrowInvokeTimeoutOnResponse { get; set; } + + /// When set, ReportInvocationErrorAsync throws RuntimeApiInvokeTimeoutException to simulate a 410 Gone. + public bool ThrowInvokeTimeoutOnError { get; set; } + public string LastTraceId { get; private set; } public byte[] FunctionInput { get; set; } public Stream LastOutputStream { get; internal set; } @@ -124,8 +130,20 @@ public Task ReportInitializationErrorAsync(string errorType, CancellationToken c public Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, CancellationToken cancellationToken = default) { + return ReportInvocationErrorAsync(awsRequestId, null, exception, cancellationToken); + } + + public string LastInvocationId { get; private set; } + + public Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default) + { + LastInvocationId = invocationId; LastRecordedException = exception; ReportInvocationErrorAsyncExceptionCalled = true; + if (ThrowInvokeTimeoutOnError) + { + throw new RuntimeApiInvokeTimeoutException(awsRequestId, "{\"errorMessage\":\"Invoke timeout\",\"errorType\":\"InvokeTimeout\"}"); + } return Task.Run(() => { }); } @@ -144,6 +162,19 @@ public Task ReportRestoreErrorAsync(Exception exception, String errorType = null public Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) { + return SendResponseAsync(awsRequestId, null, outputStream, cancellationToken); + } + + public Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default) + { + LastInvocationId = invocationId; + + if (ThrowInvokeTimeoutOnResponse) + { + SendResponseAsyncCalled = true; + throw new RuntimeApiInvokeTimeoutException(awsRequestId, "{\"errorMessage\":\"Invoke timeout\",\"errorType\":\"InvokeTimeout\"}"); + } + if (outputStream != null) { // copy the stream because it gets disposed by the bootstrap diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs index 7b70c8071..fb9591c55 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs @@ -91,13 +91,28 @@ public TestStreamingRuntimeApiClient(IEnvironmentVariables environmentVariables, public new Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, CancellationToken cancellationToken = default) { + return ReportInvocationErrorAsync(awsRequestId, null, exception, cancellationToken); + } + + public string LastInvocationId { get; private set; } + + public new Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default) + { + LastInvocationId = invocationId; LastRecordedException = exception; ReportInvocationErrorAsyncExceptionCalled = true; return Task.CompletedTask; } - public new async Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) + public new Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) { + return SendResponseAsync(awsRequestId, null, outputStream, cancellationToken); + } + + public new async Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default) + { + LastInvocationId = invocationId; + if (outputStream != null) { LastOutputStream = new MemoryStream((int)outputStream.Length); @@ -109,9 +124,10 @@ public TestStreamingRuntimeApiClient(IEnvironmentVariables environmentVariables, } internal override async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { StartStreamingResponseAsyncCalled = true; + LastInvocationId = invocationId; LastStreamingResponseStream = responseStream; // Simulate the HTTP stream being available