From 0334c94bb0a04eb82664430556bb144d9326cc37 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Thu, 30 Jul 2026 17:42:46 -0400 Subject: [PATCH 1/9] Read Lambda-Runtime-Invocation-Id header from /next RuntimeApiHeaders exposes InvocationId (null when the header is absent, preserving backward compatibility). Surfaced on LambdaContext for the invoke loop to echo back on the response and error calls. --- .../Client/RuntimeApiHeaders.cs | 12 +++++++ .../Context/LambdaContext.cs | 7 ++++ .../LambdaContextTests.cs | 35 +++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiHeaders.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiHeaders.cs index b02dc45ab..6dd922914 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiHeaders.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiHeaders.cs @@ -39,6 +39,15 @@ public interface IRuntimeApiHeaders /// Gets the tenant id for the Lambda function. /// string TenantId { get; } + + /// + /// A unique-per-invocation identifier used for cross-wiring protection. + /// Unlike , this value is never reused across retries. + /// It is echoed back on the response and error calls so the Runtime API can reject + /// responses that belong to a timed-out invocation. May be null when the Runtime API + /// does not provide the header, in which case nothing is echoed back. + /// + string InvocationId { get; } } internal class RuntimeApiHeaders : IRuntimeApiHeaders @@ -50,6 +59,7 @@ internal class RuntimeApiHeaders : IRuntimeApiHeaders internal const string HeaderDeadlineMs = "Lambda-Runtime-Deadline-Ms"; internal const string HeaderInvokedFunctionArn = "Lambda-Runtime-Invoked-Function-Arn"; internal const string HeaderAwsTenantId = "Lambda-Runtime-Aws-Tenant-Id"; + internal const string HeaderInvocationId = "Lambda-Runtime-Invocation-Id"; public RuntimeApiHeaders(Dictionary> headers) { @@ -62,6 +72,7 @@ public RuntimeApiHeaders(Dictionary> headers) InvokedFunctionArn = GetHeaderValueOrNull(caseInsensitiveHeaders, HeaderInvokedFunctionArn); TraceId = GetHeaderValueOrNull(caseInsensitiveHeaders, HeaderTraceId); TenantId = GetHeaderValueOrNull(caseInsensitiveHeaders, HeaderAwsTenantId); + InvocationId = GetHeaderValueOrNull(caseInsensitiveHeaders, HeaderInvocationId); } public string AwsRequestId { get; private set; } @@ -71,6 +82,7 @@ public RuntimeApiHeaders(Dictionary> headers) public string CognitoIdentityJson { get; private set; } public string DeadlineMs { get; private set; } public string TenantId { get; private set; } + public string InvocationId { get; private set; } private string GetHeaderValueRequired(Dictionary> headers, string header) { diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Context/LambdaContext.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Context/LambdaContext.cs index fea4a6bd2..943a61d91 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Context/LambdaContext.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Context/LambdaContext.cs @@ -85,5 +85,12 @@ public LambdaContext(RuntimeApiHeaders runtimeApiHeaders, LambdaEnvironment lamb public ILambdaSerializer Serializer { get; internal set; } internal IRuntimeApiHeaders RuntimeApiHeaders => _runtimeApiHeaders; + + /// + /// The unique-per-invocation identifier echoed back to the Runtime API on the response + /// and error calls for cross-wiring protection. Null when the Runtime API did not provide + /// the Lambda-Runtime-Invocation-Id header, in which case nothing is echoed. + /// + internal string InvocationId => _runtimeApiHeaders.InvocationId; } } diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaContextTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaContextTests.cs index 56cf83819..cb6c64e13 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaContextTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaContextTests.cs @@ -55,5 +55,40 @@ public void RuntimeApiHeadersAddedToContext() Assert.Equal("my-function-arn", context.InvokedFunctionArn); Assert.Equal("tenant-generated-id", context.TenantId); } + + [Fact] + public void InvocationIdIsReadWhenHeaderPresent() + { + var headers = new Dictionary> + { + ["Lambda-Runtime-Aws-Request-Id"] = new[] { "request-generated-id" }, + ["Lambda-Runtime-Invocation-Id"] = new[] { "invocation-generated-id" } + }; + + var runtimeApiHeaders = new RuntimeApiHeaders(headers); + var lambdaEnvironment = new LambdaEnvironment(_environmentVariables); + + var context = new LambdaContext(runtimeApiHeaders, lambdaEnvironment, new Helpers.LogLevelLoggerWriter(new SystemEnvironmentVariables())); + + Assert.Equal("invocation-generated-id", runtimeApiHeaders.InvocationId); + Assert.Equal("invocation-generated-id", context.InvocationId); + } + + [Fact] + public void InvocationIdIsNullWhenHeaderAbsent() + { + var headers = new Dictionary> + { + ["Lambda-Runtime-Aws-Request-Id"] = new[] { "request-generated-id" } + }; + + var runtimeApiHeaders = new RuntimeApiHeaders(headers); + var lambdaEnvironment = new LambdaEnvironment(_environmentVariables); + + var context = new LambdaContext(runtimeApiHeaders, lambdaEnvironment, new Helpers.LogLevelLoggerWriter(new SystemEnvironmentVariables())); + + Assert.Null(runtimeApiHeaders.InvocationId); + Assert.Null(context.InvocationId); + } } } From 67bd78b5d0623563691b624ed70541e66a38dcee Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Thu, 30 Jul 2026 17:47:57 -0400 Subject: [PATCH 2/9] Echo invocation id and handle invoke-timeout (410) in the API client InternalRuntimeApiClient adds the Lambda-Runtime-Invocation-Id request header on /response and /error when present, added to the per-request message so concurrent invocations cannot echo each other's id. HTTP 410 Gone (Runtime.InvokeTimeout) maps to a new standalone RuntimeApiInvokeTimeoutException instead of the generic unexpected-status throw. Adds invocationId overloads to RuntimeApiClient and the public IRuntimeApiClient interface (following the SnapStart precedent of extending the runtime API client surface); existing overloads delegate with a null invocation id. --- .../Client/IRuntimeApiClient.cs | 26 ++++- .../Client/InternalClientAdapted.cs | 83 +++++++++++++- .../Client/RuntimeApiClient.cs | 42 ++++++- .../RuntimeApiClientTests.cs | 108 ++++++++++++++++++ .../NoOpInternalRuntimeApiClient.cs | 4 +- .../TestMultiConcurrencyRuntimeApiClient.cs | 10 ++ .../TestHelpers/TestRuntimeApiClient.cs | 15 +++ 7 files changed, 278 insertions(+), 10 deletions(-) diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/IRuntimeApiClient.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/IRuntimeApiClient.cs index c43a949c9..3ac521875 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/IRuntimeApiClient.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/IRuntimeApiClient.cs @@ -65,7 +65,19 @@ public interface IRuntimeApiClient /// The optional cancellation token to use. /// A Task representing the asynchronous operation. Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, CancellationToken cancellationToken = default); - + + /// + /// Report an invocation error as an asynchronous operation, echoing the invocation id for + /// cross-wiring protection. + /// + /// The ID of the function request that caused the error. + /// The unique-per-invocation id to echo back to the Runtime API. When null, the header is not sent. + /// The exception to report. + /// The optional cancellation token to use. + /// A Task representing the asynchronous operation. + /// The invocation timed out before the error was submitted (cross-wiring protection). + Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default); + /// /// Triggers the snapshot to be taken, and then after resume, restores the lambda /// context from the Runtime API as an asynchronous operation when SnapStart is enabled. @@ -91,5 +103,17 @@ public interface IRuntimeApiClient /// The optional cancellation token to use. /// Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default); + + /// + /// Send a response to a function invocation to the Runtime API as an asynchronous operation, + /// echoing the invocation id for cross-wiring protection. + /// + /// The ID of the function request being responded to. + /// The unique-per-invocation id to echo back to the Runtime API. When null, the header is not sent. + /// The content of the response to the function invocation. + /// The optional cancellation token to use. + /// + /// The invocation timed out before the response was submitted (cross-wiring protection). + Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default); } } diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs index aeeb7ac4b..1506480ad 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs @@ -59,10 +59,12 @@ Task> RestoreErrorAsync(string lambda_Runtime_Fu /// Runtime makes this request in order to submit a response. /// Accepted /// A server side error occurred. + /// The invocation timed out before the response was submitted (cross-wiring protection). /// /// + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - System.Threading.Tasks.Task> ResponseAsync(string awsRequestId, System.IO.Stream outputStream, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.Task> ResponseAsync(string awsRequestId, System.IO.Stream outputStream, string invocationId, System.Threading.CancellationToken cancellationToken); /// /// Runtime makes this request in order to submit an error response. It can be either a function error, or a runtime error. Error will be served in response to the invoke. @@ -71,9 +73,11 @@ Task> RestoreErrorAsync(string lambda_Runtime_Fu /// /// /// + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// /// - System.Threading.Tasks.Task> ErrorWithXRayCauseAsync(string awsRequestId, string lambda_Runtime_Function_Error_Type, string errorJson, string xrayCause, System.Threading.CancellationToken cancellationToken); + /// The invocation timed out before the error was submitted (cross-wiring protection). + System.Threading.Tasks.Task> ErrorWithXRayCauseAsync(string awsRequestId, string lambda_Runtime_Function_Error_Type, string errorJson, string xrayCause, string invocationId, System.Threading.CancellationToken cancellationToken); } @@ -311,16 +315,18 @@ public async Task> RestoreErrorAsync(string lamb /// A server side error occurred. public System.Threading.Tasks.Task> ResponseAsync(string awsRequestId, System.IO.Stream outputStream) { - return ResponseAsync(awsRequestId, outputStream, System.Threading.CancellationToken.None); + return ResponseAsync(awsRequestId, outputStream, null, System.Threading.CancellationToken.None); } /// Runtime makes this request in order to submit a response. /// Accepted /// A server side error occurred. + /// The invocation timed out before the response was submitted (cross-wiring protection). /// /// + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - public async System.Threading.Tasks.Task> ResponseAsync(string awsRequestId, System.IO.Stream outputStream, System.Threading.CancellationToken cancellationToken) + public async System.Threading.Tasks.Task> ResponseAsync(string awsRequestId, System.IO.Stream outputStream, string invocationId, System.Threading.CancellationToken cancellationToken) { _logger.LogInformation("Starting InternalClient.ResponseAsync"); @@ -343,6 +349,12 @@ public async System.Threading.Tasks.Task> Respon request_.Method = new System.Net.Http.HttpMethod("POST"); request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + // Echo the per-invocation id back for cross-wiring protection. Only sent when the + // Runtime API provided it on /next; added to this request message (never the shared + // HttpClient) so concurrent invocations cannot echo each other's id. + if (!string.IsNullOrEmpty(invocationId)) + request_.Headers.TryAddWithoutValidation(RuntimeApiHeaders.HeaderInvocationId, invocationId); + var url_ = $"{BaseUrl.TrimEnd('/')}/runtime/invocation/{awsRequestId}/response"; request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); @@ -362,6 +374,16 @@ public async System.Threading.Tasks.Task> Respon return new SwaggerResponse((int)response_.StatusCode, headers_, new StatusResponse()); } else + if (response_.StatusCode == HttpStatusCode.Gone) + { + // 410 Gone means the invocation timed out before this response was submitted. + // The Runtime API rejects the response to prevent it being cross-wired to a + // different invocation reusing the same request id. This is not a fatal error; + // the caller logs it and moves on to the next invocation. + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new RuntimeApiInvokeTimeoutException(awsRequestId, responseData_); + } + else if (response_.StatusCode == HttpStatusCode.BadRequest) { var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -448,7 +470,7 @@ public async System.Threading.Tasks.Task> Respon /// /// /// - public async System.Threading.Tasks.Task> ErrorWithXRayCauseAsync(string awsRequestId, string lambda_Runtime_Function_Error_Type, string errorJson, string xrayCause, System.Threading.CancellationToken cancellationToken) + public async System.Threading.Tasks.Task> ErrorWithXRayCauseAsync(string awsRequestId, string lambda_Runtime_Function_Error_Type, string errorJson, string xrayCause, string invocationId, System.Threading.CancellationToken cancellationToken) { if (awsRequestId == null) throw new System.ArgumentNullException("awsRequestId"); @@ -465,6 +487,12 @@ public async System.Threading.Tasks.Task> ErrorW if (lambda_Runtime_Function_Error_Type != null) request_.Headers.TryAddWithoutValidation("Lambda-Runtime-Function-Error-Type", ConvertToString(lambda_Runtime_Function_Error_Type, System.Globalization.CultureInfo.InvariantCulture)); + // Echo the per-invocation id back for cross-wiring protection. Only sent when the + // Runtime API provided it on /next; added to this request message (never the shared + // HttpClient) so concurrent invocations cannot echo each other's id. + if (!string.IsNullOrEmpty(invocationId)) + request_.Headers.TryAddWithoutValidation(RuntimeApiHeaders.HeaderInvocationId, invocationId); + // This is the unmodeled X-Ray header to report back the cause of errors. if (xrayCause != null && System.Text.Encoding.UTF8.GetByteCount(xrayCause) < MAX_HEADER_SIZE_BYTES) { @@ -516,6 +544,15 @@ public async System.Threading.Tasks.Task> ErrorW } } else + if (response_.StatusCode == HttpStatusCode.Gone) + { + // 410 Gone means the invocation timed out before this error was submitted. + // The Runtime API rejects it to prevent cross-wiring to a different invocation + // reusing the same request id. Not fatal; the caller logs it and moves on. + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new RuntimeApiInvokeTimeoutException(awsRequestId, responseData_); + } + else if (response_.StatusCode == HttpStatusCode.BadRequest) { var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -686,4 +723,40 @@ public RuntimeApiClientException(string message, int statusCode, string response } } + /// + /// Thrown when the Runtime API responds to a /response or /error call with HTTP 410 Gone, + /// indicating the invocation had already timed out. This is part of the cross-wiring protection: + /// the Runtime API rejects the late response so it cannot be delivered to a different invocation + /// that reused the same request id. It is not a fatal error — the invoke loop logs it and + /// continues to the next invocation. + /// + /// + /// Intentionally derives directly from rather than + /// , whose constructor dereferences the response body. + /// + public class RuntimeApiInvokeTimeoutException : System.Exception + { + /// + /// The AWS request id of the invocation that timed out. + /// + public string AwsRequestId { get; private set; } + + /// + /// The raw response body returned by the Runtime API, if any. + /// + public string Response { get; private set; } + + /// + /// Constructs a new . + /// + /// The AWS request id of the invocation that timed out. + /// The raw response body returned by the Runtime API, if any. + public RuntimeApiInvokeTimeoutException(string awsRequestId, string response) + : base($"The invocation with request id '{awsRequestId}' timed out before its response was submitted. The Runtime API rejected the response with HTTP 410 Gone to prevent cross-wiring to another invocation.") + { + AwsRequestId = awsRequestId; + Response = response; + } + } + } diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs index 39cc7d055..b207d2bc5 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs @@ -134,10 +134,33 @@ public Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, var exceptionInfo = ExceptionInfo.GetExceptionInfo(exception); + return ReportInvocationErrorAsync(awsRequestId, null, exception, cancellationToken); + } + + /// + /// Report an invocation error as an asynchronous operation, echoing the invocation id for + /// cross-wiring protection. + /// + /// The ID of the function request that caused the error. + /// The unique-per-invocation id to echo back to the Runtime API. When null, the header is not sent. + /// The exception to report. + /// The optional cancellation token to use. + /// A Task representing the asynchronous operation. + /// The invocation timed out before the error was submitted (cross-wiring protection). + public Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default) + { + if (awsRequestId == null) + throw new ArgumentNullException(nameof(awsRequestId)); + + if (exception == null) + throw new ArgumentNullException(nameof(exception)); + + var exceptionInfo = ExceptionInfo.GetExceptionInfo(exception); + var exceptionInfoJson = LambdaJsonExceptionWriter.WriteJson(exceptionInfo); var exceptionInfoXRayJson = LambdaXRayExceptionWriter.WriteJson(exceptionInfo); - return _internalClient.ErrorWithXRayCauseAsync(awsRequestId, exceptionInfo.ErrorType, exceptionInfoJson, exceptionInfoXRayJson, cancellationToken); + return _internalClient.ErrorWithXRayCauseAsync(awsRequestId, exceptionInfo.ErrorType, exceptionInfoJson, exceptionInfoXRayJson, invocationId, cancellationToken); } /// @@ -201,7 +224,22 @@ internal virtual async Task StartStreamingResponseAsync( /// public async Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) { - await _internalClient.ResponseAsync(awsRequestId, outputStream, cancellationToken); + await SendResponseAsync(awsRequestId, null, outputStream, cancellationToken); + } + + /// + /// Send a response to a function invocation to the Runtime API as an asynchronous operation, + /// echoing the invocation id for cross-wiring protection. + /// + /// The ID of the function request being responded to. + /// The unique-per-invocation id to echo back to the Runtime API. When null, the header is not sent. + /// The content of the response to the function invocation. + /// The optional cancellation token to use. + /// + /// The invocation timed out before the response was submitted (cross-wiring protection). + public async Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default) + { + await _internalClient.ResponseAsync(awsRequestId, outputStream, invocationId, cancellationToken); } } } diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs index a6ce7d892..f94864389 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs @@ -186,6 +186,114 @@ public async Task SendResponseAsync_BufferedResponse_ExcludesStreamingHeaders() // --- Argument validation --- + // --- Cross-wiring: Lambda-Runtime-Invocation-Id echo + 410 handling --- + + /// + /// Mock handler that captures the request and returns a configurable status code. + /// + private class ConfigurableMockHttpMessageHandler : HttpMessageHandler + { + public HttpRequestMessage CapturedRequest { get; private set; } + private readonly HttpStatusCode _statusCode; + private readonly string _body; + + public ConfigurableMockHttpMessageHandler(HttpStatusCode statusCode, string body = null) + { + _statusCode = statusCode; + _body = body; + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + CapturedRequest = request; + var response = new HttpResponseMessage(_statusCode); + if (_body != null) + { + response.Content = new StringContent(_body); + } + return Task.FromResult(response); + } + } + + private static RuntimeApiClient CreateClientWith(ConfigurableMockHttpMessageHandler handler) + { + var httpClient = new HttpClient(handler); + var envVars = new TestEnvironmentVariables(); + envVars.SetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API", "localhost:9001"); + return new RuntimeApiClient(envVars, httpClient); + } + + [Fact] + public async Task SendResponseAsync_WithInvocationId_EchoesHeader() + { + var handler = new ConfigurableMockHttpMessageHandler(HttpStatusCode.Accepted); + var client = CreateClientWith(handler); + + var outputStream = new MemoryStream(new byte[] { 1, 2, 3 }); + await client.SendResponseAsync("req-1", "inv-uuid-abc", outputStream, CancellationToken.None); + + Assert.NotNull(handler.CapturedRequest); + Assert.True(handler.CapturedRequest.Headers.Contains("Lambda-Runtime-Invocation-Id")); + Assert.Equal("inv-uuid-abc", + handler.CapturedRequest.Headers.GetValues("Lambda-Runtime-Invocation-Id").Single()); + } + + [Fact] + public async Task SendResponseAsync_WithoutInvocationId_OmitsHeader() + { + var handler = new ConfigurableMockHttpMessageHandler(HttpStatusCode.Accepted); + var client = CreateClientWith(handler); + + var outputStream = new MemoryStream(new byte[] { 1, 2, 3 }); + await client.SendResponseAsync("req-1", null, outputStream, CancellationToken.None); + + Assert.NotNull(handler.CapturedRequest); + Assert.False(handler.CapturedRequest.Headers.Contains("Lambda-Runtime-Invocation-Id"), + "Response should not include the invocation id header when none was provided."); + } + + [Fact] + public async Task SendResponseAsync_410Gone_ThrowsRuntimeApiInvokeTimeoutException() + { + var handler = new ConfigurableMockHttpMessageHandler(HttpStatusCode.Gone, + "{\"errorMessage\":\"Invoke timeout\",\"errorType\":\"InvokeTimeout\"}"); + var client = CreateClientWith(handler); + + var outputStream = new MemoryStream(new byte[] { 1, 2, 3 }); + var ex = await Assert.ThrowsAsync( + () => client.SendResponseAsync("req-timeout", "inv-uuid-abc", outputStream, CancellationToken.None)); + + Assert.Equal("req-timeout", ex.AwsRequestId); + } + + [Fact] + public async Task ReportInvocationErrorAsync_WithInvocationId_EchoesHeader() + { + var handler = new ConfigurableMockHttpMessageHandler(HttpStatusCode.Accepted, "{}"); + var client = CreateClientWith(handler); + + await client.ReportInvocationErrorAsync("req-1", "inv-uuid-abc", new Exception("boom"), CancellationToken.None); + + Assert.NotNull(handler.CapturedRequest); + Assert.True(handler.CapturedRequest.Headers.Contains("Lambda-Runtime-Invocation-Id")); + Assert.Equal("inv-uuid-abc", + handler.CapturedRequest.Headers.GetValues("Lambda-Runtime-Invocation-Id").Single()); + } + + [Fact] + public async Task ReportInvocationErrorAsync_410Gone_ThrowsRuntimeApiInvokeTimeoutException() + { + var handler = new ConfigurableMockHttpMessageHandler(HttpStatusCode.Gone, + "{\"errorMessage\":\"Invoke timeout\",\"errorType\":\"InvokeTimeout\"}"); + var client = CreateClientWith(handler); + + var ex = await Assert.ThrowsAsync( + () => client.ReportInvocationErrorAsync("req-timeout", "inv-uuid-abc", new Exception("boom"), CancellationToken.None)); + + Assert.Equal("req-timeout", ex.AwsRequestId); + } + [Fact] public async Task StartStreamingResponseAsync_NullRequestId_ThrowsArgumentNullException() { diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/NoOpInternalRuntimeApiClient.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/NoOpInternalRuntimeApiClient.cs index c73a0382c..37f35bf38 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/NoOpInternalRuntimeApiClient.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/NoOpInternalRuntimeApiClient.cs @@ -40,12 +40,12 @@ public Task> ResponseAsync(string awsRequestId, => Task.FromResult(EmptyStatusResponse); public Task> ResponseAsync( - string awsRequestId, Stream outputStream, CancellationToken cancellationToken) + string awsRequestId, Stream outputStream, string invocationId, CancellationToken cancellationToken) => Task.FromResult(EmptyStatusResponse); public Task> ErrorWithXRayCauseAsync( string awsRequestId, string lambda_Runtime_Function_Error_Type, - string errorJson, string xrayCause, CancellationToken cancellationToken) + string errorJson, string xrayCause, string invocationId, CancellationToken cancellationToken) => Task.FromResult(EmptyStatusResponse); public Task> RestoreNextAsync(CancellationToken cancellationToken) diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestMultiConcurrencyRuntimeApiClient.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestMultiConcurrencyRuntimeApiClient.cs index 754bc1536..4d7f4493c 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestMultiConcurrencyRuntimeApiClient.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestMultiConcurrencyRuntimeApiClient.cs @@ -99,6 +99,11 @@ public async Task GetNextInvocationAsync(CancellationToken ca } public Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) + { + return SendResponseAsync(awsRequestId, null, outputStream, cancellationToken); + } + + public Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default) { if (ProcessInvocationEvents.TryGetValue(awsRequestId, out var data)) { @@ -141,6 +146,11 @@ public Task ReportInvocationErrorAsync(string awsRequestId, string errorType, Ca return Task.Run(() => { }); } + public Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default) + { + return Task.Run(() => { }); + } + public Task ReportRestoreErrorAsync(Exception exception, String errorType = null, CancellationToken cancellationToken = default) { return Task.Run(() => { }); diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs index b2ef549bc..b3639d083 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs @@ -124,6 +124,14 @@ public Task ReportInitializationErrorAsync(string errorType, CancellationToken c public Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, CancellationToken cancellationToken = default) { + return ReportInvocationErrorAsync(awsRequestId, null, exception, cancellationToken); + } + + public string LastInvocationId { get; private set; } + + public Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default) + { + LastInvocationId = invocationId; LastRecordedException = exception; ReportInvocationErrorAsyncExceptionCalled = true; return Task.Run(() => { }); @@ -144,6 +152,13 @@ public Task ReportRestoreErrorAsync(Exception exception, String errorType = null public Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) { + return SendResponseAsync(awsRequestId, null, outputStream, cancellationToken); + } + + public Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default) + { + LastInvocationId = invocationId; + if (outputStream != null) { // copy the stream because it gets disposed by the bootstrap From 619851f6dcaa754f561931405e7fae89acceabbe Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Thu, 30 Jul 2026 17:56:41 -0400 Subject: [PATCH 3/9] Log and continue on invoke-timeout (410) instead of crashing LambdaBootstrap threads the invocation id into the response and error POSTs and catches RuntimeApiInvokeTimeoutException before the general handler, logging via the internal logger and looping back to /next in both on-demand and multi-concurrency modes. The catch encloses both the response and error POST paths so a 410 on either does not crash the on-demand process. Updates the streaming test doubles to shadow the new invocationId overloads so LambdaBootstrap dispatches to their tracking logic. --- .../Bootstrap/LambdaBootstrap.cs | 19 ++++++- .../LambdaBootstrapTests.cs | 54 +++++++++++++++++++ .../StreamingE2EWithMoq.cs | 12 ++++- .../TestHelpers/TestRuntimeApiClient.cs | 16 ++++++ .../TestStreamingRuntimeApiClient.cs | 17 +++++- 5 files changed, 114 insertions(+), 4 deletions(-) diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs index 9b8186dda..627c1a6ab 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs @@ -380,11 +380,17 @@ internal async Task InvokeOnceAsync(CancellationToken cancellationToken = defaul Func processingFunc = async () => { + // Per-invocation id echoed back on /response and /error for cross-wiring protection. + // Null when the Runtime API did not send the Lambda-Runtime-Invocation-Id header, in + // which case nothing is echoed. Captured as a local so concurrent invocations in + // multi-concurrency mode each carry their own id. + string invocationId = null; if (invocation.LambdaContext is LambdaContext impl) { Client.ConsoleLogger.SetRuntimeHeaders(impl.RuntimeApiHeaders); SetInvocationTraceId(impl.RuntimeApiHeaders.TraceId); SetSerializerOnContext(impl); + invocationId = impl.InvocationId; } // Initialize ResponseStreamFactory — includes RuntimeApiClient reference @@ -420,7 +426,7 @@ internal async Task InvokeOnceAsync(CancellationToken cancellationToken = defaul } else { - await Client.ReportInvocationErrorAsync(invocation.LambdaContext.AwsRequestId, exception, cancellationToken); + await Client.ReportInvocationErrorAsync(invocation.LambdaContext.AwsRequestId, invocationId, exception, cancellationToken); } } finally @@ -450,7 +456,7 @@ internal async Task InvokeOnceAsync(CancellationToken cancellationToken = defaul _logger.LogInformation("Starting sending response"); try { - await Client.SendResponseAsync(invocation.LambdaContext.AwsRequestId, response?.OutputStream, cancellationToken); + await Client.SendResponseAsync(invocation.LambdaContext.AwsRequestId, invocationId, response?.OutputStream, cancellationToken); } finally { @@ -465,6 +471,15 @@ internal async Task InvokeOnceAsync(CancellationToken cancellationToken = defaul _logger.LogInformation("Finished InvokeOnceAsync"); } + catch (RuntimeApiInvokeTimeoutException timeout) + { + // The Runtime API rejected the response or error with HTTP 410 Gone because the + // invocation had already timed out (cross-wiring protection). This is expected and + // not fatal — the response would have been discarded anyway. Log via the internal + // logger (not the customer log) and loop back to /next in both on-demand and + // multi-concurrency modes rather than crashing the process. + _logger.LogInformation($"Invocation {timeout.AwsRequestId} timed out before its response was submitted; the Runtime API rejected it. Continuing to the next invocation."); + } catch(Exception ex) { // Only capture and continue for multi concurrency because we do not want to change diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaBootstrapTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaBootstrapTests.cs index d2b1a1556..2038ac68c 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaBootstrapTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaBootstrapTests.cs @@ -181,6 +181,60 @@ public async Task HandlerThrowsException() Assert.True(_testFunction.HandlerWasCalled); } + [Fact] + public async Task InvocationIdIsEchoedOnResponse() + { + var headers = new Dictionary> + { + { RuntimeApiHeaders.HeaderAwsRequestId, new List { "request_id" } }, + { RuntimeApiHeaders.HeaderInvocationId, new List { "invocation_id" } } + }; + var client = new TestRuntimeApiClient(_environmentVariables, headers); + + using (var bootstrap = new LambdaBootstrap(_testFunction.BaseHandlerAsync, null, null, _environmentVariables)) + { + bootstrap.Client = client; + await bootstrap.InvokeOnceAsync(); + } + + Assert.True(client.SendResponseAsyncCalled); + Assert.Equal("invocation_id", client.LastInvocationId); + } + + [Fact] + public async Task InvokeTimeoutOnResponseDoesNotThrow() + { + _testRuntimeApiClient.ThrowInvokeTimeoutOnResponse = true; + + using (var bootstrap = new LambdaBootstrap(_testFunction.BaseHandlerAsync, null, null, _environmentVariables)) + { + bootstrap.Client = _testRuntimeApiClient; + + // A 410 Gone on the response POST must be swallowed (logged and continue), not thrown. + await bootstrap.InvokeOnceAsync(); + } + + Assert.True(_testRuntimeApiClient.SendResponseAsyncCalled); + Assert.True(_testFunction.HandlerWasCalled); + } + + [Fact] + public async Task InvokeTimeoutOnErrorDoesNotThrow() + { + _testRuntimeApiClient.ThrowInvokeTimeoutOnError = true; + + using (var bootstrap = new LambdaBootstrap(_testFunction.BaseHandlerThrowsAsync, null, null, _environmentVariables)) + { + bootstrap.Client = _testRuntimeApiClient; + + // A 410 Gone on the error POST must be swallowed (logged and continue), not thrown. + await bootstrap.InvokeOnceAsync(); + } + + Assert.True(_testRuntimeApiClient.ReportInvocationErrorAsyncExceptionCalled); + Assert.True(_testFunction.HandlerWasCalled); + } + [Fact] public async Task HandlerInputAndOutputWork() { diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs index d2f448d2a..4409a6f68 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs @@ -123,7 +123,12 @@ internal override async Task StartStreamingResponseAsync( return new NoOpDisposable(); } - public new async Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) + public new Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) + { + return SendResponseAsync(awsRequestId, null, outputStream, cancellationToken); + } + + public new async Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default) { SendResponseCalled = true; if (outputStream != null) @@ -136,6 +141,11 @@ internal override async Task StartStreamingResponseAsync( } public new Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, CancellationToken cancellationToken = default) + { + return ReportInvocationErrorAsync(awsRequestId, null, exception, cancellationToken); + } + + public new Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default) { ReportInvocationErrorCalled = true; return Task.CompletedTask; diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs index b3639d083..155f5df1b 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestRuntimeApiClient.cs @@ -48,6 +48,12 @@ public TestRuntimeApiClient(IEnvironmentVariables environmentVariables, Dictiona public bool ReportInvocationErrorAsyncTypeCalled { get; private set; } public bool SendResponseAsyncCalled { get; private set; } + /// When set, SendResponseAsync throws RuntimeApiInvokeTimeoutException to simulate a 410 Gone. + public bool ThrowInvokeTimeoutOnResponse { get; set; } + + /// When set, ReportInvocationErrorAsync throws RuntimeApiInvokeTimeoutException to simulate a 410 Gone. + public bool ThrowInvokeTimeoutOnError { get; set; } + public string LastTraceId { get; private set; } public byte[] FunctionInput { get; set; } public Stream LastOutputStream { get; internal set; } @@ -134,6 +140,10 @@ public Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, LastInvocationId = invocationId; LastRecordedException = exception; ReportInvocationErrorAsyncExceptionCalled = true; + if (ThrowInvokeTimeoutOnError) + { + throw new RuntimeApiInvokeTimeoutException(awsRequestId, "{\"errorMessage\":\"Invoke timeout\",\"errorType\":\"InvokeTimeout\"}"); + } return Task.Run(() => { }); } @@ -159,6 +169,12 @@ public Task SendResponseAsync(string awsRequestId, string invocationId, Stream o { LastInvocationId = invocationId; + if (ThrowInvokeTimeoutOnResponse) + { + SendResponseAsyncCalled = true; + throw new RuntimeApiInvokeTimeoutException(awsRequestId, "{\"errorMessage\":\"Invoke timeout\",\"errorType\":\"InvokeTimeout\"}"); + } + if (outputStream != null) { // copy the stream because it gets disposed by the bootstrap diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs index 7b70c8071..42c14f352 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs @@ -91,13 +91,28 @@ public TestStreamingRuntimeApiClient(IEnvironmentVariables environmentVariables, public new Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, CancellationToken cancellationToken = default) { + return ReportInvocationErrorAsync(awsRequestId, null, exception, cancellationToken); + } + + public string LastInvocationId { get; private set; } + + public new Task ReportInvocationErrorAsync(string awsRequestId, string invocationId, Exception exception, CancellationToken cancellationToken = default) + { + LastInvocationId = invocationId; LastRecordedException = exception; ReportInvocationErrorAsyncExceptionCalled = true; return Task.CompletedTask; } - public new async Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) + public new Task SendResponseAsync(string awsRequestId, Stream outputStream, CancellationToken cancellationToken = default) { + return SendResponseAsync(awsRequestId, null, outputStream, cancellationToken); + } + + public new async Task SendResponseAsync(string awsRequestId, string invocationId, Stream outputStream, CancellationToken cancellationToken = default) + { + LastInvocationId = invocationId; + if (outputStream != null) { LastOutputStream = new MemoryStream((int)outputStream.Length); From addbda63e7550e4eaf5d7b4928183454546a53e7 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Thu, 30 Jul 2026 18:00:47 -0400 Subject: [PATCH 4/9] Echo invocation id on the streaming response path Threads the invocation id through ResponseStreamFactory and ResponseStreamContext into RawStreamingHttpClient, which writes the Lambda-Runtime-Invocation-Id header into the raw HTTP header block before the terminating blank line and only when present. --- .../Bootstrap/LambdaBootstrap.cs | 3 ++- .../RawStreamingHttpClient.cs | 9 ++++++++ .../ResponseStreamContext.cs | 6 +++++ .../ResponseStreamFactory.cs | 6 +++-- .../Client/RuntimeApiClient.cs | 5 ++-- .../LambdaResponseStreamingCoreTests.cs | 2 +- .../ResponseStreamFactoryTests.cs | 23 ++++++++++++++++++- .../RuntimeApiClientTests.cs | 10 ++++---- .../StreamingE2EWithMoq.cs | 7 ++++-- .../TestStreamingRuntimeApiClient.cs | 3 ++- 10 files changed, 59 insertions(+), 15 deletions(-) diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs index 627c1a6ab..87445c768 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/LambdaBootstrap.cs @@ -401,7 +401,8 @@ internal async Task InvokeOnceAsync(CancellationToken cancellationToken = defaul invocation.LambdaContext.AwsRequestId, isMultiConcurrency, runtimeApiClient, - cancellationToken); + cancellationToken, + invocationId); } try diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs index 11f2f1a49..2772c4235 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs @@ -23,6 +23,8 @@ using Amazon.Lambda.RuntimeSupport.Helpers; namespace Amazon.Lambda.RuntimeSupport.Client.ResponseStreaming +// RuntimeApiHeaders lives in the Amazon.Lambda.RuntimeSupport namespace, which encloses this one, +// so its HeaderInvocationId constant is in scope without an extra using. { /// /// A raw HTTP/1.1 client for sending streaming responses to the Lambda Runtime API @@ -62,11 +64,13 @@ public RawStreamingHttpClient(string hostAndPort) /// for error reporting. /// /// The Lambda request ID. + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// The response stream that provides data and error state. /// The User-Agent header value. /// Cancellation token. public async Task SendStreamingResponseAsync( string awsRequestId, + string invocationId, ResponseStream responseStream, string userAgent, CancellationToken cancellationToken = default) @@ -86,6 +90,11 @@ public async Task SendStreamingResponseAsync( headers.Append($"{StreamingConstants.ResponseModeHeader}: {StreamingConstants.StreamingResponseMode}\r\n"); headers.Append("Transfer-Encoding: chunked\r\n"); headers.Append($"Trailer: {StreamingConstants.ErrorTypeTrailer}, {StreamingConstants.ErrorBodyTrailer}\r\n"); + // Echo the per-invocation id back for cross-wiring protection. Only sent when the + // Runtime API provided it on /next. Must be added before the terminating blank line + // so it stays part of the header block rather than the body. + if (!string.IsNullOrEmpty(invocationId)) + headers.Append($"{RuntimeApiHeaders.HeaderInvocationId}: {invocationId}\r\n"); headers.Append("\r\n"); var headerBytes = Encoding.ASCII.GetBytes(headers.ToString()); diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamContext.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamContext.cs index 970c43138..b7cc3d16c 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamContext.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamContext.cs @@ -30,6 +30,12 @@ internal class ResponseStreamContext /// public string AwsRequestId { get; set; } + /// + /// The unique-per-invocation id echoed back on the streaming response for cross-wiring + /// protection. Null when the Runtime API did not provide it, in which case nothing is echoed. + /// + public string InvocationId { get; set; } + /// /// Whether CreateStream() has been called for this invocation. /// diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamFactory.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamFactory.cs index 0170ddb27..534ed8fe6 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamFactory.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/ResponseStreamFactory.cs @@ -62,7 +62,7 @@ public static ResponseStream CreateStream(byte[] prelude) // This runs concurrently — SerializeToStreamAsync will block // until the handler finishes writing or reports an error. context.SendTask = context.RuntimeApiClient.StartStreamingResponseAsync( - context.AwsRequestId, lambdaStream, context.CancellationToken); + context.AwsRequestId, context.InvocationId, lambdaStream, context.CancellationToken); return lambdaStream; } @@ -71,11 +71,13 @@ public static ResponseStream CreateStream(byte[] prelude) internal static void InitializeInvocation( string awsRequestId, bool isMultiConcurrency, - RuntimeApiClient runtimeApiClient, CancellationToken cancellationToken) + RuntimeApiClient runtimeApiClient, CancellationToken cancellationToken, + string invocationId = null) { var context = new ResponseStreamContext { AwsRequestId = awsRequestId, + InvocationId = invocationId, StreamCreated = false, Stream = null, RuntimeApiClient = runtimeApiClient, diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs index b207d2bc5..9c81130b6 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs @@ -198,11 +198,12 @@ public Task ReportRestoreErrorAsync(Exception exception, String errorType = null /// This Task completes when the stream is finalized (MarkCompleted or error). /// /// The ID of the function request being responded to. + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// The ResponseStream that will provide the streaming data. /// The optional cancellation token to use. /// A Task representing the in-flight HTTP POST. The returned IDisposable is the RawStreamingHttpClient that owns the TCP connection. internal virtual async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { if (awsRequestId == null) throw new ArgumentNullException(nameof(awsRequestId)); if (responseStream == null) throw new ArgumentNullException(nameof(responseStream)); @@ -210,7 +211,7 @@ internal virtual async Task StartStreamingResponseAsync( var userAgent = _httpClient.DefaultRequestHeaders.UserAgent.ToString(); var rawClient = new RawStreamingHttpClient(LambdaEnvironment.RuntimeServerHostAndPort); - await rawClient.SendStreamingResponseAsync(awsRequestId, responseStream, userAgent, cancellationToken); + await rawClient.SendStreamingResponseAsync(awsRequestId, invocationId, responseStream, userAgent, cancellationToken); return rawClient; } diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaResponseStreamingCoreTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaResponseStreamingCoreTests.cs index 99e655fa8..0e790965e 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaResponseStreamingCoreTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/LambdaResponseStreamingCoreTests.cs @@ -429,7 +429,7 @@ public NoOpStreamingRuntimeApiClient(IEnvironmentVariables envVars) : base(envVars, new TestHelpers.NoOpInternalRuntimeApiClient()) { } internal override async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { // Provide the HTTP output stream so writes don't block await responseStream.SetHttpOutputStreamAsync(new MemoryStream(), cancellationToken); diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/ResponseStreamFactoryTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/ResponseStreamFactoryTests.cs index 52c6cfd92..d57433000 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/ResponseStreamFactoryTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/ResponseStreamFactoryTests.cs @@ -49,11 +49,14 @@ public MockStreamingRuntimeApiClient() { } + public string LastInvocationId { get; private set; } + internal override async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { StartStreamingCalled = true; LastAwsRequestId = awsRequestId; + LastInvocationId = invocationId; LastResponseStream = responseStream; await SendTaskCompletion.Task; return new NoOpDisposable(); @@ -149,6 +152,24 @@ public void CreateStream_CallsStartStreamingResponseAsync() Assert.NotNull(mock.LastResponseStream); } + /// + /// Validates that the invocation id stored on the context is passed through to + /// StartStreamingResponseAsync so the streaming response echoes it for cross-wiring protection. + /// + [Fact] + public void CreateStream_PassesInvocationIdToStartStreaming() + { + var mock = new MockStreamingRuntimeApiClient(); + ResponseStreamFactory.InitializeInvocation( + "req-inv", isMultiConcurrency: false, + mock, CancellationToken.None, "invocation_id"); + + ResponseStreamFactory.CreateStream(Array.Empty()); + + Assert.True(mock.StartStreamingCalled); + Assert.Equal("invocation_id", mock.LastInvocationId); + } + // --- GetSendTask --- /// diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs index f94864389..f78e8f700 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/RuntimeApiClientTests.cs @@ -84,7 +84,7 @@ public async Task StartStreamingResponseAsync_IncludesStreamingResponseModeHeade var stream = new ResponseStream(Array.Empty()); var client = CreateClientWithMockHandler(stream, out var handler); - await client.StartStreamingResponseAsync("req-1", stream, CancellationToken.None); + await client.StartStreamingResponseAsync("req-1", null, stream, CancellationToken.None); Assert.NotNull(handler.CapturedRequest); Assert.True(handler.CapturedRequest.Headers.Contains(StreamingConstants.ResponseModeHeader)); @@ -107,7 +107,7 @@ public async Task StartStreamingResponseAsync_IncludesChunkedTransferEncodingHea var stream = new ResponseStream(Array.Empty()); var client = CreateClientWithMockHandler(stream, out var handler); - await client.StartStreamingResponseAsync("req-2", stream, CancellationToken.None); + await client.StartStreamingResponseAsync("req-2", null, stream, CancellationToken.None); Assert.NotNull(handler.CapturedRequest); Assert.True(handler.CapturedRequest.Headers.TransferEncodingChunked); @@ -128,7 +128,7 @@ public async Task StartStreamingResponseAsync_DeclaresTrailerHeaderUpfront() var stream = new ResponseStream(Array.Empty()); var client = CreateClientWithMockHandler(stream, out var handler); - await client.StartStreamingResponseAsync("req-3", stream, CancellationToken.None); + await client.StartStreamingResponseAsync("req-3", null, stream, CancellationToken.None); Assert.NotNull(handler.CapturedRequest); Assert.True(handler.CapturedRequest.Headers.Contains("Trailer")); @@ -301,7 +301,7 @@ public async Task StartStreamingResponseAsync_NullRequestId_ThrowsArgumentNullEx var client = CreateClientWithMockHandler(stream, out _); await Assert.ThrowsAsync( - () => client.StartStreamingResponseAsync(null, stream, CancellationToken.None)); + () => client.StartStreamingResponseAsync(null, null, stream, CancellationToken.None)); } [Fact] @@ -311,7 +311,7 @@ public async Task StartStreamingResponseAsync_NullResponseStream_ThrowsArgumentN var client = CreateClientWithMockHandler(stream, out _); await Assert.ThrowsAsync( - () => client.StartStreamingResponseAsync("req-5", null, CancellationToken.None)); + () => client.StartStreamingResponseAsync("req-5", null, null, CancellationToken.None)); } } } diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs index 4409a6f68..c93b49430 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/StreamingE2EWithMoq.cs @@ -105,10 +105,13 @@ public CapturingStreamingRuntimeApiClient( }; } + public string LastInvocationId { get; private set; } + internal override async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { StartStreamingCalled = true; + LastInvocationId = invocationId; LastResponseStream = responseStream; // Use a real MemoryStream as the HTTP output stream so we capture actual bytes @@ -404,7 +407,7 @@ public MockMultiConcurrencyStreamingClient() : base(new TestEnvironmentVariables(), new NoOpInternalRuntimeApiClient()) { } internal override async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { // Provide the HTTP output stream so writes don't block await responseStream.SetHttpOutputStreamAsync(new MemoryStream()); diff --git a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs index 42c14f352..fb9591c55 100644 --- a/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs +++ b/Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestStreamingRuntimeApiClient.cs @@ -124,9 +124,10 @@ public TestStreamingRuntimeApiClient(IEnvironmentVariables environmentVariables, } internal override async Task StartStreamingResponseAsync( - string awsRequestId, ResponseStream responseStream, CancellationToken cancellationToken = default) + string awsRequestId, string invocationId, ResponseStream responseStream, CancellationToken cancellationToken = default) { StartStreamingResponseAsyncCalled = true; + LastInvocationId = invocationId; LastStreamingResponseStream = responseStream; // Simulate the HTTP stream being available From f8d7ffbac60f6413c0d3372bf85f324386670ac2 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Thu, 30 Jul 2026 18:01:32 -0400 Subject: [PATCH 5/9] Add change file for cross-wiring invocation-id support Amazon.Lambda.RuntimeSupport, Minor increment. --- .../changes/4133a18f-b6f9-4f24-886e-217756fbb670.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .autover/changes/4133a18f-b6f9-4f24-886e-217756fbb670.json diff --git a/.autover/changes/4133a18f-b6f9-4f24-886e-217756fbb670.json b/.autover/changes/4133a18f-b6f9-4f24-886e-217756fbb670.json new file mode 100644 index 000000000..93522a7b9 --- /dev/null +++ b/.autover/changes/4133a18f-b6f9-4f24-886e-217756fbb670.json @@ -0,0 +1,11 @@ +{ + "Projects": [ + { + "Name": "Amazon.Lambda.RuntimeSupport", + "Type": "Minor", + "ChangelogMessages": [ + "Add support for the Lambda-Runtime-Invocation-Id header for cross-wiring invoke protection. The runtime echoes the header back on the response and error calls when the Runtime API provides it, and treats an HTTP 410 Gone (invoke timeout) response as a non-fatal condition, logging it and continuing to the next invocation." + ] + } + ] +} From 2e31307148b683f96ec9e1de3f2b575e4c2fdd9f Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Fri, 31 Jul 2026 12:05:29 -0400 Subject: [PATCH 6/9] Add missing invocationId XML param doc on ErrorWithXRayCauseAsync The Release build treats the missing param tag as CS1573; add it so the build stays warning-clean. --- .../Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs index 1506480ad..1036d6642 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/InternalClientAdapted.cs @@ -468,6 +468,7 @@ public async System.Threading.Tasks.Task> Respon /// /// /// + /// The unique-per-invocation id to echo back for cross-wiring protection. When null, the header is not sent. /// /// public async System.Threading.Tasks.Task> ErrorWithXRayCauseAsync(string awsRequestId, string lambda_Runtime_Function_Error_Type, string errorJson, string xrayCause, string invocationId, System.Threading.CancellationToken cancellationToken) From 8ed9c6a589af864a0fcf8376c53ba80d21e6e3e0 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Fri, 31 Jul 2026 13:00:41 -0400 Subject: [PATCH 7/9] Remove redundant comments in RawStreamingHttpClient --- .../Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs index 2772c4235..6df4336fd 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Bootstrap/ResponseStreaming/RawStreamingHttpClient.cs @@ -23,8 +23,6 @@ using Amazon.Lambda.RuntimeSupport.Helpers; namespace Amazon.Lambda.RuntimeSupport.Client.ResponseStreaming -// RuntimeApiHeaders lives in the Amazon.Lambda.RuntimeSupport namespace, which encloses this one, -// so its HeaderInvocationId constant is in scope without an extra using. { /// /// A raw HTTP/1.1 client for sending streaming responses to the Lambda Runtime API @@ -91,8 +89,7 @@ public async Task SendStreamingResponseAsync( headers.Append("Transfer-Encoding: chunked\r\n"); headers.Append($"Trailer: {StreamingConstants.ErrorTypeTrailer}, {StreamingConstants.ErrorBodyTrailer}\r\n"); // Echo the per-invocation id back for cross-wiring protection. Only sent when the - // Runtime API provided it on /next. Must be added before the terminating blank line - // so it stays part of the header block rather than the body. + // Runtime API provided it on /next. if (!string.IsNullOrEmpty(invocationId)) headers.Append($"{RuntimeApiHeaders.HeaderInvocationId}: {invocationId}\r\n"); headers.Append("\r\n"); From c26abf007c284a88d55491b874b6ff313a744a90 Mon Sep 17 00:00:00 2001 From: gcbeatty Date: Fri, 31 Jul 2026 17:35:57 +0000 Subject: [PATCH 8/9] Remove redundant ExceptionInfo computation in ReportInvocationErrorAsync The non-invocationId overload computed ExceptionInfo.GetExceptionInfo but never used the result, then delegated to the invocationId overload which recomputes it. Drop the dead local; addresses PR review feedback. --- .../src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs index 9c81130b6..75c2feaba 100644 --- a/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs +++ b/Libraries/src/Amazon.Lambda.RuntimeSupport/Client/RuntimeApiClient.cs @@ -132,8 +132,6 @@ public Task ReportInvocationErrorAsync(string awsRequestId, Exception exception, if (exception == null) throw new ArgumentNullException(nameof(exception)); - var exceptionInfo = ExceptionInfo.GetExceptionInfo(exception); - return ReportInvocationErrorAsync(awsRequestId, null, exception, cancellationToken); } From 6ca75e39c2f79444fbc84859c20287741a36f28a Mon Sep 17 00:00:00 2001 From: gcbeatty Date: Fri, 31 Jul 2026 18:47:16 +0000 Subject: [PATCH 9/9] Update Annotations diagnostic line pins for InternalClientAdapted.cs changes The Annotations source-generator tests compile the real InternalClientAdapted.cs and assert exact source coordinates in it. The cross-wiring changes shifted RuntimeApiSerializationContext and the JsonSerializerContext.Default usages down, so update the expected line numbers to match (columns unchanged). Verified the full Annotations suite passes 583/583 on net8.0 and net10.0. --- .../SourceGeneratorTests.cs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Libraries/test/Amazon.Lambda.Annotations.SourceGenerators.Tests/SourceGeneratorTests.cs b/Libraries/test/Amazon.Lambda.Annotations.SourceGenerators.Tests/SourceGeneratorTests.cs index d60483457..7e757f152 100644 --- a/Libraries/test/Amazon.Lambda.Annotations.SourceGenerators.Tests/SourceGeneratorTests.cs +++ b/Libraries/test/Amazon.Lambda.Annotations.SourceGenerators.Tests/SourceGeneratorTests.cs @@ -2389,18 +2389,18 @@ private static DiagnosticResult[] GetExpectedRuntimeSupportDiagnostics() return new[] { // These are here because the System.Text.Json source generator isn't included in test compilations, so these members aren't generated. - DiagnosticResult.CompilerError("CS0534").WithSpan(clientFile, 85, 30, 85, 60).WithArguments(runtimeApiContext, "System.Text.Json.Serialization.JsonSerializerContext.GeneratedSerializerOptions.get"), - DiagnosticResult.CompilerError("CS0534").WithSpan(clientFile, 85, 30, 85, 60).WithArguments(runtimeApiContext, "System.Text.Json.Serialization.JsonSerializerContext.GetTypeInfo(System.Type)"), - DiagnosticResult.CompilerError("CS7036").WithSpan(clientFile, 85, 30, 85, 60).WithArguments("options", "System.Text.Json.Serialization.JsonSerializerContext.JsonSerializerContext(System.Text.Json.JsonSerializerOptions?)"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 157, 136, 157, 143).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 172, 135, 172, 142).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 275, 131, 275, 138).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 371, 135, 371, 142).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 386, 135, 386, 142).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 401, 135, 401, 142).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 510, 136, 510, 143).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 525, 135, 525, 142).WithArguments(runtimeApiContext, "Default"), - DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 540, 135, 540, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0534").WithSpan(clientFile, 89, 30, 89, 60).WithArguments(runtimeApiContext, "System.Text.Json.Serialization.JsonSerializerContext.GeneratedSerializerOptions.get"), + DiagnosticResult.CompilerError("CS0534").WithSpan(clientFile, 89, 30, 89, 60).WithArguments(runtimeApiContext, "System.Text.Json.Serialization.JsonSerializerContext.GetTypeInfo(System.Type)"), + DiagnosticResult.CompilerError("CS7036").WithSpan(clientFile, 89, 30, 89, 60).WithArguments("options", "System.Text.Json.Serialization.JsonSerializerContext.JsonSerializerContext(System.Text.Json.JsonSerializerOptions?)"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 161, 136, 161, 143).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 176, 135, 176, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 279, 131, 279, 138).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 393, 135, 393, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 408, 135, 408, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 423, 135, 423, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 539, 136, 539, 143).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 563, 135, 563, 142).WithArguments(runtimeApiContext, "Default"), + DiagnosticResult.CompilerError("CS0117").WithSpan(clientFile, 578, 135, 578, 142).WithArguments(runtimeApiContext, "Default"), // These are here because the internalvisibleto attribute isn't included in test compilations, so these types are inaccessible.