sst@2.49.4: [Regression] Multi-region deployments fail due to stack region not being considered at deploy-time

[gcxnpl]

Hi! 👋

Firstly, thanks for your work on this project! 🙂

Today I used patch-package to patch sst@2.49.4 for the project I'm working on.

In one of our stacks, we have a setup where one stack deploys to ap-southeast-2 and one deploys to us-east-1. When we upgraded to the mentioned version of SST from 2.48.5 the us-east-1 stack was deploying in ap-southeast-2.

I built the projects on both versions of SST and noted the build manifests looked identical, so the issue seemed to be with how SST ingests the manifest during deploy-time, potentially related to the migration of using the native CDK module instead of the fork the SST team was maintaining. It seemed like the region of the stack as a whole was being utilised instead of the per-stack region configuration exposed within the manifest. crossRegionReferences was enabled in all relevant stacks as well.

The patch below solved the problem however it is not a holistic solution - I patched until it resolved our issues. I'm raising this patch to get eyes on the problem from SMEs so that we can get a conclusive fix merged in. Thanks.

diff --git a/node_modules/sst/stacks/deploy.js b/node_modules/sst/stacks/deploy.js
index 7e764c6..c801c05 100644
--- a/node_modules/sst/stacks/deploy.js
+++ b/node_modules/sst/stacks/deploy.js
@@ -5,6 +5,8 @@ import { useAWSClient, useAWSProvider } from "../credentials.js";
 import { Logger } from "../logger.js";
 import { filterOutputs, isFailed, monitor, } from "./monitor.js";
 import { VisibleError } from "../error.js";
+
+
 export async function publishAssets(stacks) {
     Logger.debug("Publishing assets");
     const { cdk } = useProject().config;
@@ -74,7 +76,7 @@ export async function deployMany(stacks) {
 export async function deploy(stack) {
     const bus = useBus();
     const { cdk } = useProject().config;
-    Logger.debug("Deploying stack", stack.id);
+    Logger.debug("Deploying stack", stack.stackName, stack.id, stack.environment);
     try {
         const cfnStack = await getCloudFormationStack(stack);
         await addInUseExports(stack, cfnStack);
@@ -86,7 +88,7 @@ export async function deploy(stack) {
         await buildAndPublishAssets(deployment, stack);
         if (cfnStack?.StackStatus === "ROLLBACK_COMPLETE" ||
             cfnStack?.StackStatus === "ROLLBACK_FAILED") {
-            await deleteCloudFormationStack(stack.stackName);
+            await deleteCloudFormationStack(stack.stackName, stack.environment.region);
         }
         const stackParams = await buildCloudFormationStackParams(deployment, stack, cdk);
         try {
@@ -238,8 +240,10 @@ async function buildCloudFormationStackParams(deployment, stack, cdkOptions) {
     const templateUrl = s3Url
         ? `https://s3.${env.resolvedEnvironment.region}.amazonaws.com/${s3Url[1]}/${s3Url[2]}`
         : stack.stackTemplateAssetObjectUrl;
+    Logger.debug("Building CloudFormation with environment details", env.resolvedEnvironment);
     return {
         StackName: stack.stackName,
+        StackRegion: env.resolvedEnvironment.region,
         TemplateURL: templateUrl,
         RoleARN: executionRoleArn,
         //TemplateBody: bodyParameter.TemplateBody,
@@ -258,7 +262,10 @@ async function buildCloudFormationStackParams(deployment, stack, cdkOptions) {
 }
 async function getCloudFormationStack(stack) {
     const { CloudFormationClient, DescribeStacksCommand } = await import("@aws-sdk/client-cloudformation");
-    const client = useAWSClient(CloudFormationClient);
+    Logger.debug("Getting CloudFormation stack details", stack.stackName, stack.environment.region);
+    const client = new CloudFormationClient({
+        region: stack.environment.region,
+    });
     try {
         const { Stacks: stacks } = await client.send(new DescribeStacksCommand({
             StackName: stack.stackName,
@@ -280,17 +287,30 @@ async function getCloudFormationStack(stack) {
 }
 async function createCloudFormationStack(input) {
     const { CloudFormationClient, CreateStackCommand } = await import("@aws-sdk/client-cloudformation");
-    const client = useAWSClient(CloudFormationClient);
-    await client.send(new CreateStackCommand(input));
+    Logger.debug("Creating CloudFormation stack", input);
+    const client = new CloudFormationClient({
+        region: input.StackRegion,
+    });
+    // Remove StackRegion from input as it's not a valid CloudFormation parameter
+    const { StackRegion, ...cfnInput } = input;
+    await client.send(new CreateStackCommand(cfnInput));
 }
 async function updateCloudFormationStack(input) {
     const { CloudFormationClient, UpdateStackCommand } = await import("@aws-sdk/client-cloudformation");
-    const client = useAWSClient(CloudFormationClient);
-    await client.send(new UpdateStackCommand(input));
+    Logger.debug("Updating CloudFormation stack", input);
+    const client = new CloudFormationClient({
+        region: input.StackRegion,
+    });
+    // Remove StackRegion from input as it's not a valid CloudFormation parameter
+    const { StackRegion, ...cfnInput } = input;
+    await client.send(new UpdateStackCommand(cfnInput));
 }
-async function deleteCloudFormationStack(stackName) {
+async function deleteCloudFormationStack(stackName, region) {
     const { CloudFormationClient, DescribeStacksCommand, DeleteStackCommand } = await import("@aws-sdk/client-cloudformation");
-    const client = useAWSClient(CloudFormationClient);
+    Logger.debug("Deleting CloudFormation stack", stackName, region);
+    const client = new CloudFormationClient({
+        region: region,
+    });
     try {
         await client.send(new DeleteStackCommand({ StackName: stackName }));
         while (true) {

This issue body was partially generated by patch-package.

[macku]

For more context, we are using the crossRegionReferences flag and have a multi-region app due to the AWS WAF configuration requirement to be in a different region (us-east-1) then the one we deploy the app logic to (ap-southeast-2)