Add Umbraco load-testing pipeline (Locust + ALT)

.gitignore

@@ -4,3 +4,13 @@
 *.exe
 *.backup
 *.tfstate
+
+# Re-allow project files that the broad rules above would otherwise ignore.
+!loadtests/tiers.json
+!loadtests/scenarios/**/*.json
+!dashboards/*.workbook.json
+!**/.terraform.lock.hcl
+
+.terraform/
+.idea/
+__pycache__/

Terraform/main.tf

@@ -1,27 +1,48 @@
 terraform {
   required_providers {
     azurerm = {
-      source = "hashicorp/azurerm"
-      version = ">=4.20.0"
+      source  = "hashicorp/azurerm"
+      version = "~> 4.20"
     }
   }
 
   required_version = ">= 1.3.9"
+
+  # Local state is fine for the ephemeral RG-per-run model. Uncomment the backend below
+  # for shared state (pre-create the `tfstate` container in the history storage account).
+  # backend "azurerm" {
+  #   resource_group_name  = "umbraco-loadtest-history-rg"
+  #   storage_account_name = "<your-history-storage-account>"
+  #   container_name       = "tfstate"
+  #   key                  = "loadtest.tfstate"
+  # }
 }
 
 provider "azurerm" {
   features {}
 }
 
+# Read tier catalog from loadtests/tiers.json.
+locals {
+  tier_specs = jsondecode(file("${path.module}/../loadtests/tiers.json")).tiers
+}
+
 module "umbraco" {
-  source                  = "./modules/umbraco"
+  source = "./modules/umbraco"
+
   resource_name_prefix    = var.resource_name_prefix
-  resource_group_location = var.resource_group_location
   resource_group_name     = var.resource_group_name
-  umbraco_cms_versions    = var.umbraco_cms_versions
-  # Azure Login Credentials 
-  client_id     = var.client_id
-  client_secret = var.client_secret
-  tenant_id     = var.tenant_id
-  app_service_plan_sku = var.app_service_plan_sku
-}
+  resource_group_location = var.resource_group_location
+
+  tier_specs        = local.tier_specs
+  test_cases        = var.test_cases
+  seeder_preset     = var.seeder_preset
+  pool_dtu_override = var.pool_dtu_override
+  app_sku_override  = var.app_sku_override
+
+  budget_alert_amount        = var.budget_alert_amount
+  budget_alert_threshold_pct = var.budget_alert_threshold_pct
+  budget_alert_emails        = var.budget_alert_emails
+
+  build_id = var.build_id
+}

Terraform/modules/umbraco/main.tf

@@ -1,55 +1,191 @@
-# Resource group
+locals {
+  tiers_in_use = toset([for v in var.test_cases : v.tier])
+
+  # Azure SQL admin login must start with a letter; prefix one since random_string can start with a digit.
+  sql_admin_login = "u${random_string.admin_login.result}"
+
+  # Per-DB DTU cap per tier, with the queue-time override applied if set.
+  # The override lets a run size SQL independently of the app tier to isolate
+  # which side is the bottleneck.
+  tier_db_dtu = {
+    for t in local.tiers_in_use :
+    t => var.pool_dtu_override != 0 ? var.pool_dtu_override : var.tier_specs[t].dtu_max
+  }
+
+  # Pool eDTU capacity: smallest valid Standard pool that can hold a DB at the
+  # tier cap. Standard pool minimum is 50; bump to 100 when the per-DB cap
+  # requires it. Beyond 100 the next valid Standard step is 200.
+  tier_pool_dtu = {
+    for t, db_dtu in local.tier_db_dtu :
+    t => db_dtu <= 50 ? 50 : db_dtu <= 100 ? 100 : 200
+  }
+
+  common_tags = {
+    project    = "umbraco-loadtest"
+    managed_by = "terraform"
+    build_id   = var.build_id
+  }
+}
+
 resource "azurerm_resource_group" "rg" {
   name     = var.resource_group_name
   location = var.resource_group_location
+  tags     = local.common_tags
+}
+
+# Cost guard: monthly budget on the ephemeral RG that emails when MTD spend
+# crosses the threshold percentage. Skipped entirely when budget_alert_emails
+# is empty (the default) so it stays opt-in.
+#
+# Trade-offs:
+# * Monthly grain is Azure's smallest budget window — there's no per-run /
+#   per-hour. A single run accruing $4 won't fire (well below 80% × $50);
+#   the alert protects against MULTIPLE forgotten gates compounding within
+#   a billing month, OR a runaway loop.
+# * The budget resource itself is destroyed when terraform destroy runs at
+#   end-of-pipeline cleanup, so an "orphaned" RG that survives cleanup also
+#   loses its budget — at which point you'd want a subscription-scope budget
+#   instead. Out of scope here; this RG-scope budget covers the normal case.
+resource "azurerm_consumption_budget_resource_group" "ephemeral" {
+  count             = length(var.budget_alert_emails) > 0 ? 1 : 0
+  name              = "${var.resource_group_name}-budget"
+  resource_group_id = azurerm_resource_group.rg.id
+
+  amount     = var.budget_alert_amount
+  time_grain = "Monthly"
+
+  # Start of current month, in UTC. formatdate keeps it stable across re-applies
+  # within the same month; a re-apply in a new month will roll the start date
+  # forward, which is the desired behavior for monthly grain.
+  #
+  # end_date is set explicitly ~10 years out. azurerm provider defaults this
+  # to start_date + 1 year, which silently disables the budget after 12
+  # months from initial apply — easy to miss because Terraform doesn't drift-
+  # detect a "budget that no longer monitors anything". Pushing it ~decade
+  # out makes the time window long enough that re-applies (which always
+  # happen on each pipeline run) will refresh start_date well before
+  # end_date matters.
+  time_period {
+    start_date = formatdate("YYYY-MM-01'T'00:00:00'Z'", timestamp())
+    end_date   = formatdate("YYYY-MM-01'T'00:00:00'Z'", timeadd(timestamp(), "87600h"))
+  }
+
+  notification {
+    enabled        = true
+    threshold      = var.budget_alert_threshold_pct
+    operator       = "GreaterThan"
+    threshold_type = "Actual"
+    contact_emails = var.budget_alert_emails
+  }
+
+  lifecycle {
+    # start_date drifts via timestamp() on every plan; ignoring its diff
+    # avoids spurious "in-place update" plans that don't change semantics.
+    ignore_changes = [time_period]
+  }
 }
 
-# Random string for SQL server login
 resource "random_string" "admin_login" {
-  length     = 16
-  special    = false
-  depends_on = [azurerm_resource_group.rg]
+  length  = 15
+  special = false
 }
 
+# SQL admin password. Azure SQL needs 3 of 4 char categories; force upper+lower+numeric.
 resource "random_password" "admin_password" {
-  length     = 16
-  special    = false
-  depends_on = [azurerm_resource_group.rg]
+  length      = 24
+  special     = false
+  min_upper   = 1
+  min_lower   = 1
+  min_numeric = 1
 }
 
-# App Service Plan
+# One plan per tier in use; same-tier cases share it (only one app hot at a time).
+# Override lets a run size the app independently of the tier (paired with
+# pool_dtu_override on the SQL side) to isolate app-vs-SQL bottlenecks.
 resource "azurerm_service_plan" "appserviceplan" {
-  name                = "${var.resource_name_prefix}-appserviceplan"
+  for_each            = local.tiers_in_use
+  name                = "${var.resource_name_prefix}-asp-${lower(each.key)}"
   location            = azurerm_resource_group.rg.location
   resource_group_name = azurerm_resource_group.rg.name
   os_type             = "Windows"
-
-  # We are using a variable for the sku version because we want to be able to change it if needed. 
-  sku_name            = var.app_service_plan_sku
+  sku_name            = var.app_sku_override != "" ? var.app_sku_override : var.tier_specs[each.key].app_sku
+  tags                = merge(local.common_tags, { tier = each.key })
+}
+
+# SQL server per tier - hosts the tier's Elastic Pool. One server per tier keeps
+# the case-level resource graph identical between same-tier cases (they share
+# server + pool) and matches the Cloud model where each plan has its own pool.
+resource "azurerm_mssql_server" "sql_server" {
+  for_each                     = local.tiers_in_use
+  name                         = "${var.resource_name_prefix}-sql-${lower(each.key)}"
+  resource_group_name          = azurerm_resource_group.rg.name
+  location                     = azurerm_resource_group.rg.location
+  version                      = "12.0"
+  administrator_login          = local.sql_admin_login
+  administrator_login_password = random_password.admin_password.result
+  minimum_tls_version          = "1.2"
+  tags                         = merge(local.common_tags, { tier = each.key })
 }
 
-resource "azurerm_load_test" "load_test" {
-  location            = var.resource_group_location
-  name                = "${var.resource_name_prefix}-loadtest"
-  resource_group_name = var.resource_group_name
+resource "azurerm_mssql_firewall_rule" "allow_azure_services" {
+  for_each         = local.tiers_in_use
+  name             = "AllowAzureServices-${lower(each.key)}"
+  server_id        = azurerm_mssql_server.sql_server[each.key].id
+  start_ip_address = "0.0.0.0"
+  end_ip_address   = "0.0.0.0"
+}
 
-  depends_on = [azurerm_service_plan.appserviceplan]
+# Standard-tier Elastic Pool per tier. Pool eDTU is the smallest valid Standard
+# size that can hold a DB at the tier cap; per-DB max DTU equals the cap.
+# max_size_gb=50 is the Standard-tier minimum; our test data is well under that,
+# so the floor is enough and avoids paying for storage we won't use.
+resource "azurerm_mssql_elasticpool" "pool" {
+  for_each            = local.tiers_in_use
+  name                = "${var.resource_name_prefix}-pool-${lower(each.key)}"
+  resource_group_name = azurerm_resource_group.rg.name
+  location            = azurerm_resource_group.rg.location
+  server_name         = azurerm_mssql_server.sql_server[each.key].name
+  max_size_gb         = 50
+  tags                = merge(local.common_tags, { tier = each.key })
+
+  sku {
+    name     = "StandardPool"
+    tier     = "Standard"
+    capacity = local.tier_pool_dtu[each.key]
+  }
+
+  per_database_settings {
+    min_capacity = 0
+    max_capacity = local.tier_db_dtu[each.key]
+  }
 }
 
-# We create a module called versions, the reason for that is because we want to have multiple app services with different Umbraco Versions.
+# Azure Load Testing resource lives in a long-lived RG (see scripts/ensure-history-infra.ps1).
+
 module "versions" {
-  # We use a for_each so it creates a module for each version of Umbraco we have defined in variables.
-  for_each                = var.umbraco_cms_versions
-  source                  = "./versions"
+  for_each = var.test_cases
+  source   = "./versions"
+
   resource_name_prefix    = var.resource_name_prefix
   resource_group_name     = azurerm_resource_group.rg.name
   resource_group_location = azurerm_resource_group.rg.location
-  service_plan_id         = azurerm_service_plan.appserviceplan.id
-  dotnet_version          = each.value["dotnet_version"]
-  umbraco_cms_version     = each.value["umbraco_version"]
-  admin_login             = random_string.admin_login.result
-  admin_password          = random_password.admin_password.result
-  client_id               = var.client_id
-  client_secret           = var.client_secret
-  tenant_id               = var.tenant_id
-}
+  service_plan_id         = azurerm_service_plan.appserviceplan[each.value.tier].id
+
+  test_case_id         = each.key
+  dotnet_version       = each.value.dotnet_version
+  umbraco_version      = each.value.umbraco_version
+  scenario             = each.value.scenario
+  app_settings_overlay = each.value.app_settings_overlay
+
+  admin_login    = local.sql_admin_login
+  admin_password = random_password.admin_password.result
+
+  sql_server_name              = azurerm_mssql_server.sql_server[each.value.tier].name
+  sql_server_id                = azurerm_mssql_server.sql_server[each.value.tier].id
+  sql_server_fqdn              = azurerm_mssql_server.sql_server[each.value.tier].fully_qualified_domain_name
+  elastic_pool_id              = azurerm_mssql_elasticpool.pool[each.value.tier].id
+  sql_firewall_rule_dependency = azurerm_mssql_firewall_rule.allow_azure_services[each.value.tier].id
+
+  seeder_preset = var.seeder_preset
+  common_tags   = local.common_tags
+}