diff --git a/.gitignore b/.gitignore index 4b07eb2..f4be562 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,13 @@ *.exe *.backup *.tfstate + +# Re-allow project files that the broad rules above would otherwise ignore. +!loadtests/tiers.json +!loadtests/scenarios/**/*.json +!dashboards/*.workbook.json +!**/.terraform.lock.hcl + +.terraform/ +.idea/ +__pycache__/ diff --git a/LoadTestVersions.yaml b/LoadTestVersions.yaml deleted file mode 100644 index 7f0a871..0000000 --- a/LoadTestVersions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -version: v0.1 -testId: LoadTestVersions -displayName: Load Test Versions -testPlan: loadtests/JmeterTest.jmx -description: Load Test Homepage -engineInstances: 1 \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..5ca13b1 --- /dev/null +++ b/README.md @@ -0,0 +1,927 @@ +# Umbraco Load Testing Infrastructure + +A starting point for load testing Umbraco CMS on Azure using Terraform, Locust, and Azure Load Testing. The focus is establishing CMS performance baselines, but the infrastructure, pipeline, and test framework are designed to be reusable — other teams (Cloud, DXP, Workflow, Commerce, …) can fork the repo, add their own packages and configuration, and build team-specific scenarios on top of the same foundation. + +## Goals + +- **Establish CMS performance baselines** — repeatable, comparable metrics for Umbraco under standard conditions across versions. +- **Enable version comparison** — run the same scenario against multiple Umbraco versions to detect regressions or improvements. +- **Build infrastructure capacity benchmarks** — what each App Service + SQL DTU combination can handle, as reference data for sizing decisions. (The shipped tiers mirror Umbraco Cloud's progression — dedicated App Service SKUs `P0v3 → P3v3` paired with per-DB DTU caps `20 / 50 / 100 / 200` in Standard-tier Elastic Pools. The `appSkuOverride` and `poolDtuOverride` queue parameters and `tiers.json` are the levers for sweeping other combinations.) +- **Provide a reusable foundation** — a working pipeline + harness that other teams can extend without rebuilding infra. + +Thresholds (fail-the-pipeline gates) are intentionally **deferred** until baselines exist — once we know what "normal" looks like per scenario per tier, Locust thresholds can be wired in to fail on regression. + +## Overview + +This project provisions isolated Azure environments for arbitrary combinations of **(Umbraco version × infrastructure tier × scenario)**, seeds them with test data using [Umbraco.Cms.TestDataSeeder](https://www.nuget.org/packages/Umbraco.Cms.TestDataSeeder/), and runs Locust load tests via Azure Load Testing service. + +**Supported Umbraco versions: v13–v18 (subject to seeder availability).** Each major maps to a specific .NET runtime (see [Umbraco-major → .NET-runtime map](#umbraco-major--net-runtime-map) below). Today only **v17** has a published `Umbraco.Cms.TestDataSeeder` build (`17.0.0-beta.2`); v13–v16 and v18 queues fail at validation with a clear message until the seeder ships for those majors. Update the maps in `scripts/resolve-run-config.ps1` (validator-side) and `Terraform/modules/umbraco/scripts/install-umbraco-cms-on-appservice.ps1` (install-side) in lockstep when a new seeder version ships. + +The `DeliveryApi` scenario is **v17+ only** — its `Program.cs` overlay uses v17's builder shape. The validator rejects the `DeliveryApi` + < v17 combination before provisioning. Use the `Default` scenario for older majors. + +Locust tests execute on Azure Load Testing's managed infrastructure (dedicated Standard_D4d_v4 VMs), not on the pipeline agent. This ensures consistent, reliable performance measurements. + +A pipeline run is parameterised by a list of **test cases**. Each case picks: + +- **Umbraco/.NET version pair** (e.g. `17.0.0` on `v10.0`) +- **Infrastructure tier** — `Starter` / `Standard` / `Pro` / `Enterprise`, defined in [`loadtests/tiers.json`](loadtests/tiers.json) (App Service Plan SKU + per-DB DTU cap inside a Standard-tier Elastic Pool) +- **Scenario** — a folder under `loadtests/scenarios/` containing the Umbraco `appsettings.json` overlay for that scenario plus optional `scenario.yaml` load-profile overrides + +Cases on the same tier in one run share an App Service Plan; cases on different tiers each get their own. Tests within a run run sequentially (one App Service hot at a time) so each measurement gets the full plan capacity. + +## Prerequisites + +- Azure subscription with appropriate permissions. The pipeline service principal needs: + - Standard create/manage rights on the ephemeral and history resource groups (Contributor is enough for resources). + - **`Microsoft.Storage/storageAccounts/listKeys/action`** on the history storage account — Storage Account Contributor (or any role that includes `listKeys/action`) is enough. Downstream scripts (`publish-load-test-results.ps1`, `_history-helpers.ps1`) fetch the account key at runtime and authenticate with `--account-key`. RBAC + `--auth-mode login` would be a stricter alternative but requires `Microsoft.Authorization/roleAssignments/write` for the SP, which is a heavier permissions ask. + - **`Microsoft.Authorization/roleAssignments/write`** on the history resource group (or specifically on the Data Collection Rule once it exists) — `ensure-monitoring-infra.ps1` grants the same SP "Monitoring Metrics Publisher" on the DCR so it can POST to the Logs Ingestion API. User Access Administrator on the history RG is sufficient. + - **Directory read** in Microsoft Entra ID — the pipeline calls `az ad sp show` to resolve its own object ID for the role grant above. Default tenants allow this for any authenticated principal; locked-down tenants may need an explicit `Directory Readers` role assignment on the SP. +- Azure DevOps organization with: + - Service connection to Azure (`terraform-umbraco-load-testing-az-connection`) + - Variable group `umbraco-loadtest-history` with at minimum: `historyResourceGroup`, `historyLocation`, `historyLoadTestName`, `historyStorageAccount` (override the placeholder `loadtestchangeme` with a globally-unique 3-24 lowercase alphanumeric value), `historyContainer`. +- Terraform >= 1.13.3 (pinned at 1.13.3 in CI; pipelines install via TerraformInstaller@0) +- PowerShell Core (pwsh) 7.3+ — earlier versions silently swallow native command failures (`dotnet build` errors etc.) because `$ErrorActionPreference = "Stop"` doesn't apply to native exit codes; 7.3 introduced `$PSNativeCommandUseErrorActionPreference` which the install script sets. + +## First-time setup + +A new team forking this project should: + +1. **Pick a globally-unique storage account name** (3-24 lowercase alphanumeric chars). This will host the long-lived run history. Override `historyStorageAccount` in the variable group with this value — the placeholder `loadtestchangeme` is rejected by `ensure-history-infra.ps1`. +2. **Create the AzDO variable group** `umbraco-loadtest-history` with the five history variables above. +3. **Configure the service principal** with the permissions listed in Prerequisites — Contributor on the subscription (or scoped narrower) is sufficient. +4. **Queue the pipeline once.** The first run creates the long-lived history infra (RG, ALT resource, storage account, container) **and the monitoring infra** (Log Analytics workspace, custom table, DCR, Workbook) alongside the per-case provisioning. The Workbook URL is printed in the `ensureMonitoringInfra` stage log — pin it to your Azure portal dashboard. +5. **(Local-dev users)** `az login` and verify you can list keys for the history storage account — `show-trends.ps1` / `check-regression.ps1` / `compare-runs.ps1` (history mode) run locally need the same `listKeys/action` the pipeline SP uses: + ```bash + az storage account keys list -n -g umbraco-loadtest-history-rg --query "[0].keyName" -o tsv + ``` + If that command works, the analysis tools will work. If it fails with "AuthorizationFailed", grant yourself Storage Account Contributor (or higher) on the SA scope. +6. **Queue 3-5 baseline runs** with the same configuration to populate history (see "Establishing a baseline" below). Until cells have ≥ 3 prior runs, the pipeline's regression-check stage reports "insufficient baseline" and exits 0 — it's safe to enable from day one. + +> ⚠️ Before queueing your first non-default run, skim the [Pitfalls](#pitfalls) section — security, name-length, overlay-precedence, and cleanup gotchas live there. + +**History storage scales sub-linearly thanks to a lifecycle policy.** `ensure-history-infra.ps1` tiers blobs from Hot → Cool after 30 days (override via `-LifecycleCoolAfterDays`). Cool tier is ~3× cheaper than Hot for storage, retrieval is still instant, and the per-read cost is negligible at PS-tool / Workbook query frequency. + +Archive tier transition is **disabled by default** (`-LifecycleArchiveAfterDays = 0`) because the policy filter is coarse — it applies to every blob in the container, including the small `summary.ndjson` files the PS analysis tools read. Archive saves another ~3× on storage but takes **hours** to rehydrate; reads against year-old `summary.ndjson` would fail with HTTP 409 until rehydration completes. Enable Archive only if you've segregated raw zips into a separate prefix and tightened the policy filter, or you're OK with the rehydration delay. + +## Project Structure + +``` +├── azure-pipeline.yml # Main load test pipeline (manual queue) +├── README.md +│ +├── dashboards/ +│ └── loadtest.workbook.json # Azure Workbook: Trends / Tiers / Versions / Compare / Runs / Glossary over LoadTestSummary_CL +│ +├── templates/ +│ └── load-test-job.yml # Per-case load test template (testCaseId lookup pattern) +│ +├── scripts/ +│ ├── resolve-run-config.ps1 # Pipeline entry: resolves queue-time params + invokes validator +│ ├── prepare-test-cases.ps1 # Validator: validates testCases, flattens scenario appsettings, +│ │ # resolves load profile, emits testCasesJson + resolvedTestCases +│ ├── ensure-history-infra.ps1 # Idempotently provisions long-lived RG, Azure Load Testing, storage +│ ├── generate-loadtest-config.ps1 # Per-case ALT YAML config (testId, appComponents, failureCriteria) +│ ├── stop-all-app-services.ps1 # Pre-test and end-of-run sweep: stop App Services in the case set +│ ├── publish-load-test-results.ps1 # Exports per-test NDJSON + raw artifacts to history storage +│ ├── compare-runs.ps1 # Markdown delta report between two runs (CSV or history) +│ ├── show-trends.ps1 # (version × tier) p95/p99/error% matrix from history NDJSON +│ ├── check-regression.ps1 # Compare latest run vs baseline-median; non-zero exit on regression (gate) +│ ├── _helpers.ps1 # Shared helpers dot-sourced by other scripts (Get-Pct, Get-StorageAccountKey, …) +│ └── _history-helpers.ps1 # Shared helpers for the history-NDJSON consumers (dot-sourced) +│ +├── loadtests/ +│ ├── _helpers.py # Shared workload mixin + inventory/Delivery API probes used by scenario locustfiles +│ ├── scenarios//locustfile.py # Scenario-specific workload, imports from _helpers +│ ├── tiers.json # Tier catalog (Starter / Standard / Pro / Enterprise → SKUs + DTU caps) +│ └── scenarios/ +│ ├── Default/ +│ │ ├── AdditionalSetup/ +│ │ │ └── appsettings.json # {} — identity overlay +│ │ ├── locustfile.py +│ │ └── scenario.yaml +│ └── DeliveryApi/ +│ ├── AdditionalSetup/ +│ │ ├── appsettings.json # enables Umbraco:CMS:DeliveryApi +│ │ └── Program.cs # registers .AddDeliveryApi() in builder chain +│ ├── locustfile.py +│ └── scenario.yaml +│ +└── Terraform/ + ├── main.tf # Root module + ├── variables.tf # Input variables (testCases-shaped test_cases) + ├── output.tf # test_case_outputs map keyed by testCaseId + ├── terraform.tfvars.example # Example configuration + │ + └── modules/umbraco/ + ├── main.tf # Reads tiers.json; for_each App Service Plan over tiers in use + ├── variables.tf + ├── output.tf + │ + ├── scripts/ + │ └── install-umbraco-cms-on-appservice.ps1 + │ + └── versions/ # Per-case resources + ├── main.tf # SQL Server, Database, App Service (merges overlay into app_settings) + ├── variables.tf + └── output.tf +``` + +## Configuration + +### Pipeline Parameters + +The queue UI splits into three concerns: **what to test**, **which tiers to test it on**, and **how hard to test**. Each lives on its own knob so you can mix freely (e.g. "stress profile against Standard only" or "smoke profile against all four tiers"). + +**What to test:** + +| Parameter | Description | Default | Options | +|-----------|-------------|---------|---------| +| `umbracoVersion` | Umbraco CMS version. Free-text — accepts prereleases (`17.0.0-rc.1`, `17.1.0-beta.2`). The validator accepts v13, v17, v18 today (the majors with a published `Umbraco.Cms.TestDataSeeder` build — v18 reuses the v17 seeder as a fallback); v14/v15/v16 fail validation with a "seeder hasn't shipped" message. The major segment maps to the .NET runtime automatically (see table below). | 17.0.0 | free text | +| `scenario` | Scenario folder name (must match a folder under `loadtests/scenarios/`) | Default | extend the `values` list when adding scenarios | + +**Which tiers:** + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `runStarter` | Run the Starter tier (P0v3 app, 20 DTU SQL) | true | +| `runStandard` | Run the Standard tier (P1v3 app, 50 DTU SQL) | false | +| `runPro` | Run the Pro tier (P2v3 app, 100 DTU SQL) | false | +| `runEnterprise` | Run the Enterprise tier (P3v3 app, 200 DTU SQL) | false | + +At least one tier must be selected — the validator fails the run if all are unchecked. + +**Load profile (intensity):** + +| Profile | Seeder preset | VUs | Spawn rate | Duration | Engines | +|---|---|---|---|---|---| +| `smoke` | Small | 50 | 10/s | 60s | 1 | +| `standard` | Medium | 100 | 10/s | 300s | 1 | +| `stress` | Large | 300 | 50/s | 600s | 2 | + +The profile only encodes load intensity — the same profile can drive any combination of tiers. Tuning a profile is a single-place edit to the inline `switch` in `azure-pipeline.yml`'s "Resolve profile + validate scenario" step. + +**.NET runtime is derived, not selected.** The prep step maps the Umbraco major version → required .NET runtime; the pipeline installs the matching SDK and Terraform sets the App Service runtime accordingly. Extend the map in `scripts/resolve-run-config.ps1` (and the seeder-version map in the install script) when a new Umbraco major ships. + +#### Umbraco-major → .NET-runtime map + +| Umbraco major | App Service runtime | SDK installed by pipeline | +|---:|:---:|:---:| +| 13 | `v8.0` | `8.x` | +| 14 | `v8.0` | `8.x` | +| 15 | `v9.0` | `9.x` | +| 16 | `v9.0` | `9.x` | +| 17 | `v10.0` | `10.x` | +| 18 | `v10.0` | `10.x` | + +The v18 mapping should be verified against the actual v18 release notes before relying on it. + +**For multi-version comparisons in a single queue** (e.g. 17.0.0 vs 17.0.1 on the same tier): queue the pipeline twice — once per version. The ALT Compare runs view aggregates across pipeline runs anyway (testId is per-scenario, not per-pipeline-run), so two queues end up in the same comparison view. + +**Run configuration (orthogonal knobs):** + +| Parameter | Description | Default | Options | +|-----------|-------------|---------|---------| +| `azureRegion` | Azure region | West Europe | West Europe, North Europe, East US, West US 2 | +| `resourcePrefix` | Resource name prefix (max 16 chars) | umbraco-loadtest | — | +| `skipWarmup` | Skip warmup (test cold-start / cache warm-up behaviour) | false | true, false | +| `validationTimeoutMinutes` | How long resources stay alive after tests | 60 | 15, 30, 60, 120, 240 | +| `poolDtuOverride` | Force every case onto a specific per-DB DTU cap (decouples DB sizing from tier) | Auto | Auto, 10, 20, 50, 100, 200 | +| `appSkuOverride` | Force every case onto a specific App Service Plan SKU (decouples app sizing from tier) | Auto | Auto, P0v3, P1v3, P2v3, P3v3 | +| `seederPresetOverride` | Force every case onto a specific TestDataSeeder preset (decouples content size from load profile) | Auto | Auto, Small, Medium, Large, Massive | + +**Pool DTU override.** When set to a value other than `Auto`, every test case in the run uses the same per-DB DTU cap regardless of the tier's default — useful for "is SQL actually the bottleneck?" experiments. Default caps are Starter→20, Standard→50, Pro→100, Enterprise→200 (see `tiers.json`). The Elastic Pool's eDTU capacity is sized automatically to the smallest valid Standard pool that can hold a DB at the chosen cap. + +**App SKU override.** Counterpart to `poolDtuOverride` on the app side. When set to a value other than `Auto`, every test case uses the same App Service Plan SKU regardless of the tier's default — useful for "is the app actually the bottleneck?" experiments. Default SKUs are Starter→P0v3, Standard→P1v3, Pro→P2v3, Enterprise→P3v3. + +**Seeder preset override.** Decouples content size from the load profile. `Auto` keeps the existing coupling (smoke→Small, standard→Medium, stress→Large); explicit values unlock off-diagonal combinations like Massive content + smoke load or Small content + stress load, and are the only way to reach the Massive preset. Approximate seeder times: Small ~10 min, Medium ~30 min, Large ~60 min, Massive ~120 min. + +The validator (`scripts/prepare-test-cases.ps1`) catches typos, missing scenario folders, and duplicate `(umbraco, tier, scenario)` triples *before* any Azure resource is provisioned. It also enforces sensible ranges on the load profile values the profile resolver hands it (`userAmount` 1–1000, `spawnRate` 1–100, `testDuration` 30–7200 seconds). + +## Tiers + +`loadtests/tiers.json` is the **single source of truth** for tier names + SKUs. Both Terraform (provisioning) and the PowerShell validator read this same file: + +```json +{ + "tiers": { + "Starter": { "app_sku": "P0v3", "dtu_max": 20 }, + "Standard": { "app_sku": "P1v3", "dtu_max": 50 }, + "Pro": { "app_sku": "P2v3", "dtu_max": 100 }, + "Enterprise": { "app_sku": "P3v3", "dtu_max": 200 } + } +} +``` + +`dtu_max` is the per-DB DTU cap inside the tier's Elastic Pool. Terraform computes the pool's eDTU capacity from this cap (smallest valid Standard pool size that can hold a DB at the cap). + +Add a tier by adding a key here. Both the validator and Terraform will pick it up automatically — but to make a new tier queueable from the pipeline UI you also need to add a matching `run{Name}` boolean parameter in `azure-pipeline.yml` and a corresponding `if eq(parameters.run{Name}, true)` block in the tier-expansion list. + +A pipeline run only provisions plans + pools for tiers actually referenced by its resolved test cases — an all-Standard run creates one App Service Plan + one SQL server + one Elastic Pool; a mixed-tier run creates one per distinct tier in use. + +**SKU choice — why each tier gets a dedicated P-SKU.** Umbraco Cloud differentiates plans via dedicated App Service Plan SKUs (P0v3 / P1v3 / P2v3 / P3v3 across the four tiers), not via per-site quotas on a shared pool — so we provision one dedicated plan per tier in use with the same SKU progression. SQL side mirrors Cloud exactly: Standard-tier Elastic Pools with per-DB DTU caps of 20 / 50 / 100 / 200 DTUs for Starter / Standard / Pro / Enterprise. The two `*Override` queue-time parameters let an operator decouple app sizing from SQL sizing for bottleneck diagnosis (e.g. "P3v3 app + 20 DTU SQL" isolates the SQL contribution). + +## Scenarios + +A **scenario** is an Umbraco-side configuration variant. The layout mirrors how Umbraco's own [acceptance test repo](https://github.com/umbraco/Umbraco-CMS) organises tests — a folder per scenario, optionally with an `AdditionalSetup/appsettings.json` carrying the configuration overlay. A scenario with no Umbraco config to override (like `Default`) can either omit `AdditionalSetup/` entirely or ship an empty `{}`: + +``` +loadtests/scenarios/ + Default/ + AdditionalSetup/ + appsettings.json # {} — identity overlay + locustfile.py + scenario.yaml # description; no profile overrides + DeliveryApi/ # ships in the repo + AdditionalSetup/ + appsettings.json # enables Umbraco:CMS:DeliveryApi (PublicAccess on) + Program.cs # code overlay: registers .AddDeliveryApi() + locustfile.py + scenario.yaml + RedisCache/ # add when needed + AdditionalSetup/ + appsettings.json # Redis-specific Umbraco keys + locustfile.py + scenario.yaml # optional load profile overrides +``` + +The shipped scenarios are **`Default`** (traditional customer — rendered pages + media + write path) and **`DeliveryApi`** (headless customer — Content Delivery API + media + write path, no rendered-page traffic). Each scenario ships its own `locustfile.py` next to its `AdditionalSetup/` and declares its tasks explicitly, so the workload that runs for a given scenario is fully visible in that one file. Shared building blocks (inventory probe, Delivery API probe, `pick_url` helper) live in `loadtests/_helpers.py` and are imported by each scenario's locustfile. + +### Naming convention + +Name scenarios after **what they configure**, not what they test. Examples that fit the convention: `Default`, `RedisCache`, `LuceneDisabled`, `BackofficeOnly`. Examples that don't: `BulkPublishTest`, `PerfRun3` — those are tests, not configs. + +This matches Umbraco's pattern (`ContentSettingConfig`, `DeliveryApi`, `SMTP` …) and means future per-scenario test plans will live naturally inside the same folder. + +### Naming constraints + +Scenario names participate in Azure resource names (App Service is capped at 60 chars), so the validator enforces: + +- **≤ 15 characters** (e.g. `RedisCache` ✓, `BackofficeOnly` ✓, `ContentDeliveryApi` ✗) +- **alphanumeric + hyphens only** (no underscores, dots, spaces). Folder names are matched case-strictly on every agent (the validator enumerates the actual folders and rejects mismatches with a "did you mean 'X'?" hint). + +The `resource_name_prefix` Terraform variable is similarly capped at **16 chars** (validated). Default is `umbraco-loadtest`. The 60-char App Service budget breaks down as: + +``` +${prefix}-appservice-${umbraco}-${tier}-${scenario} + ≤16 12 ≤7 ≤8 ≤15 + connectors = 60 max +``` + +Long prerelease tags eat into the budget — see [Pitfalls › Name length](#name-length-long-umbraco-prereleases-break-the-60-char-app-service-cap). + +### Sampler naming + +A **sampler** is one operation within a scenario — a `@task` method in a Locust file, or an HTTP Request label in a JMeter `.jmx`. The workbook keys cells by `(scenario × umbraco_version × infra_tier × scenario_name)` and treats sampler names as opaque strings. Two consequences: + +- A renamed sampler spawns a new cell. Historical baseline for the old name doesn't transfer — the regression gate restarts at zero for the new name. +- Two scenarios that share a sampler name (e.g. `BackofficeV13` and `BackofficeV17` both with a `Login` sampler) line up in the Compare tab's per-sampler delta view. Different names for the same operation (`Login` vs `BackofficeLogin`) won't. + +Reserved character: sampler names **must not contain `__`** — that's the cell-key delimiter parsed by `check-regression.ps1` and `_history-helpers.ps1`. + +For cross-version test plans where the same operation is implemented against two different APIs (the obvious case is backoffice — v13's `/umbraco/backoffice/UmbracoApi/...` vs v17+'s `/umbraco/management/api/v1/...`), agreeing on sampler labels up front is the cheapest discipline. A suggested canonical set for authenticated / backoffice flows: + +| Sampler | Operation | +|---|---| +| `Login` | Authenticate (any flavour — session cookie, OAuth token, etc.) | +| `ContentList` | List or page through the content tree | +| `ContentRead` | Read one content item by id | +| `ContentSave` | Save (unpublished) one content item | +| `ContentPublish` | Publish one content item | +| `ContentUnpublish` | Unpublish one content item | +| `MediaList` | List media folders or items | +| `MediaUpload` | Upload one media file | +| `UserList` | List users (admin-only) | +| `Search` | Full-text search query | + +Front-end (anonymous) scenarios continue to use the existing convention from `Default` / `DeliveryApi` (`Homepage`, `Section`, `Category`, `Page`, `Detail`, `Media`, `ContactFormSubmit`, `DeliveryApiList`, `DeliveryApiItem`). + +Locust takes the sampler name from the method name (or the `name=` kwarg on `client.get/post`); JMeter takes it from the HTTP Request element's name. Same string lands in `LoadTestSummary_CL.scenario_name` either way — and from there into every workbook surface that filters or groups by sampler. + +### `appsettings.json` overlay + +The contents of a scenario's `AdditionalSetup/appsettings.json` are **flattened** by the validator to App Service envvar form (`Section:Sub:Key` → `Section__Sub__Key`) and **merged into the base `app_settings` block** of the deployed App Service. Overlay keys win over base keys. + +Example — `loadtests/scenarios/RedisCache/AdditionalSetup/appsettings.json`: + +```json +{ + "Umbraco": { + "CMS": { + "DistributedLockingMechanism": "RedisDistributedLockingMechanism" + } + }, + "ConnectionStrings": { + "Redis": { + "ConnectionString": "redis://..." + } + } +} +``` + +Becomes (in the App Service `app_settings`): + +``` +Umbraco__CMS__DistributedLockingMechanism = RedisDistributedLockingMechanism +ConnectionStrings__Redis__ConnectionString = redis://... +``` + +Overlay keys win over base keys — see [Pitfalls › Overlay precedence](#overlay-precedence-a-scenario-can-clobber-base-settings) for what that can clobber if you're not deliberate. + +### Code overlays (`*.cs`, `*.cshtml`, `App_Plugins/`, …) + +Some Umbraco features can't be flipped via `appsettings.json` alone — they need source-code changes (custom composers, builder-chain extensions, backoffice extensions). Mirroring how Umbraco's acceptance tests handle this, **any file in `AdditionalSetup/` other than `appsettings.json` is treated as a code overlay**: copied into the dotnet project tree before `dotnet build`, preserving relative paths. + +(The shipped `DeliveryApi` scenario is itself an example: it needs *both* an `appsettings.json` overlay to enable the feature *and* a `Program.cs` overlay calling `.AddDeliveryApi()` in the builder chain to register the API's DI services. Without the code overlay, hitting `/umbraco/delivery/api/v2/content` returns 500 with `Unable to resolve service for type 'IRequestSegmentService'`.) + +Another example — a hypothetical `CustomComposer` scenario that adds an event handler via composer: + +``` +loadtests/scenarios/CustomComposer/ + AdditionalSetup/ + appsettings.json # any related config + Composers/MyComposer.cs # custom Umbraco composer +``` + +When the install script deploys the scenario: + +1. `dotnet new umbraco -n …` creates the project (with a default `Program.cs`). +2. The seeder package is added (`dotnet add package …`). +3. **Code overlay is applied**: every non-`appsettings.json` file under `AdditionalSetup/` is copied to the same relative path under the project root (e.g. `AdditionalSetup/Composers/MyComposer.cs` → `/Composers/MyComposer.cs`, `AdditionalSetup/Program.cs` → `/Program.cs` overwriting the template-generated one). +4. `dotnet build` picks up the overlay automatically. + +Convention notes: +- Mirror the dotnet project structure inside `AdditionalSetup/`. A file at `AdditionalSetup/Composers/MyComposer.cs` lands at `/Composers/MyComposer.cs`. +- For Umbraco 14+ backoffice extensions, drop your built JS/TS into `AdditionalSetup/wwwroot/App_Plugins/{Name}/`. +- Scenarios with broken C# will fail `dotnet build` — the install script propagates the failure to the pipeline run. +- An empty `AdditionalSetup/` (or one containing only `appsettings.json`, like `Default`) just skips the overlay step. + +### `scenario.yaml` schema + +Optional load profile overrides: + +```yaml +description: "Free-text description (folder-level docs only; not read by code)" # optional +loadProfile: # optional whole block + users: 200 # overrides the profile's user count when present + spawnRate: 20 # overrides the profile's spawn rate when present + duration: 600 # overrides the profile's duration (s) when present +``` + +All fields optional. The validator reads only `loadProfile.*`; `description` is a folder-level docs note and isn't consumed anywhere. A missing `scenario.yaml` (or an empty `loadProfile` block) means the case uses the queue-time pipeline-level defaults. Override resolution happens once in the validator — every downstream consumer (Terraform, Azure Load Testing, NDJSON publisher) sees the resolved values, not the override logic. + +### Adding a new scenario + +1. If your scenario needs an Umbraco config overlay, create `loadtests/scenarios/{Name}/AdditionalSetup/appsettings.json` (and any code overlay files alongside, e.g. `Program.cs`). A scenario with no overlay either omits the folder or ships `AdditionalSetup/appsettings.json` with `{}` (the validator handles both). +2. Create `loadtests/scenarios/{Name}/locustfile.py` declaring the scenario's workload (import probes / `pick_url` from `_helpers.py` as needed). +3. Optionally add `loadtests/scenarios/{Name}/scenario.yaml` with description + load profile overrides. +4. Add `{Name}` to the `scenario` parameter's `values:` list in `azure-pipeline.yml` so it appears in the queue-time dropdown. + +The validator (`scripts/prepare-test-cases.ps1`) is the source-of-truth check — it enumerates the folders on every run, prints the available list at the top of its log, and rejects unknown names with a "did you mean?" hint. The dropdown is a queue-time discovery aid; no HCL or downstream-pipeline edits are needed. + +### Load Pattern + +Every test follows a **ramp-up → steady-state → ramp-down** shape so the measurement window reflects sustained behaviour, not arrival shock: + +- **Ramp-up** — Locust spawns VUs at the profile's `spawn rate` until all `users` are active (e.g. `standard` profile = 100 VUs at 10/s ≈ 10 s ramp). +- **Steady-state** — all VUs run their weighted task mix for the profile's `duration` (the metric window). +- **Ramp-down** — Azure Load Testing terminates VUs when the duration expires. + +When comparing runs, only the steady-state samples are meaningful — ramp-up/down samples skew tail latency and should be filtered out in any deeper analysis. + +### Workload distribution + +Each scenario's locustfile declares its `@task` methods explicitly, so the workload that runs for a given scenario is fully visible in that one file. The Default scenario (traditional content-browsing customer) uses **weighted Locust tasks** so virtual users don't all hammer the same flow — they distribute across the seeded site the way real traffic would. Current weights (sum to 113): + +| Task | Weight | Path pattern | +|---|---:|---| +| `homepage` | 5 | `/` | +| `section` | 10 | seeded section roots | +| `category` | 20 | seeded category pages | +| `page` | 30 | seeded content pages | +| `detail` | 35 | seeded detail/leaf pages | +| `media` | 5 | media URLs | +| `submit_contact_form` | 8 | `POST /umbraco/api/contactform/submit` (write path) | + +The non-homepage tasks are **inventory-driven**: at test start, locust calls `/umbraco/api/seederstatus/inventory` to discover the actual URLs the seeder generated, so the same test code works against any seeder preset without per-run config. Tasks raise (visible as a 100%-error task in the report) when a bucket is empty — no silent fallbacks, so a broken seeder or misconfigured scenario surfaces loudly instead of distorting the workload. Adjust weights in a scenario's own `locustfile.py` to change that scenario's traffic mix; the DeliveryApi scenario is structured the same way but exercises the Content Delivery API endpoints instead of rendered pages. + +### Deterministic URL selection + +`loadtests/_helpers.py` seeds Python's `random` at module import (fixed seed `42` by default), so `random.choice()` over the seeded URL inventory follows the same sequence across runs. Cell-to-cell variance from "this run happened to hit Detail-7 a lot, that run hit Detail-23" drops out — leaving infrastructure jitter as the dominant signal in run-to-run deltas (which is the comparison you actually want for regression checks). Set the `LOCUST_RANDOM_SEED` env var to a different integer if you specifically want randomised content selection (e.g. to validate that the harness ISN'T sensitive to URL choice). Caveat: each Locust engine/worker process re-seeds at import, so full request-by-request reproducibility isn't promised; the aggregate URL distribution per run is what stays stable. + +### Cold-cache vs warm-cache testing + +By default, the pipeline **warms up** the App Service (5-minute poll for `200` on `/`) before starting the load test, so measurements reflect steady-state cache-warm behaviour — the most stable comparison surface across tiers and versions. + +Set `skipWarmup: true` to skip the warmup. The load test then hits a freshly-started App Service with cold caches, measuring the full delivery pipeline including initial cache population — useful for understanding cache warm-up latency, restart behaviour, and the front-edge of a request burst against a cold app. + +## Results + +The pipeline writes results to four places: + +- **Azure Load Testing portal**: dashboard with client-side metrics (response time, throughput, errors) and server-side metrics (CPU, memory, network, disk). The Azure Load Testing resource lives in a **long-lived, shared resource group** (see "Infrastructure" below) so run history accumulates across pipeline runs. There's **one load test per scenario** (testId `umbraco-lt-{scenario}`), with every (version, tier) run nested under it — so the portal's "Compare runs" view lets you pick multiple runs and overlay their metrics natively. Each run is named `{umbracoVersion} {tier} {poolDtuMax}DTU #{buildId}` — the per-DB DTU cap is in the name so override runs (e.g. `Standard 100DTU`) are distinguishable from default-pairing runs (`Standard 50DTU`) in the portal's Compare view. +- **Pipeline artifacts**: per-case ZIP under `loadtest-results-{sanitised-testCaseId}` on the build, useful for forensic deep-dives. Expires with the pipeline's build retention policy. +- **History storage account** (long-lived): per-case NDJSON summary at `{scenario}/{major}/{umbracoVersion}/{tier}/{yyyy-MM-dd}_{buildId}/summary.ndjson` plus the raw artifact dump under `raw/`. Scenario is top-level because it defines what's *comparable* — different scenarios hit different endpoints / seed different data, so their numbers can't be compared directly. Within a scenario, prefix-listing maps to the natural pivots: `Default/17/` trends a major, `Default/17/17.0.0/` is all tiers in one build, `Default/17/*/Starter/` sweeps versions on one tier. Each row carries the full run metadata (commit, version, tier, scenario, SKUs, seeder preset, user count), so cross-run queries don't need joins. +- **Log Analytics workspace** (long-lived): the same NDJSON rows mirrored into the `LoadTestSummary_CL` custom table for KQL querying. The Workbook (see "Dashboard" below) reads from here. Blob storage remains source of truth — Log Analytics is a queryable mirror. + +NDJSON is ingestible directly by Azure Data Explorer, pandas, Postgres `COPY`, etc. — pick whatever query layer fits, the data shape stays the same. + +### Comparing two runs + +For the everyday "did this version/tier actually move the needle?" question, `scripts/compare-runs.ps1` pulls per-sampler aggregates straight out of history storage — no manual artifact download: + +```powershell +# Version vs version on a fixed tier +./scripts/compare-runs.ps1 ` + -Scenario Default -Tier Standard ` + -BaselineVersion 17.0.0 -CandidateVersion 17.0.1 ` + -StorageAccountName $env:HISTORY_STORAGE_ACCOUNT ` + -ContainerName loadtest-history ` + -OutputPath compare.md + +# Tier vs tier on a fixed version +./scripts/compare-runs.ps1 ` + -Scenario Default -Version 17.0.0 ` + -BaselineTier Starter -CandidateTier Pro ` + -StorageAccountName $env:HISTORY_STORAGE_ACCOUNT ` + -ContainerName loadtest-history +``` + +`-Aggregate latest` (default) compares the most recent run for each cell; `-Aggregate median5` compares the median across the last 5 runs (more stable on noisy tails). Auth: `az login` + permission to list keys on the history storage account (Storage Account Contributor or higher). + +The script emits a markdown report with: +- **Per-sampler** breakdown (Detail, Page, Category, etc.) ordered by traffic share, with deltas bolded when they cross the significance threshold (default 10%) +- Run-id list per side so you can trace which runs contributed +- A "How to read this" footer (p95/p99 are the tier-discriminating metrics; max is single-sample noise; cached paths won't move regardless of tier) + +History mode skips the aggregate-across-all-samplers row because true aggregate percentiles need raw request samples, which aren't preserved in the NDJSON summary. Per-sampler is the actionable view anyway. + +**CSV fallback** — when history storage isn't accessible (local Locust runs, offline analysis, or a one-off comparison of pipeline-artifact CSVs): + +```powershell +./scripts/compare-runs.ps1 ` + -BaselinePath ./starter-17.0.0/engine1_results.csv ` + -CandidatePath ./standard-17.0.0/engine1_results.csv ` + -BaselineLabel "Starter 17.0.0" ` + -CandidateLabel "Standard 17.0.0" +``` + +CSV mode reads raw request samples and computes true aggregate percentiles, so it produces an additional **Aggregate** row that history mode can't. + +### Trending across many runs + +`scripts/compare-runs.ps1` answers "A vs B"; for "show me everything we've run on this scenario", use `scripts/show-trends.ps1`. It reads every `summary.ndjson` under a scenario's history-storage prefix and prints a markdown matrix of (Umbraco version × tier) → p95/p99/error% per sampler. Single-run cells use the run as-is; cells with 2+ runs show the **median plus stddev** so you can see at a glance whether the numbers are stable enough to baseline against. + +The script authenticates via account-key — `az login` first, then it fetches the storage account key at runtime and uses `--account-key` for blob ops. Your user needs `Microsoft.Storage/storageAccounts/listKeys/action` on the history SA (Storage Account Contributor or higher). + +```powershell +./scripts/show-trends.ps1 ` + -Scenario Default -Major 17 ` + -HistoryResourceGroup umbraco-loadtest-history-rg ` + -StorageAccountName $env:HISTORY_STORAGE_ACCOUNT ` + -ContainerName loadtest-history ` + -OutputPath trends.md +``` + +Optional `-Sampler Detail` filters to a single Locust task (handy when the matrix gets long). Output layout — one table per sampler: + +``` +| Version | Starter | Standard | Pro | +|---------|-------------------------------|-----------------------------|-----------------------------| +| 17.0.0 | 450 ±18 / 1200 ±60 (0.1%) n=4 | 264 ±9 / 807 ±42 (0%) n=4 | 144 ±7 / 821 ±55 (0%) n=4 | +| 17.0.1 | 420 / 1100 (0%) | 250 / 780 (0%) | 140 / 800 (0%) | +``` + +Single-run cells are `p95 / p99 (err%)`; multi-run cells are `p95 ±stddev / p99 ±stddev (err%) n=K`. A small ±stddev across n≥3 runs is the green light to use those numbers as a regression baseline — see the next section. + +### Establishing a baseline + +Before turning on regression gating, you need to know what "stable" looks like. The protocol: + +1. **Queue the same configuration 3× back-to-back.** Same Umbraco version, same scenario, same tiers, same profile, same SQL SKU. Don't change parameters between runs. +2. **Run `show-trends.ps1` and read the ±stddev.** A useful rule of thumb: if `stddev / median < 5%` on p95 across the 3 runs, the cell is stable enough to gate on. If it's wider, run a 4th and 5th — sometimes the first run is colder than the rest. If it's still wide after 5, the workload itself is too noisy and the gate would false-positive in CI. +3. **Once you have ≥3 stable runs in the bank**, `check-regression.ps1` will start scoring new runs against them automatically (it uses the most recent N prior runs as the baseline window). + +Baselines decay — after a major Umbraco release, a tier-SKU shift, or a meaningful seeder/scenario change, the prior baseline numbers no longer reflect "what stable looks like" and you should re-baseline that cell. The script doesn't enforce this automatically; it's a judgment call on what counts as a "meaningful" change. + +### Regression gating + +`scripts/check-regression.ps1` reads the same NDJSON history, takes the latest run for each (version × tier × sampler) cell as the candidate, and compares it to the median of the previous N runs (default: last 5, minimum 3). Cells that exceed any threshold are flagged as regressions; the script exits non-zero so it can run as a pipeline gate. + +```powershell +./scripts/check-regression.ps1 ` + -Scenario Default -Major 17 ` + -HistoryResourceGroup umbraco-loadtest-history-rg ` + -StorageAccountName $env:HISTORY_STORAGE_ACCOUNT ` + -ContainerName loadtest-history ` + -OutputPath regression-report.md +``` + +(Same auth requirement as `show-trends.ps1` — `az login` + Storage Account Contributor or higher on the history SA.) + +Defaults: + +| Threshold | Default | What it means | +|---|---|---| +| `-P95Threshold` | 10% | Latest p95 > baseline-median p95 × 1.10 → regression | +| `-P99Threshold` | 15% | Latest p99 > baseline-median p99 × 1.15 (wider band — tail latency is noisier) | +| `-ErrorAbsoluteThreshold` | 0.5pp | Latest error_rate is more than 0.5 percentage points above baseline median | +| `-MinBaselineRuns` | 3 | Cell needs at least this many prior runs to be gateable | +| `-BaselineWindow` | 5 | Cap on how many recent prior runs feed the baseline (keeps it sensitive to recent state) | + +Cells with fewer than `-MinBaselineRuns` prior runs are reported as "insufficient baseline" and **never** trigger a fail — you can't regress against nothing. This means turning the gate on doesn't break the first few runs of a brand-new scenario or tier; the gate activates per-cell as baselines accrue. + +Pass `-NoFailOnRegression` to render the report without failing (useful for "show me what would break if I turned this on"). + +The script is wired into the pipeline as the `regressionCheck` job after `runLoadTests`. It's permissive by default (cells with < `MinBaselineRuns` prior runs report "insufficient baseline" and exit 0), so it's safe to leave on from day one — the gate activates per-cell as baselines accrue. + +### Infrastructure: ephemeral vs long-lived + +The pipeline manages two separate resource groups: + +| Resource group | Lifetime | Contents | +|---|---|---| +| `${prefix}-rg` (ephemeral) | Created and destroyed per pipeline run | App Service plans (one per used tier), App Services + SQL servers + databases (one per case) | +| `umbraco-loadtest-history-rg` (long-lived) | Created once, never deleted by the pipeline | Shared Azure Load Testing resource, storage account for results history, Log Analytics workspace + custom table + DCR/DCE for the Workbook, the Workbook itself | + +The long-lived RG is provisioned idempotently at the start of every pipeline run by `scripts/ensure-history-infra.ps1` (storage / ALT) and `scripts/ensure-monitoring-infra.ps1` (Log Analytics / DCR / Workbook role grant) — first run creates, subsequent runs no-op. Override the names via the `historyResourceGroup`, `historyLoadTestName`, `historyStorageAccount`, `historyContainer`, `historyWorkspaceName`, `historyDceName`, `historyDcrName` pipeline variables (or pin them in a variable group) if multiple teams share the subscription. **The storage account name must be globally unique and 3-24 lowercase alphanumeric chars.** + +## Pipeline Workflow + +The pipeline runs in seven stages. Stage boundaries are visible in the AzDO run summary so failures isolate cleanly: a failed `provision` stage tells you Terraform broke; a failed `loadTest` stage tells you the test itself broke. The `cleanup` stage runs on `always()` so the ephemeral RG gets torn down (or offered for manual keep) on every outcome — including pipeline cancellation. + +``` +validateTestCases Validate testCases JSON, read scenario folders, resolve load profile + (smoke / standard / stress) into seederPreset + engineInstances + VUs. + +ensureHistoryInfra Idempotent: shared Azure Load Testing resource + storage + container. + First run creates; subsequent runs no-op. + +ensureMonitoringInfra Idempotent: Log Analytics workspace + custom table + DCR/DCE + Workbook. + Exposes the Logs Ingestion target as cross-stage variables. + +provision checkResourceGroup → setup (init + validate + plan) → apply. + Provisions one App Service Plan per used tier, plus per-case App Services + and SQL DBs. Emits test_case_outputs map. + +loadTest runLoadTests: each case warms up, runs Locust on ALT, publishes results + to history storage, the build artifact, and Log Analytics. + +regression Compare candidate run against baseline-median; fail the pipeline when a + cell exceeds threshold AND has ≥3 prior runs. + +cleanup checkResourceGroupForCleanup → manualValidation (configurable window) → + deleteResourceGroup if rejected/cancelled/expired. Always runs. +``` + +## Data Seeder Presets + +| Preset | Documents | Media | Members | Approx. Time | +|--------|-----------|-------|---------|--------------| +| Small | ~100 | ~50 | ~20 | 2-5 min | +| Medium | ~500 | ~200 | ~100 | 5-15 min | +| Large | ~2000 | ~500 | ~500 | 15-30 min | +| Massive | ~10000 | ~2000 | ~2000 | 30-60 min | + +The seeder preset is **run-level** — applied uniformly to every case. (A scenario *can* override it via the `appsettings.json` overlay key `Umbraco.Cms.TestDataSeeder__Options__Preset` if you really need it per-case — see [Pitfalls › Overlay precedence](#overlay-precedence-a-scenario-can-clobber-base-settings).) + +## Usage + +### Running via Azure Pipelines + +1. Run the pipeline manually from Azure DevOps. +2. Pick the **load profile** (`smoke` / `standard` / `stress`), **Umbraco version** (free text — prereleases ok), and **scenario** (defaults to `Default`). +3. Tick the **tiers** to run against (`runStarter` / `runStandard` / `runPro` / `runEnterprise` — at least one). Defaults to Starter only. +4. Adjust the orthogonal knobs (region, prefix, cold start, skip load tests, validation window) only if you need to. +5. Wait for validation → ensure-history-infra → ensure-monitoring-infra → provisioning → load tests → regression check to complete. +6. Review results in Azure Load Testing portal, pipeline artifacts, and history storage NDJSON. The `regression-report` artifact has the post-run regression check output. +7. Approve or reject resource cleanup within the validation window (default 60 min). + +### Smoke-testing changes + +When iterating on scripts or Terraform, the full pipeline (~20-30 min) is too slow a feedback loop. Run a **profile-only smoke**: `loadProfile=smoke`, `runStarter=true` (everything else default). Full stack runs (provision + build + seed + 60s load test + publish + regression check) in ~12-15 min. Exercises the full ephemeral-infra cycle and the load-test path on the smallest seeder preset. + +### Running Terraform Locally + +```bash +cd Terraform +terraform init +cp terraform.tfvars.example terraform.tfvars +# Edit terraform.tfvars — note the testCaseId-keyed map shape +terraform plan +terraform apply +``` + +### Running Locust Locally + +```bash +cd loadtests +pip install locust +locust -f scenarios/Default/locustfile.py --host https:// +# Or run the DeliveryApi scenario: +# locust -f scenarios/DeliveryApi/locustfile.py --host https:// +# Open http://localhost:8089 to configure and start the test +``` + +## Key metrics + +Every per-case NDJSON row carries the metrics below (one row per Locust task, plus aggregate fields visible in the ALT portal). Use percentiles over averages — averages hide spikes. + +**Client-side (from Locust / `engine1_results.csv` → NDJSON):** +- `request_count`, `failure_count`, `error_rate` — failure rate is the first thing to check; a fast-but-erroring run is not a successful run. +- `avg_ms`, `p50_ms`, `p90_ms`, `p95_ms`, `p99_ms`, `min_ms`, `max_ms` — `p95` and `p99` are the tier-discriminating metrics. `max` is single-sample noise. +- `requests_per_sec` — throughput per task and aggregate. + +**Server-side (Azure Load Testing portal, server metrics tab):** +- App Service: CPU %, memory %, network in/out, disk queue. +- SQL: DTU %, CPU %, log IO %, sessions. + +Server-side ties directly to infrastructure sizing — when a tier saturates CPU or DTU at the profile's load, that's the ceiling of that SKU pair. + +The publish step queries Azure Monitor for these metrics over the load-test window and injects mean/max into the NDJSON metadata (`plan_CpuPercentage_avg`, `sql_dtu_consumption_percent_max`, etc.) — so they appear alongside latency in `show-trends.ps1` / `check-regression.ps1` output without a separate query. + +## Getting your first comparison + +After "First-time setup" is done, the path from "infra works" to "I detected a regression" is: + +**Step 1 — Establish a baseline.** Queue the same configuration 3× back-to-back. Pick the one you actually care about; don't try to baseline everything at once. + +``` +Pipeline parameters for the 3 baseline runs: + umbracoVersion: 17.0.0 + scenario: Default + runStarter: true (others false — start narrow) + loadProfile: standard + poolDtuOverride: Auto + skipWarmup: false +``` + +Wait for all 3 to finish, then approve cleanup on each. + +**Step 2 — Check stability.** Run `show-trends.ps1` and look at the `±stddev` on p95. + +```powershell +./scripts/show-trends.ps1 ` + -Scenario Default -Major 17 ` + -StorageAccountName $env:HISTORY_STORAGE_ACCOUNT ` + -ContainerName loadtest-history +``` + +Cells with `n=3` and `stddev / median < 5%` on p95 are stable enough to baseline. If a cell is wider than that, queue 2 more runs and re-check; the first run is sometimes cold relative to the others. If it's still wide after 5, the workload itself is too noisy for that cell to be a useful baseline target. + +**Step 3 — Queue a candidate.** Once stable, queue the run you actually want to evaluate — typically a different `umbracoVersion`, or the same version with a `poolDtuOverride` to test SQL-tier effects. + +``` +Pipeline parameters for the candidate: + umbracoVersion: 17.0.1 (← the change being evaluated) + scenario: Default + runStarter: true + loadProfile: standard + ... +``` + +**Step 4 — The pipeline auto-checks.** The `regressionCheck` job at the end of the run compares the candidate's per-sampler stats against the median of the prior 5 runs in that cell. The pipeline fails if any cell exceeds the thresholds (default p95 +10%, p99 +15%, error_rate +0.5pp). The full breakdown is in the `regression-report` build artifact. + +**Step 5 — Spot-check manually.** For deeper investigation (or comparing two specific runs out-of-band), use `compare-runs.ps1` in history mode: + +```powershell +./scripts/compare-runs.ps1 ` + -Scenario Default -Tier Starter ` + -BaselineVersion 17.0.0 -CandidateVersion 17.0.1 ` + -StorageAccountName $env:HISTORY_STORAGE_ACCOUNT ` + -ContainerName loadtest-history ` + -Aggregate median5 -OutputPath compare.md +``` + +`-Aggregate median5` compares medians across 5 runs each side — the right call when you have stable baselines and want to filter out single-run noise. + +**Step 6 — Interpret deltas.** The report bolds cells crossing the significance threshold (default 10%). Skim the bold cells. Common patterns: + +- **All read paths regressed by ~10-30%** — likely a code change in the content/render hot path. +- **One sampler regressed but others didn't** — likely a code change scoped to that endpoint. +- **p99 regressed but p95 didn't** — usually a tail-latency issue (GC, lock contention, transient SQL slowness). p99 is noisier; rule out before raising alarm. +- **All samplers up uniformly + SQL `dtu_consumption_percent_max` near 100%** — SQL DTU saturation, not code. Try `poolDtuOverride=100` (or 200) and re-run. +- **All samplers up + `plan_CpuPercentage_max` near 100%** — App Service saturation. Same diagnostic question, different lever (App Service tier). + +The `regression-report` artifact + the per-sampler table from `compare-runs.ps1` together usually tell you whether to **investigate the code** or **revisit the infra sizing**. + +## Dashboard + +`dashboards/loadtest.workbook.json` is an Azure Workbook that queries `LoadTestSummary_CL` in Log Analytics and offers six views: + +- **Trends** — chronological per-run chart of the chosen metric (p95 / p99 / avg / error rate / RPS / server CPU peak / DB load peak), one line per `(scenario × version × tier)`; side-by-side latency + resource-pressure charts share a run-indexed x-axis for direct visual correlation of code-bound vs infra-bound symptoms; matrix table below with median ±stddev and a plain-language **Stability** label per cell (*stable / moderate / noisy / few runs*) flagging cells where a small regression threshold would be lost in run-to-run noise. Sampler filter is multi-select. +- **Tiers** — pick scenario + version, see latest run per tier as a bar chart + a Capacity-verdict table with **Headroom** and a **Bottleneck** column naming the hottest resource and its peak (e.g. `Database load 92%`). Tier rows sort by capacity rank (Starter → Standard → Pro → Enterprise). Answers "what do I get for upgrading the tier — and what saturated first?" +- **Versions** — pick scenario + tier, see per-sampler median latency grouped by Umbraco version. Answers "did this version regress on this tier?" +- **Compare** — pick two runs + Δ% threshold; per-sampler delta table with red/green conditional formatting; server-side delta block with a **Note** column distinguishing "no change" from "no data". Failed runs (`no_metrics`) are excluded from the run pickers. +- **Runs** — filtered run list with **Bottleneck** + regression-verdict columns; pick a run from the drill dropdown to see the regression breakdown (which specific samplers flagged), per-sampler latency detail, **and per-minute resource-pressure charts** (% metrics and HTTP error counts on separate axes) sourced from a companion `LoadTestSeries_CL` table — answers "when *in* the run did p99 spike?" / "did SQL DTU saturate before App CPU?" — the questions the summary scalars can't. +- **Glossary** — vocabulary reference for every column / verdict / metric used in the other tabs. + +Global filter bar (Workspace, time range, Scenario / Version / Tier dropdowns) scopes Trends / Compare / Runs. Tiers and Versions have their own scoped pickers (Tiers is the cross-tier view, Versions is the cross-version view). Workbook URLs encode the filter state, so links are shareable. + +Auth piggybacks on Azure RBAC: anyone with **Reader** on the Log Analytics workspace (or its parent RG) can view the Workbook. No separate identity to manage. + +> **Single-team scope today.** The Workbook GUID, the `LoadTestSummary_CL` table, and the workspace itself are shared resources with no per-team partitioning. If multiple teams ever fork this and target the same subscription, they'll need to override `historyWorkspaceName` / `historyDceName` / `historyDcrName` (already supported) **and** pass a fork-specific `-WorkbookId` to `deploy-workbook.ps1` to avoid clobbering each other's dashboard customisations. Row-level filters per team aren't implemented — anyone with workspace Reader sees every team's data. + +### Setup + +Nothing to run manually. The pipeline's `ensureMonitoringInfra` stage runs every time and idempotently provisions: + +1. **Log Analytics workspace** (`historyWorkspaceName`, default `umbraco-loadtest-laws`) +2. **Custom tables**: + - `LoadTestSummary_CL` — one row per (run × sampler), plus regression-check + metadata-only marker rows. The Workbook's primary table. + - `LoadTestSeries_CL` — one row per (run × metric × minute), populated from the same Azure Monitor data that `LoadTestSummary_CL`'s `*_avg / *_max` scalars come from but kept un-aggregated. Powers the per-run drill-down chart. +3. **Data Collection Endpoint + Rule** for the Logs Ingestion API (a single DCR carries both stream declarations) +4. **Monitoring Metrics Publisher** role on the DCR for the pipeline service principal (resolved automatically from the service connection — no manual SP-ID lookup) +5. **The Workbook itself**, re-applied from `dashboards/loadtest.workbook.json` so changes to the file ship to Azure on the next pipeline run + +After the first pipeline run, the printed Workbook URL is in the `ensureMonitoringInfra` stage log (look for "Workbook deployed" — the URL line below it). Pin the Workbook to your Azure portal dashboard for one-click access. First-time data takes ~5–10 minutes to surface in a brand-new custom table. + +If you want to override the resource names (e.g. multiple teams sharing one subscription), set `historyWorkspaceName`, `historyDceName`, `historyDcrName` in the `umbraco-loadtest-history` variable group. + +### Manual deploy (rare) + +You can run either script manually if you need to provision monitoring outside a pipeline run, or push a Workbook tweak without queueing the full pipeline: + +```powershell +./scripts/ensure-monitoring-infra.ps1 ` + -HistoryResourceGroup umbraco-loadtest-history-rg ` + -HistoryLocation "West Europe" ` + -WorkspaceName umbraco-loadtest-laws ` + -DceName umbraco-loadtest-dce ` + -DcrName umbraco-loadtest-dcr ` + -IngestPrincipalId (az ad sp show --id --query id -o tsv) + +./scripts/deploy-workbook.ps1 ` + -HistoryResourceGroup umbraco-loadtest-history-rg ` + -HistoryLocation "West Europe" ` + -WorkspaceName umbraco-loadtest-laws +``` + +### Maintenance + +**Prereqs both scripts assume**: `az` CLI logged in (`az login`), pwsh 7.3+. + +**Iterating on the Workbook** — edit `dashboards/loadtest.workbook.json` directly, or edit in the portal's Advanced Editor and paste the result back. Re-run `deploy-workbook.ps1` to push. The deploy uses a stable GUID (`-WorkbookId` parameter) so re-runs update in place. + +**Schema changes** — if you add a field in `publish-load-test-results.ps1`, mirror it in the `$columns` array in `ensure-monitoring-infra.ps1` and re-run that script. The DCR PUT is an in-place schema update; existing data is preserved. Fields without a matching column are dropped at ingestion (no failure). + +**Backfilling old runs** — the Workbook only sees what's been ingested into Log Analytics. Anything in blob storage from before the monitoring infra existed (or any run whose original publish step succeeded the blob upload but failed the Logs Ingestion call) is invisible to the Workbook. Replay it with: + +```powershell +./scripts/backfill-monitoring.ps1 ` + -HistoryResourceGroup umbraco-loadtest-history-rg ` + -StorageAccountName ` + -ContainerName loadtest-history ` + -WorkspaceName umbraco-loadtest-laws ` + -DceName umbraco-loadtest-dce ` + -DcrName umbraco-loadtest-dcr +``` + +Idempotent by default — queries existing `run_id`s in the table and skips blobs whose run is already there. `-Force` re-ingests everything (creates duplicates; use only after a teardown). + +**Access control** — grant `Log Analytics Reader` (or any role that includes read on the workspace) to anyone who needs to view the Workbook. Revoke by removing the role assignment. + +**Teardown** — remove the Workbook + monitoring without affecting load-test history: + +```powershell +az resource delete --ids "/subscriptions//resourceGroups/umbraco-loadtest-history-rg/providers/Microsoft.Insights/workbooks/" +az monitor data-collection rule delete -n umbraco-loadtest-dcr -g umbraco-loadtest-history-rg +az monitor data-collection endpoint delete -n umbraco-loadtest-dce -g umbraco-loadtest-history-rg +az monitor log-analytics workspace delete -n umbraco-loadtest-laws -g umbraco-loadtest-history-rg --yes +``` + +The history storage account, container, and ALT all stay untouched. + +### Troubleshooting + +- **Workbook loads but tables/charts are empty.** Check the Log Analytics workspace directly: `LoadTestSummary_CL | take 50`. If empty, `publish-load-test-results.ps1` isn't reaching the Logs Ingestion API — check pipeline log for the "Posting N row(s) to Log Analytics" line. Most common cause: the SP doesn't have Monitoring Metrics Publisher on the DCR (re-run `ensure-monitoring-infra.ps1` to repair). +- **First-ever ingest after table creation appears to do nothing.** New custom tables take 5–10 minutes for ingestion to surface. Wait, then re-query. +- **Permission denied opening the Workbook.** Grant `Log Analytics Reader` (or Contributor on the workspace) to the user. +- **`deploy-workbook.ps1` fails with "Resource not found" on the workspace.** Run `ensure-monitoring-infra.ps1` first; the deploy script needs the workspace to set its `sourceId`. + +## Roadmap + +Status of in-progress and not-yet-started work. + +### Planned scenarios + +The pipeline currently ships with the `Default` and `DeliveryApi` scenarios. The following are planned baseline coverage; each will live as its own folder under `loadtests/scenarios/` with an `appsettings.json` overlay (and code overlay where needed) plus a scenario-specific `locustfile.py` that imports from `loadtests/_helpers.py`. + +**Front-end user journeys** +- **Homepage and navigation flow** — land on the homepage, navigate menus, browse pages. +- **Content listing with pagination** — step through paginated listing pages that query and render multiple content items. +- **Media-heavy pages** — request pages with multiple images and media items. +- **Member registration and login** — register, log in/out, navigate authenticated member areas. (Blocked on the seeder package adding member front-end views — currently it creates members but not the login/register MVC views.) + +**Backoffice operations** +- **Save and publish** — modify a content node and publish. +- **Save complex document type** — save a doctype with many properties + many related content items. +- **Content tree browsing** — navigate and expand the content tree at scale. +- **Bulk operations** — publish or unpublish many nodes in batch. +- **Multiple backoffice users** — concurrent editorial operations. + +**Mixed traffic** +- **Frontend browsing + concurrent backoffice editing** — front-end load with simultaneous editorial save/publish. CMS workloads see this combination in production and it can shift cache invalidation patterns and DB contention in ways neither pure-frontend nor pure-backoffice tests will surface. + +### Surface server-side metrics in `show-trends` / `check-regression` + +Server-side metrics (`plan_CpuPercentage_avg`, `sql_dtu_consumption_percent_max`, etc.) are now captured into NDJSON metadata by `publish-load-test-results.ps1` over the load-test window. They appear alongside latency for any consumer that queries the NDJSON directly. Still pending: `show-trends.ps1` and `check-regression.ps1` only render the latency fields today. Extending them to also surface server-side cells (e.g. a "DTU saturation" matrix alongside p95) and to gate on saturation thresholds (e.g. "fail when SQL DTU max > 95%") is the natural next iteration. Per-tier saturation thresholds become first-class regression conditions when this lands. + +### Deeper resource-exhaustion monitoring + +Beyond the metrics above, the harder-to-see failure modes need richer instrumentation: + +- **App / process** — thread pool exhaustion, request queue length. +- **Network** — sockets in TIME_WAIT, HTTP.sys queue length. +- **Disk / filesystem** — IO queue depth, throughput. + +These need Azure Monitor diagnostic settings on the App Service or the App Service Diagnostics extension; not wired up yet. + +### Thresholds + +`check-regression.ps1` is wired into the pipeline as a post-load-test job (`regressionCheck`). It runs after every load-test run and either passes (no regressions or insufficient baseline) or fails the pipeline (real regression detected). The mechanism is **always on** but starts permissive: each (scenario × version × tier × sampler) cell needs ≥ 3 prior runs before it gates — until then, the cell is reported as "insufficient baseline" and contributes no fails. As baselines accrue per the "Establishing a baseline" protocol, the gate activates per-cell automatically. The post-run `regression-report` build artifact captures the full per-cell breakdown. + +## Pitfalls + +Things that have caught people out at least once. Skim before queueing your first non-default run. + +### Approved-kept RGs are not auto-cleaned + +The cleanup behaviour: + +- **Reject the manual validation** (or let it time out, default 60 min) → the ephemeral RG is deleted automatically. +- **Cancel the pipeline run** → still deletes, because `cleanup` runs `condition: always()`. +- **Approve (Resume) the manual validation to keep resources for inspection** → the RG is **not** deleted, ever, by the pipeline. You have to `az group delete -n ${prefix}-rg --yes` yourself when you're done. + +So when you click Resume, write yourself a reminder. There's no scheduled reaper, no auto-expiry beyond the validation window, and a forgotten approved-kept RG persists until you (or the next pipeline run, which will fail with "resource group already exists" and surface the orphan) catches it. + +### Security: hardcoded backoffice creds on a public-internet App Service + +The Terraform unattended-install config bakes a known admin login (`loadtest@example.invalid` / `LoadTest123!`) into every App Service so any team member can poke around in the backoffice. The App Services are public-internet by default (no IP allowlist) and the hostname is predictable from the test case ID. The risk window is the lifetime of the ephemeral RG — keep `validationTimeoutMinutes` to the minimum you actually need, and prefer rejecting cleanup explicitly when you're done. + +This is fine for ephemeral load-test environments with no real data; it would not be fine for anything else. If you fork this for a workload that handles real data, replace the hardcoded creds with a per-run random password and add an IP allowlist (or vnet integration). + +### Overlay precedence: a scenario can clobber base settings + +A scenario's `AdditionalSetup/appsettings.json` is **merged into the base App Service `app_settings` with overlay keys winning on collision**. That's deliberate flexibility, but a sufficiently aggressive overlay can stomp on `Umbraco__CMS__Unattended__*` (breaks unattended install) or `Umbraco.Cms.TestDataSeeder__Options__Preset` (overrides the run-level seeder preset). Be deliberate about what your overlay touches. + +### Name length: long Umbraco prereleases break the 60-char App Service cap + +Azure App Service names are capped at 60 chars. The computed name is `${prefix}-appservice-${umbraco}-${tier}-${scenario}` (≤16 + 12 + ≤length(version) + ≤length(tier) + ≤15 + connectors). Long Umbraco prerelease tags (`17.0.0-rc.1.beta.2`) eat into the budget. Terraform fails the run early via a `lifecycle.precondition` with a clear error message, but you'd rather not get there — prefer release versions (`X.Y.Z`) and shorten scenario names if running prereleases on a long-named tier. + +### Capacity: Massive preset is slow + +The seeder timeout for Massive is **120 minutes per case** (10s polling × 720 attempts) — the worst-case ceiling, not the typical seed time (30-60 min per the Data Seeder Presets table). A full 4-tier Massive run can take ~6-8 hours in the apply stage alone before any load testing happens. The Terraform Apply task has a 720-minute budget, so this fits, but you're using most of it. + +Practical guidance: use Massive when you specifically need the data volume. For most baselining and comparison work, Medium (the `standard` profile) is the right grain — and runs in a fraction of the time. + +## Troubleshooting + +### Common Issues + +**Preflight fails with "tier 'X' is not in tiers.json"** +- Check `loadtests/tiers.json` — the tier catalog. Either fix the typo in your `testCases` entry or add the tier to the catalog. + +**Preflight fails with "scenario folder not found"** +- The scenario folder must exist at `loadtests/scenarios/{Name}/`. Folder lookup is case-strict on every agent — the validator will suggest the closest match if the casing differs. `AdditionalSetup/appsettings.json` is optional (only needed for scenarios with an Umbraco config overlay). + +**Preflight fails with "duplicate testCaseId"** +- You have two cases with the same `(umbraco, tier, scenario)` triple. Either remove the duplicate, or change one of the dimensions (e.g. different scenario folder). + +**Seeder not completing** +- Large presets can take up to 60 minutes +- Check seeder status: `https:///umbraco/api/seederstatus/status` + +**Template version mismatch** +- Ensure Umbraco template version matches CMS version +- Pre-release versions require NuGet sources (automatically configured) + +## Local checks before queueing + +```bash +cd Terraform && terraform fmt -check -recursive && terraform init -backend=false && terraform validate +Invoke-ScriptAnalyzer -Path . -Recurse -Severity Warning,Error -ExcludeRule PSAvoidUsingWriteHost +git ls-files 'loadtests/**/locustfile.py' 'loadtests/_helpers.py' | ForEach-Object { python -m py_compile $_ } +yamllint -d "{rules: {document-start: disable, line-length: disable, truthy: disable}}" loadtests/scenarios/ +git ls-files '*.json' | ForEach-Object { Get-Content -LiteralPath $_ -Raw | ConvertFrom-Json | Out-Null } +``` + +Running these locally catches the common typos (trailing commas in the Workbook JSON, unescaped `$` in PowerShell, indentation in scenario yaml) without burning a pipeline run. + +## Azure resource tagging + +Every provisioned resource carries: + +| Tag | Where | Value | +|---|---|---| +| `project` | All | `umbraco-loadtest` | +| `managed_by` | Ephemeral resources | `terraform` | +| `managed_by` | Long-lived history infra | `ensure-script` | +| `build_id` | Ephemeral resources | `$(Build.BuildId)` from the pipeline (or `local` for hand runs) | +| `tier` | App Service Plan | The tier name (`Starter` / `Standard` / `Pro` / `Enterprise`) | +| `test_case_id` | App Service, SQL Server, SQL DB | The full testCaseId | +| `umbraco_version` | App Service, SQL Server, SQL DB | The Umbraco CMS version | +| `scenario` | App Service, SQL Server, SQL DB | The scenario folder name | + +Azure Portal can group/filter resources by any of these tags — `managed_by` separates per-run ephemeral resources from the long-lived history infra. + +Pre-existing untagged history infra (created before this change) won't be retroactively tagged. Either re-tag manually (`az group update -n umbraco-loadtest-history-rg --set tags.project=umbraco-loadtest tags.managed_by=ensure-script`) or recreate the RG. + diff --git a/Terraform/.terraform.lock.hcl b/Terraform/.terraform.lock.hcl new file mode 100644 index 0000000..aa38b4d --- /dev/null +++ b/Terraform/.terraform.lock.hcl @@ -0,0 +1,60 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.70.0" + constraints = "~> 4.20" + hashes = [ + "h1:Z+ZRyZiyV3F5CsVv4OFhrdkw4TysL6wwEjX0ckZmNlY=", + "zh:0de702004cc5f1e4e00a723f9fdf2effa4d12f05e73d4b47a1d3b977f12ed5aa", + "zh:1681f026c4d802d57792d353ea97d5fa796728f854abb341c800d77b53b1e6df", + "zh:25f88ff759daf01310f092f06677189f5ff46b7ed0d44ea5c859ae1c1d09ba47", + "zh:5cf21c3cb9700d30e4f444f54f3051239a90bd0f66afb816d4404972022a6a65", + "zh:674e9289684754583b3b80c9cb65d054bd5f4556475ac50ffb7d7e3626f48f2f", + "zh:723b57d1fe6477f17e1b57663cdcc32993dcd18b7ff62352bd3c23a6df0070b1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:990bf38be76c64ced4b612702c96373f284fefb4d8079ba8b40b834b28509729", + "zh:b3b062136c309f533a63baaa9789231be477be61daf51c518eeb1262a1eac683", + "zh:cb1009c7c318c894e773292896a9abe26f674253f5b4ad02176b7cea05134540", + "zh:dfb304377037ca7b800bfd1eb7586fb50a6ca3438bfeeb4a177502c850e33464", + "zh:eed0ddae2e7d29a53494c3a87dc89bff442dc558b4ac23887f50e58b3e9b23e0", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.4" + hashes = [ + "h1:+Ag4hSb4qQjNtAS6gj2+gsGl7v0iB/Bif6zZZU8lXsw=", + "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", + "zh:7b9c7b16f118fbc2b05a983817b8ce2f86df125857966ad356353baf4bff5c0a", + "zh:85e33ab43e0e1726e5f97a874b8e24820b6565ff8076523cc2922ba671492991", + "zh:9d32ac3619cfc93eb3c4f423492a8e0f79db05fec58e449dee9b2d5873d5f69f", + "zh:9e15c3c9dd8e0d1e3731841d44c34571b6c97f5b95e8296a45318b94e5287a6e", + "zh:b4c2ab35d1b7696c30b64bf2c0f3a62329107bd1a9121ce70683dec58af19615", + "zh:c43723e8cc65bcdf5e0c92581dcbbdcbdcf18b8d2037406a5f2033b1e22de442", + "zh:ceb5495d9c31bfb299d246ab333f08c7fb0d67a4f82681fbf47f2a21c3e11ab5", + "zh:e171026b3659305c558d9804062762d168f50ba02b88b231d20ec99578a6233f", + "zh:ed0fe2acdb61330b01841fa790be00ec6beaac91d41f311fb8254f74eb6a711f", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.8.1" + hashes = [ + "h1:osH3aBqEARwOz3VBJKdpFKJJCNIdgRC6k8vPojkLmlY=", + "zh:08dd03b918c7b55713026037c5400c48af5b9f468f483463321bd18e17b907b4", + "zh:0eee654a5542dc1d41920bbf2419032d6f0d5625b03bd81339e5b33394a3e0ae", + "zh:229665ddf060aa0ed315597908483eee5b818a17d09b6417a0f52fd9405c4f57", + "zh:2469d2e48f28076254a2a3fc327f184914566d9e40c5780b8d96ebf7205f8bc0", + "zh:37d7eb334d9561f335e748280f5535a384a88675af9a9eac439d4cfd663bcb66", + "zh:741101426a2f2c52dee37122f0f4a2f2d6af6d852cb1db634480a86398fa3511", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a902473f08ef8df62cfe6116bd6c157070a93f66622384300de235a533e9d4a9", + "zh:b85c511a23e57a2147355932b3b6dce2a11e856b941165793a0c3d7578d94d05", + "zh:c5172226d18eaac95b1daac80172287b69d4ce32750c82ad77fa0768be4ea4b8", + "zh:dab4434dba34aad569b0bc243c2d3f3ff86dd7740def373f2a49816bd2ff819b", + "zh:f49fd62aa8c5525a5c17abd51e27ca5e213881d58882fd42fec4a545b53c9699", + ] +} diff --git a/Terraform/main.tf b/Terraform/main.tf index 760c7b2..f922973 100644 --- a/Terraform/main.tf +++ b/Terraform/main.tf @@ -1,27 +1,48 @@ terraform { required_providers { azurerm = { - source = "hashicorp/azurerm" - version = ">=4.20.0" + source = "hashicorp/azurerm" + version = "~> 4.20" } } required_version = ">= 1.3.9" + + # Local state is fine for the ephemeral RG-per-run model. Uncomment the backend below + # for shared state (pre-create the `tfstate` container in the history storage account). + # backend "azurerm" { + # resource_group_name = "umbraco-loadtest-history-rg" + # storage_account_name = "" + # container_name = "tfstate" + # key = "loadtest.tfstate" + # } } provider "azurerm" { features {} } +# Read tier catalog from loadtests/tiers.json. +locals { + tier_specs = jsondecode(file("${path.module}/../loadtests/tiers.json")).tiers +} + module "umbraco" { - source = "./modules/umbraco" + source = "./modules/umbraco" + resource_name_prefix = var.resource_name_prefix - resource_group_location = var.resource_group_location resource_group_name = var.resource_group_name - umbraco_cms_versions = var.umbraco_cms_versions - # Azure Login Credentials - client_id = var.client_id - client_secret = var.client_secret - tenant_id = var.tenant_id - app_service_plan_sku = var.app_service_plan_sku -} \ No newline at end of file + resource_group_location = var.resource_group_location + + tier_specs = local.tier_specs + test_cases = var.test_cases + seeder_preset = var.seeder_preset + pool_dtu_override = var.pool_dtu_override + app_sku_override = var.app_sku_override + + budget_alert_amount = var.budget_alert_amount + budget_alert_threshold_pct = var.budget_alert_threshold_pct + budget_alert_emails = var.budget_alert_emails + + build_id = var.build_id +} diff --git a/Terraform/modules/umbraco/main.tf b/Terraform/modules/umbraco/main.tf index ec04a8b..7914852 100644 --- a/Terraform/modules/umbraco/main.tf +++ b/Terraform/modules/umbraco/main.tf @@ -1,55 +1,191 @@ -# Resource group +locals { + tiers_in_use = toset([for v in var.test_cases : v.tier]) + + # Azure SQL admin login must start with a letter; prefix one since random_string can start with a digit. + sql_admin_login = "u${random_string.admin_login.result}" + + # Per-DB DTU cap per tier, with the queue-time override applied if set. + # The override lets a run size SQL independently of the app tier to isolate + # which side is the bottleneck. + tier_db_dtu = { + for t in local.tiers_in_use : + t => var.pool_dtu_override != 0 ? var.pool_dtu_override : var.tier_specs[t].dtu_max + } + + # Pool eDTU capacity: smallest valid Standard pool that can hold a DB at the + # tier cap. Standard pool minimum is 50; bump to 100 when the per-DB cap + # requires it. Beyond 100 the next valid Standard step is 200. + tier_pool_dtu = { + for t, db_dtu in local.tier_db_dtu : + t => db_dtu <= 50 ? 50 : db_dtu <= 100 ? 100 : 200 + } + + common_tags = { + project = "umbraco-loadtest" + managed_by = "terraform" + build_id = var.build_id + } +} + resource "azurerm_resource_group" "rg" { name = var.resource_group_name location = var.resource_group_location + tags = local.common_tags +} + +# Cost guard: monthly budget on the ephemeral RG that emails when MTD spend +# crosses the threshold percentage. Skipped entirely when budget_alert_emails +# is empty (the default) so it stays opt-in. +# +# Trade-offs: +# * Monthly grain is Azure's smallest budget window — there's no per-run / +# per-hour. A single run accruing $4 won't fire (well below 80% × $50); +# the alert protects against MULTIPLE forgotten gates compounding within +# a billing month, OR a runaway loop. +# * The budget resource itself is destroyed when terraform destroy runs at +# end-of-pipeline cleanup, so an "orphaned" RG that survives cleanup also +# loses its budget — at which point you'd want a subscription-scope budget +# instead. Out of scope here; this RG-scope budget covers the normal case. +resource "azurerm_consumption_budget_resource_group" "ephemeral" { + count = length(var.budget_alert_emails) > 0 ? 1 : 0 + name = "${var.resource_group_name}-budget" + resource_group_id = azurerm_resource_group.rg.id + + amount = var.budget_alert_amount + time_grain = "Monthly" + + # Start of current month, in UTC. formatdate keeps it stable across re-applies + # within the same month; a re-apply in a new month will roll the start date + # forward, which is the desired behavior for monthly grain. + # + # end_date is set explicitly ~10 years out. azurerm provider defaults this + # to start_date + 1 year, which silently disables the budget after 12 + # months from initial apply — easy to miss because Terraform doesn't drift- + # detect a "budget that no longer monitors anything". Pushing it ~decade + # out makes the time window long enough that re-applies (which always + # happen on each pipeline run) will refresh start_date well before + # end_date matters. + time_period { + start_date = formatdate("YYYY-MM-01'T'00:00:00'Z'", timestamp()) + end_date = formatdate("YYYY-MM-01'T'00:00:00'Z'", timeadd(timestamp(), "87600h")) + } + + notification { + enabled = true + threshold = var.budget_alert_threshold_pct + operator = "GreaterThan" + threshold_type = "Actual" + contact_emails = var.budget_alert_emails + } + + lifecycle { + # start_date drifts via timestamp() on every plan; ignoring its diff + # avoids spurious "in-place update" plans that don't change semantics. + ignore_changes = [time_period] + } } -# Random string for SQL server login resource "random_string" "admin_login" { - length = 16 - special = false - depends_on = [azurerm_resource_group.rg] + length = 15 + special = false } +# SQL admin password. Azure SQL needs 3 of 4 char categories; force upper+lower+numeric. resource "random_password" "admin_password" { - length = 16 - special = false - depends_on = [azurerm_resource_group.rg] + length = 24 + special = false + min_upper = 1 + min_lower = 1 + min_numeric = 1 } -# App Service Plan +# One plan per tier in use; same-tier cases share it (only one app hot at a time). +# Override lets a run size the app independently of the tier (paired with +# pool_dtu_override on the SQL side) to isolate app-vs-SQL bottlenecks. resource "azurerm_service_plan" "appserviceplan" { - name = "${var.resource_name_prefix}-appserviceplan" + for_each = local.tiers_in_use + name = "${var.resource_name_prefix}-asp-${lower(each.key)}" location = azurerm_resource_group.rg.location resource_group_name = azurerm_resource_group.rg.name os_type = "Windows" - - # We are using a variable for the sku version because we want to be able to change it if needed. - sku_name = var.app_service_plan_sku + sku_name = var.app_sku_override != "" ? var.app_sku_override : var.tier_specs[each.key].app_sku + tags = merge(local.common_tags, { tier = each.key }) +} + +# SQL server per tier - hosts the tier's Elastic Pool. One server per tier keeps +# the case-level resource graph identical between same-tier cases (they share +# server + pool) and matches the Cloud model where each plan has its own pool. +resource "azurerm_mssql_server" "sql_server" { + for_each = local.tiers_in_use + name = "${var.resource_name_prefix}-sql-${lower(each.key)}" + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + version = "12.0" + administrator_login = local.sql_admin_login + administrator_login_password = random_password.admin_password.result + minimum_tls_version = "1.2" + tags = merge(local.common_tags, { tier = each.key }) } -resource "azurerm_load_test" "load_test" { - location = var.resource_group_location - name = "${var.resource_name_prefix}-loadtest" - resource_group_name = var.resource_group_name +resource "azurerm_mssql_firewall_rule" "allow_azure_services" { + for_each = local.tiers_in_use + name = "AllowAzureServices-${lower(each.key)}" + server_id = azurerm_mssql_server.sql_server[each.key].id + start_ip_address = "0.0.0.0" + end_ip_address = "0.0.0.0" +} - depends_on = [azurerm_service_plan.appserviceplan] +# Standard-tier Elastic Pool per tier. Pool eDTU is the smallest valid Standard +# size that can hold a DB at the tier cap; per-DB max DTU equals the cap. +# max_size_gb=50 is the Standard-tier minimum; our test data is well under that, +# so the floor is enough and avoids paying for storage we won't use. +resource "azurerm_mssql_elasticpool" "pool" { + for_each = local.tiers_in_use + name = "${var.resource_name_prefix}-pool-${lower(each.key)}" + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + server_name = azurerm_mssql_server.sql_server[each.key].name + max_size_gb = 50 + tags = merge(local.common_tags, { tier = each.key }) + + sku { + name = "StandardPool" + tier = "Standard" + capacity = local.tier_pool_dtu[each.key] + } + + per_database_settings { + min_capacity = 0 + max_capacity = local.tier_db_dtu[each.key] + } } -# We create a module called versions, the reason for that is because we want to have multiple app services with different Umbraco Versions. +# Azure Load Testing resource lives in a long-lived RG (see scripts/ensure-history-infra.ps1). + module "versions" { - # We use a for_each so it creates a module for each version of Umbraco we have defined in variables. - for_each = var.umbraco_cms_versions - source = "./versions" + for_each = var.test_cases + source = "./versions" + resource_name_prefix = var.resource_name_prefix resource_group_name = azurerm_resource_group.rg.name resource_group_location = azurerm_resource_group.rg.location - service_plan_id = azurerm_service_plan.appserviceplan.id - dotnet_version = each.value["dotnet_version"] - umbraco_cms_version = each.value["umbraco_version"] - admin_login = random_string.admin_login.result - admin_password = random_password.admin_password.result - client_id = var.client_id - client_secret = var.client_secret - tenant_id = var.tenant_id -} \ No newline at end of file + service_plan_id = azurerm_service_plan.appserviceplan[each.value.tier].id + + test_case_id = each.key + dotnet_version = each.value.dotnet_version + umbraco_version = each.value.umbraco_version + scenario = each.value.scenario + app_settings_overlay = each.value.app_settings_overlay + + admin_login = local.sql_admin_login + admin_password = random_password.admin_password.result + + sql_server_name = azurerm_mssql_server.sql_server[each.value.tier].name + sql_server_id = azurerm_mssql_server.sql_server[each.value.tier].id + sql_server_fqdn = azurerm_mssql_server.sql_server[each.value.tier].fully_qualified_domain_name + elastic_pool_id = azurerm_mssql_elasticpool.pool[each.value.tier].id + sql_firewall_rule_dependency = azurerm_mssql_firewall_rule.allow_azure_services[each.value.tier].id + + seeder_preset = var.seeder_preset + common_tags = local.common_tags +} diff --git a/Terraform/modules/umbraco/output.tf b/Terraform/modules/umbraco/output.tf index c86ecc7..a37bfbd 100644 --- a/Terraform/modules/umbraco/output.tf +++ b/Terraform/modules/umbraco/output.tf @@ -1,27 +1,16 @@ -output "versions_output" { +output "test_case_outputs" { + description = "Per-case Terraform outputs, keyed by testCaseId. Only fields consumed by the pipeline are exposed." value = { - for version_list, module_versions in module.versions : - version_list => ({ "appserviceName" = module_versions.umbraco_version_values.appserviceName, "appserviceHostname" = module_versions.umbraco_version_values.appserviceHostname, "umbraco_cms_version" = module_versions.umbraco_version_values.umbraco_cms_version }) + for k, m in module.versions : k => { + hostname = m.test_case_values.app_service_hostname + app_service_name = m.test_case_values.app_service_name + app_service_plan_id = m.test_case_values.app_service_plan_id + # app_sku and pool_dtu reflect the override when set, so downstream metadata + # records what was actually provisioned, not the tier's nominal value. + app_service_sku = var.app_sku_override != "" ? var.app_sku_override : var.tier_specs[var.test_cases[k].tier].app_sku + pool_dtu_max = var.pool_dtu_override != 0 ? var.pool_dtu_override : var.tier_specs[var.test_cases[k].tier].dtu_max + sql_database_name = m.test_case_values.sql_database_name + sql_database_id = m.test_case_values.sql_database_id + } } } - -output "hostnames" { - value = [ - for module_versions in module.versions : - module_versions.umbraco_version_values.appserviceHostname - ] -} - -output "cms_versions" { - value = [ - for module_versions in module.versions : - module_versions.umbraco_version_values.umbraco_cms_version - ] -} - -output "app_service_name" { - value = [ - for module_versions in module.versions : - module_versions.umbraco_version_values.appserviceName - ] -} \ No newline at end of file diff --git a/Terraform/modules/umbraco/scripts/install-umbraco-cms-on-appservice.ps1 b/Terraform/modules/umbraco/scripts/install-umbraco-cms-on-appservice.ps1 index 63ca647..ea57435 100644 --- a/Terraform/modules/umbraco/scripts/install-umbraco-cms-on-appservice.ps1 +++ b/Terraform/modules/umbraco/scripts/install-umbraco-cms-on-appservice.ps1 @@ -1,102 +1,330 @@ +#requires -Version 7.3 + +# Build, publish, and zip-deploy a fresh Umbraco CMS project to the target App +# Service, then poll the seeder status endpoint until seeding completes. +# Invoked from terraform's null_resource.deploy_umbraco local-exec; expects SP +# credentials in env vars (ARM_CLIENT_ID + ARM_TENANT_ID + one of +# ARM_CLIENT_SECRET / ARM_OIDC_TOKEN). Stops the App Service when done so the +# load-test step can start cleanly later. + [CmdletBinding()] param ( - [Parameter(Mandatory = $true)] - [string] - $rgName, - [Parameter(Mandatory = $true)] - [string] - $appserviceName, - [Parameter(Mandatory = $true)] - [string] - $appserviceHostname, - [Parameter(Mandatory = $true)] - [string] - $umbracoVersion, - [Parameter(Mandatory= $true)] - [string] - $client_id, - [Parameter(Mandatory= $true)] - [string] - $client_secret, - [Parameter(Mandatory= $true)] - [string] - $tenant_id + [Parameter(Mandatory = $true)] [string]$ResourceGroupName, + [Parameter(Mandatory = $true)] [string]$AppServiceName, + [Parameter(Mandatory = $true)] [string]$AppServiceHostname, + [Parameter(Mandatory = $true)] [string]$UmbracoVersion, + [Parameter(Mandatory = $true)] [string]$Scenario, + [Parameter(Mandatory = $true)] [string]$SeederPreset, + # File the script writes once the seeder finishes — the load-test job + # reads it to surface seeder_duration_seconds in the published metrics. + # On Skipped/Failed seeder, duration_seconds is written as null. + [Parameter(Mandatory = $true)] [string]$SeederResultPath ) -# Remove dots from Umbraco version for creating folder -$updatedVersionName = $umbracoVersion.Replace('.','') +$ErrorActionPreference = "Stop" -$pathToApp = "./NewUmbracoProject$updatedVersionName" -$nameToApp = "NewUmbracoProject$updatedVersionName" +# Make native commands (dotnet, az, Compress-Archive interop) honour $ErrorActionPreference +# so a non-zero exit fails the script instead of silently continuing past a broken build +# or deploy. Requires pwsh 7.3+. Without this, $LASTEXITCODE has to be checked after every +# native command. +$PSNativeCommandUseErrorActionPreference = $true -# Create a new folder for Umbraco Template installation -mkdir $updatedVersionName +# ARM_CLIENT_ID + ARM_TENANT_ID are always required; one of ARM_CLIENT_SECRET +# (client-secret auth) or ARM_OIDC_TOKEN (WIF) must also be set. +foreach ($name in @('ARM_CLIENT_ID', 'ARM_TENANT_ID')) { + if (-not [Environment]::GetEnvironmentVariable($name)) { + Write-Error "Required env var $name is not set." + exit 1 + } +} +if (-not $env:ARM_CLIENT_SECRET -and -not $env:ARM_OIDC_TOKEN) { + Write-Error "Either ARM_CLIENT_SECRET (client-secret auth) or ARM_OIDC_TOKEN (WIF) must be set." + exit 1 +} -# Switch location to the new directory -Set-Location $updatedVersionName +# Umbraco.Cms.TestDataSeeder package version per Umbraco major. Update an entry +# here when a new seeder build ships; null means the seeder isn't available yet +# for that major and the run will fail-fast below with a clear message. +# The version is baked into the build cache key, so bumps auto-invalidate stale builds. +$seederPackageVersions = @{ + 13 = "13.0.0-beta.1" + 17 = "17.0.0-beta.2" + # No dedicated v18 build yet; v17 seeder works on v18 (the seeder's surface + # area is stable across the v17→v18 jump). Bump to a v18 build if/when one + # ships and the v17 fallback drifts. + 18 = "17.0.0-beta.2" + # v14/v15/v16: no published seeder yet. resolve-run-config.ps1 fails the + # run at validation with a clear message; add entries here in lockstep when + # those builds ship. +} -# Add nuget package sources for Umbraco prereleases and nightly builds -dotnet nuget add source "https://www.myget.org/F/umbracoprereleases/api/v3/index.json" -n "Umbraco Prereleases" -dotnet nuget add source "https://www.myget.org/F/umbraconightly/api/v3/index.json" -n "Umbraco Nightly" +$umbracoMajor = [int](($UmbracoVersion -split '\.')[0]) +$seederPackageVersion = $seederPackageVersions[$umbracoMajor] +if (-not $seederPackageVersion) { + Write-Error "No Umbraco.Cms.TestDataSeeder version mapped for Umbraco $UmbracoVersion (major $umbracoMajor). Add an entry to `$seederPackageVersions in this script when the package ships for that major." + exit 1 +} + +# Captured before Set-Location so finally{} can restore cwd on failure. +$terraformCwd = (Get-Location).Path + +$updatedVersionName = $UmbracoVersion.Replace('.', '') +$pathToApp = "./NewUmbracoProject$updatedVersionName" +$nameToApp = "NewUmbracoProject$updatedVersionName" +$absoluteBuildDir = Join-Path $terraformCwd $updatedVersionName -# Install Umbraco Template and create the project -dotnet new install Umbraco.Templates::$umbracoVersion -dotnet new umbraco -n $nameToApp +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Deploying Umbraco $UmbracoVersion ($Scenario)" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan -cd $nameToApp +# Always build from scratch — the local/shared build cache was removed because +# the time saving didn't materialise in practice. Each pipeline run pays the +# ~5-8 minute build cost once per (version, scenario) the first time it's +# touched. Cleanup happens in the finally{} block regardless of outcome. -# Adds the starter kit Clean -# If the Umbraco version is 13.0.0 and above, the newest version of Clean is installed. -# Otherwise, version 3.1.4 of Clean will be installed. -if ($umbracoVersion -ge "16.0.0") { - dotnet add package clean -} elseif ($umbracoVersion -ge "15.0.0" -and $umbracoVersion -lt "16.0.0") { - dotnet add package clean --version 5.2.2 -} elseif ($umbracoVersion -ge "13.0.0" -and $umbracoVersion -lt "15.0.0") { - dotnet add package clean --version 4.1.0 -} else { - dotnet add package clean --version 3.1.4 +# Clean any leftover from a previous failed run, otherwise dotnet new would fail. +if (Test-Path -LiteralPath $updatedVersionName) { + Write-Host "Cleaning leftover build dir: $updatedVersionName" + Remove-Item -Recurse -Force -LiteralPath $updatedVersionName } -# Build the project to retrieve files from the Clean Starter Kit -dotnet build +try { + New-Item -ItemType Directory -Path $updatedVersionName -Force | Out-Null + Set-Location -LiteralPath $updatedVersionName + + # Prerelease and nightly feeds — needed for Umbraco versions not yet on nuget.org. + # `dotnet nuget add source` returns non-zero when the source name already + # exists, which would throw under $PSNativeCommandUseErrorActionPreference. + # Wrap so a re-run on a warm agent (or local-dev re-apply) is a no-op. + try { dotnet nuget add source "https://www.myget.org/F/umbracoprereleases/api/v3/index.json" -n "Umbraco Prereleases" 2>$null | Out-Null } catch {} + try { dotnet nuget add source "https://www.myget.org/F/umbraconightly/api/v3/index.json" -n "Umbraco Nightly" 2>$null | Out-Null } catch {} + + Write-Host "Installing Umbraco.Templates::$UmbracoVersion..." + dotnet new install Umbraco.Templates::$UmbracoVersion + + Write-Host "Creating new Umbraco project: $nameToApp..." + dotnet new umbraco -n $nameToApp + + Set-Location -LiteralPath $nameToApp + + Write-Host "Adding Umbraco.Cms.TestDataSeeder $seederPackageVersion..." + dotnet add package Umbraco.Cms.TestDataSeeder --version $seederPackageVersion + if ($LASTEXITCODE -ne 0) { + Write-Error "dotnet add package Umbraco.Cms.TestDataSeeder failed (exit $LASTEXITCODE)." + exit 1 + } + + # Copy scenario code overlay (everything except appsettings.json) into the project tree. + $additionalSetupCandidate = Join-Path $terraformCwd "../loadtests/scenarios/$Scenario/AdditionalSetup" + $resolvedAdditional = $null + if (Test-Path -LiteralPath $additionalSetupCandidate) { + $resolvedAdditional = (Resolve-Path -LiteralPath $additionalSetupCandidate).Path + } + + if ($resolvedAdditional) { + $overlayFiles = @(Get-ChildItem -Path $resolvedAdditional -Recurse -File | + Where-Object { $_.Name -ne 'appsettings.json' }) + + if ($overlayFiles.Count -gt 0) { + Write-Host "" + Write-Host "Applying scenario '$Scenario' code overlay ($($overlayFiles.Count) file(s))..." -ForegroundColor Cyan + $projectRoot = (Get-Location).Path + foreach ($f in $overlayFiles) { + $rel = $f.FullName.Substring($resolvedAdditional.Length).TrimStart([IO.Path]::DirectorySeparatorChar, '/', '\') + $dest = Join-Path $projectRoot $rel + $destDir = Split-Path -Parent $dest + if ($destDir -and -not (Test-Path -LiteralPath $destDir)) { + New-Item -ItemType Directory -Path $destDir -Force | Out-Null + } + Copy-Item -LiteralPath $f.FullName -Destination $dest -Force + Write-Host " + $rel" + } + } else { + Write-Host "Scenario '$Scenario' has no code-overlay files." + } + } + else { + Write-Host "Scenario '$Scenario' has no AdditionalSetup folder - skipping code overlay." + } + + Write-Host "Building project..." + dotnet build --configuration Release -cd .. + Set-Location -LiteralPath .. -# Publish the app and create a zip file -dotnet publish $pathToApp -c Release -o $pathToApp/publish -Compress-Archive -Path $pathToApp/publish/* -DestinationPath $pathToApp/publish.zip + Write-Host "Publishing application..." + dotnet publish $pathToApp -c Release -o $pathToApp/publish -# Log in to Azure using service principal credentials -az login --service-principal --username $client_id --password $client_secret --tenant $tenant_id + Write-Host "Creating deployment package..." + Compress-Archive -Path $pathToApp/publish/* -DestinationPath $pathToApp/publish.zip -Force -# Deploy the Umbraco CMS to the app service -az webapp deployment source config-zip --src $pathToApp/publish.zip -n $appserviceName -g $rgName + # Auth + deploy happen INSIDE the try block so publish.zip is still present + # (the finally cleanup below removes the whole build dir). Previously the + # build cache kept a copy of the zip outside the build dir; without the + # cache, we must complete the deploy before tearing down the build tree. + Set-Location -LiteralPath $terraformCwd -# Clean up the app folder -Remove-Item -Recurse -Force $pathToApp + # WIF (federated token) takes priority over client-secret auth when both are set. + Write-Host "Authenticating to Azure..." + if ($env:ARM_OIDC_TOKEN) { + az login --service-principal --username $env:ARM_CLIENT_ID --tenant $env:ARM_TENANT_ID --federated-token $env:ARM_OIDC_TOKEN | Out-Null + } else { + az login --service-principal --username $env:ARM_CLIENT_ID --password $env:ARM_CLIENT_SECRET --tenant $env:ARM_TENANT_ID | Out-Null + } -# Return to the root folder of the Terraform project -Set-Location .. + # Pin the subscription explicitly — `az login` defaults to whichever sub the SP + # happens to land on, which for multi-sub SPs is not necessarily the one Terraform + # just provisioned the App Service in. ARM_SUBSCRIPTION_ID is set by the pipeline + # and inherits into this local-exec process. + if ($env:ARM_SUBSCRIPTION_ID) { + az account set --subscription $env:ARM_SUBSCRIPTION_ID + } else { + Write-Warning "ARM_SUBSCRIPTION_ID not set; deploy will target the SP's default subscription." + } -# Clean up the Umbraco Template install folder -Remove-Item -Force $updatedVersionName + $deployZip = Join-Path $absoluteBuildDir "$nameToApp/publish.zip" + Write-Host "Deploying to App Service: $AppServiceName..." + az webapp deployment source config-zip --src $deployZip -n $AppServiceName -g $ResourceGroupName +} +finally { + # Restore cwd and clean the build dir on every path (success and failure). + Set-Location -LiteralPath $terraformCwd -ErrorAction SilentlyContinue + if (Test-Path -LiteralPath $absoluteBuildDir) { + Write-Host "Cleaning up build artifacts..." + Remove-Item -Recurse -Force -LiteralPath $absoluteBuildDir -ErrorAction SilentlyContinue + } +} -# Ping the App Service to trigger the installation process -function Get-UrlStatusCode([string] $Url) -{ - try - { - (Invoke-WebRequest -Uri $Url -UseBasicParsing -DisableKeepAlive).StatusCode +function Write-SeederResult { + param( + [Parameter(Mandatory)] [string]$Status, + [Nullable[double]]$DurationSeconds = $null + ) + $payload = [ordered]@{ + status = $Status + duration_seconds = $DurationSeconds } - catch [Net.WebException] - { - [int]$_.Exception.Response.StatusCode + $dir = Split-Path -Parent $SeederResultPath + if ($dir -and -not (Test-Path $dir)) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + } + $payload | ConvertTo-Json -Compress | Set-Content -Path $SeederResultPath -Encoding utf8 +} + +# Wait for the data seeder to finish. The polling doubles as App Service warm-up. +$seederStatusUrl = "https://$AppServiceHostname/umbraco/api/seederstatus/status" +$maxAttemptsByPreset = @{ Small = 60; Medium = 180; Large = 360; Massive = 720 } # 10/30/60/120 min at 10s cadence +$maxAttempts = $maxAttemptsByPreset[$SeederPreset] ?? 360 +Write-Host "Seeder timeout for preset '$SeederPreset': $($maxAttempts * 10 / 60) minutes." +$attempt = 0 +$seederComplete = $false +$seederSuccess = $false + +Write-Host "" +Write-Host "Waiting for data seeder to complete..." -ForegroundColor Yellow + +while ($attempt -lt $maxAttempts -and -not $seederComplete) { + Start-Sleep -Seconds 10 + $attempt++ + + try { + # -SkipHttpErrorCheck so 503 lands here, not in catch{}. + $response = Invoke-WebRequest -Uri $seederStatusUrl -UseBasicParsing -TimeoutSec 30 -SkipHttpErrorCheck + $seederStatusCode = $response.StatusCode + + # Parse the body in its own guard. A non-JSON 5xx (App Service warmup + # HTML, gateway error page) would otherwise fall into the outer catch + # and get misreported as "Waiting for seeder endpoint..." for the full + # timeout budget — hiding the real failure for up to 120 minutes on + # the Massive preset. + try { + $responseBody = $response.Content | ConvertFrom-Json + } catch { + Write-Host " [$attempt/$maxAttempts] HTTP $seederStatusCode (non-JSON body); retrying..." + continue + } + + # Terminal-OK states: Completed, CompletedWithErrors, Skipped. + if ($seederStatusCode -eq 200 -and $responseBody.Status -in @("Completed", "CompletedWithErrors", "Skipped")) { + Write-Host "" + if ($responseBody.Status -eq "Skipped") { + Write-Host "Data seeder was disabled in scenario config - skipping wait." -ForegroundColor Green + Write-SeederResult -Status "Skipped" + } else { + $elapsedSeconds = [math]::Round($responseBody.ElapsedMs / 1000, 2) + $verb = ($responseBody.Status -eq "CompletedWithErrors") ? "completed with errors" : "completed successfully" + Write-Host "Data seeder $verb!" -ForegroundColor Green + Write-Host " Duration: $elapsedSeconds seconds" + Write-Host " Executed: $($responseBody.ExecutedCount)" + Write-Host " Failed: $($responseBody.FailedCount)" + Write-SeederResult -Status $responseBody.Status -DurationSeconds $elapsedSeconds + } + $seederComplete = $true + $seederSuccess = $true + } + elseif ($seederStatusCode -eq 503) { + Write-Host "" + Write-Host "ERROR: Seeder reported failure - $($responseBody.ErrorMessage)" -ForegroundColor Red + Write-SeederResult -Status "Failed" + $seederComplete = $true + } + else { + $status = if ($responseBody.CurrentSeeder) { $responseBody.CurrentSeeder } else { $responseBody.Status } + Write-Host " [$attempt/$maxAttempts] Status: $status" + } + } + catch { + Write-Host " [$attempt/$maxAttempts] Waiting for seeder endpoint..." + } +} + +# Best-effort `az webapp stop` with retries. Azure's management API can return +# a transient 503 ('Service Unavailable') right after a deployment finishes, +# even though the resource is healthy. Retrying with short backoff handles the +# common case; if all retries fail, log a warning and continue — the App +# Service stays running but the load-test stage's `az webapp start` is +# idempotent, so nothing functional breaks downstream. +function Stop-AppServiceBestEffort { + param([Parameter(Mandatory)] [string]$Name, [Parameter(Mandatory)] [string]$ResourceGroup) + $delays = @(5, 10, 20) + for ($i = 0; $i -lt $delays.Count; $i++) { + $prevPref = $PSNativeCommandUseErrorActionPreference + $PSNativeCommandUseErrorActionPreference = $false + try { + az webapp stop -n $Name -g $ResourceGroup + $exit = $LASTEXITCODE + } finally { + $PSNativeCommandUseErrorActionPreference = $prevPref + } + if ($exit -eq 0) { return $true } + $isLast = ($i -eq $delays.Count - 1) + if ($isLast) { + Write-Host "##vso[task.logissue type=warning]az webapp stop failed after $($delays.Count) attempts (last exit $exit). App Service stays running until the load-test stage starts it." + return $false + } + Write-Host " az webapp stop exit $exit; retrying in $($delays[$i])s..." + Start-Sleep -Seconds $delays[$i] + } + return $false +} + +if (-not $seederSuccess) { + Write-Host "" + Write-Host "Seeder did not complete - stopping App Service and exiting non-zero" -ForegroundColor Red + # 503 path already wrote 'Failed'; this catches the loop-timeout case. + if (-not (Test-Path $SeederResultPath)) { + Write-SeederResult -Status "TimedOut" } + Stop-AppServiceBestEffort -Name $AppServiceName -ResourceGroup $ResourceGroupName | Out-Null + exit 1 } -$statusCode = Get-UrlStatusCode $appserviceHostname -Write-Host "StatusCode is: $statusCode" +# Stop the app service until the load-test step starts it again. +Write-Host "" +Write-Host "Stopping App Service until load test..." -ForegroundColor Cyan +Stop-AppServiceBestEffort -Name $AppServiceName -ResourceGroup $ResourceGroupName | Out-Null -# Stop the app service -az webapp stop -n $appserviceName -g $rgName \ No newline at end of file +Write-Host "" +Write-Host "========================================" -ForegroundColor Green +Write-Host "Deployment complete: $UmbracoVersion" -ForegroundColor Green +Write-Host "========================================" -ForegroundColor Green diff --git a/Terraform/modules/umbraco/variables.tf b/Terraform/modules/umbraco/variables.tf index 6b90524..98ccf46 100644 --- a/Terraform/modules/umbraco/variables.tf +++ b/Terraform/modules/umbraco/variables.tf @@ -1,41 +1,73 @@ variable "resource_group_location" { - type = string + type = string + description = "Azure region for resource deployment" } variable "resource_group_name" { - type = string + type = string + description = "Name of the Azure resource group" } variable "resource_name_prefix" { - description = "This name will prefix all the created resources" - validation { - condition = can(regex("^[0-9a-z]([-0-9a-z]{0,100}[0-9a-z])?$", var.resource_name_prefix)) - error_message = "The prefix can contain only lowercase letters, numbers, and '-', but can't start or end with '-' or have more than 100 characters." - } + type = string + description = "Prefix for all created resources. Validated at the root module (max 16 chars)." } -variable "umbraco_cms_versions" { +variable "tier_specs" { type = map(object({ - dotnet_version = string - umbraco_version = string + app_sku = string + dtu_max = number })) + description = "Decoded tier catalog (loadtests/tiers.json), keyed by tier name." } -variable "client_id" { - type = string - sensitive = true +variable "test_cases" { + type = map(object({ + dotnet_version = string + umbraco_version = string + tier = string + scenario = string + app_settings_overlay = map(string) + })) + description = "Map of test cases keyed by '{umbraco}__{tier}__{scenario}'." +} + +variable "seeder_preset" { + type = string + description = "Data seeder preset (Small, Medium, Large, Massive)" + default = "Medium" +} + +variable "pool_dtu_override" { + type = number + description = "Override per-DB DTU cap for every case (0 = use each tier's default from tier_specs)." + default = 0 } -variable "client_secret" { - type = string - sensitive = true +variable "app_sku_override" { + type = string + description = "Override App Service Plan SKU for every case ('' = use each tier's default from tier_specs)." + default = "" } -variable "tenant_id" { - type = string - sensitive = true +variable "build_id" { + type = string + description = "Pipeline build ID, surfaced as a resource tag" + default = "local" } -variable "app_service_plan_sku" { - type = string -} \ No newline at end of file +# Cost guard. See root variables.tf for rationale. +variable "budget_alert_amount" { + type = number + default = 50 +} + +variable "budget_alert_threshold_pct" { + type = number + default = 80 +} + +variable "budget_alert_emails" { + type = list(string) + default = [] +} diff --git a/Terraform/modules/umbraco/versions/main.tf b/Terraform/modules/umbraco/versions/main.tf index ce147d8..46548c0 100644 --- a/Terraform/modules/umbraco/versions/main.tf +++ b/Terraform/modules/umbraco/versions/main.tf @@ -1,48 +1,63 @@ -# Replace . with - in Umbraco versions. This is necessary for resource naming. locals { - short_version_name = (split("-", var.umbraco_cms_version)[0]) - version_name = replace(local.short_version_name, ".", "-") -} + # "17.0.0__Standard__Default" -> "17-0-0-standard-default" + case_suffix = replace(lower(var.test_case_id), "/[._]+/", "-") -# SQL server -resource "azurerm_mssql_server" "msserver" { - name = "${var.resource_name_prefix}-sqlserver-${local.version_name}" - resource_group_name = var.resource_group_name - location = var.resource_group_location - version = "12.0" - administrator_login = var.admin_login - administrator_login_password = var.admin_password - minimum_tls_version = "1.2" - - # Added a timeout for creating the server because it can sometimes fail and run for 60 minutes - timeouts { - create = "7m" - } -} + # Predicted hostname; reading default_hostname here would self-cycle. + app_service_name = "${var.resource_name_prefix}-appservice-${local.case_suffix}" + app_service_hostname_predict = "${local.app_service_name}.azurewebsites.net" + + case_tags = merge(var.common_tags, { + test_case_id = var.test_case_id + umbraco_version = var.umbraco_version + scenario = var.scenario + }) -# Allow all Azure services to access the SQL Server -resource "azurerm_mssql_firewall_rule" "firewallrule" { - name = "Allow all azure services-${local.version_name}" - server_id = azurerm_mssql_server.msserver.id - start_ip_address = "0.0.0.0" - end_ip_address = "0.0.0.0" + # Hash the scenario AdditionalSetup tree so deploy_umbraco re-runs on any overlay edit. + # The validator allows scenarios to omit the folder entirely; try() returns [] + # when the directory doesn't exist, producing a stable empty-set hash. + scenario_overlay_dir = "${path.root}/../loadtests/scenarios/${var.scenario}/AdditionalSetup" + scenario_overlay_files = try(sort(tolist(fileset(local.scenario_overlay_dir, "**"))), []) + scenario_overlay_hash = sha256(join("|", [ + for f in local.scenario_overlay_files : "${f}=${filesha256("${local.scenario_overlay_dir}/${f}")}" + ])) } -# Database -resource "azurerm_mssql_database" "db" { - name = "${var.resource_name_prefix}-db-${local.version_name}" - server_id = azurerm_mssql_server.msserver.id - collation = "SQL_Latin1_General_CP1_CI_AS" - max_size_gb = 5 - sku_name = "S0" +# Database - joins the per-tier Elastic Pool. The Cloud model is one shared +# pool per plan; per-DB DTU cap is enforced by the pool's per_database_settings. +# Setting sku_name = "ElasticPool" is the documented way to attach to a pool. +resource "azurerm_mssql_database" "database" { + name = "${var.resource_name_prefix}-db-${local.case_suffix}" + server_id = var.sql_server_id + collation = "SQL_Latin1_General_CP1_CI_AS" + sku_name = "ElasticPool" + elastic_pool_id = var.elastic_pool_id + tags = local.case_tags } # App Service -resource "azurerm_windows_web_app" "appservice" { - name = "${var.resource_name_prefix}-appservice-${local.version_name}" +resource "azurerm_windows_web_app" "app_service" { + name = local.app_service_name location = var.resource_group_location resource_group_name = var.resource_group_name service_plan_id = var.service_plan_id + tags = local.case_tags + + # Catch the 60-char Azure App Service name cap with a clear message instead + # of a generic Azure 400 deep in apply. Long Umbraco prerelease tags + long + # prefixes can blow this budget; shorten one of (prefix, version, scenario). + lifecycle { + precondition { + condition = length(local.app_service_name) <= 60 + error_message = "Computed App Service name '${local.app_service_name}' is ${length(local.app_service_name)} chars (Azure cap is 60). Shorten the prefix, Umbraco version, or scenario name." + } + } + + # Force HTTPS; disable basic-auth deploy paths; disable session affinity so load + # tests round-robin across plan instances. + https_only = true + ftp_publish_basic_authentication_enabled = false + webdeploy_publish_basic_authentication_enabled = false + client_affinity_enabled = false site_config { application_stack { @@ -51,31 +66,76 @@ resource "azurerm_windows_web_app" "appservice" { } } - app_settings = { - "Umbraco.Core.LocalTempStorage" = "EnvironmentTemp" - "Umbraco.Examine.LuceneDirectoryFactory" = "Examine.LuceneEngine.Directories.SyncTempEnvDirectoryFactory, Examine" - "Umbraco__CMS__Unattended__InstallUnattended" = "true" - "Umbraco__CMS__Unattended__UnattendedUserName" = "John Doe" - "Umbraco__CMS__Unattended__UnattendedUserEmail" = "admin@admin.admin" - "Umbraco__CMS__Unattended__UnattendedUserPassword" = "1234567890" - "SCM_DO_BUILD_DURING_DEPLOYMENT" = true - "Serilog__MinimumLevel__Override__Microsoft" = "Information" - } + # Overlay wins on key collision. + app_settings = merge( + { + "Umbraco.Core.LocalTempStorage" = "EnvironmentTemp" + "Umbraco.Examine.LuceneDirectoryFactory" = "Examine.LuceneEngine.Directories.SyncTempEnvDirectoryFactory, Examine" + + "Umbraco__CMS__Unattended__InstallUnattended" = "true" + "Umbraco__CMS__Unattended__UnattendedUserName" = "Load Test Admin" + "Umbraco__CMS__Unattended__UnattendedUserEmail" = "loadtest@example.invalid" + # Hardcoded so anyone on the team can log into the backoffice with known creds. + "Umbraco__CMS__Unattended__UnattendedUserPassword" = "LoadTest123!" + + # Pre-built artifacts are zip-deployed via `az webapp deployment source config-zip`, + # so Oryx/Kudu shouldn't try to build again. False shaves a few seconds per deploy + # and avoids edge cases where Oryx misidentifies the artifact. + "SCM_DO_BUILD_DURING_DEPLOYMENT" = "false" + "Serilog__MinimumLevel__Override__Microsoft" = "Information" + + "Umbraco.Cms.TestDataSeeder__Options__Enabled" = "true" + "Umbraco.Cms.TestDataSeeder__Options__Preset" = var.seeder_preset + "Umbraco.Cms.TestDataSeeder__Options__DomainSuffix" = local.app_service_hostname_predict + }, + var.app_settings_overlay + ) connection_string { - name = "umbracoDbDSN" - type = "SQLAzure" - value = "Server=tcp:${azurerm_mssql_server.msserver.fully_qualified_domain_name},1433;Initial Catalog=${azurerm_mssql_database.db.name};Persist Security Info=False;User ID=${azurerm_mssql_server.msserver.administrator_login}@${azurerm_mssql_server.msserver.name};Password=${azurerm_mssql_server.msserver.administrator_login_password};MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=120;" + name = "umbracoDbDSN" + type = "SQLAzure" + value = join(";", [ + "Server=tcp:${var.sql_server_fqdn},1433", + "Initial Catalog=${azurerm_mssql_database.database.name}", + "Persist Security Info=False", + "User ID=${var.admin_login}@${var.sql_server_name}", + "Password=${var.admin_password}", + "MultipleActiveResultSets=False", + "Encrypt=True", + "TrustServerCertificate=False", + "Connection Timeout=120", + ]) } } -# Runs the script to create and deploy Umbraco CMS with the defined version. -resource "null_resource" "deploy_umbraco_windows_host" { +# Runs the script to build, publish, and deploy Umbraco CMS. +resource "null_resource" "deploy_umbraco" { + triggers = { + umbraco_version = var.umbraco_version + app_service_id = azurerm_windows_web_app.app_service.id + scenario = var.scenario + scenario_overlay_hash = local.scenario_overlay_hash + # Referenced here purely to establish the implicit dependency on the + # parent module's firewall rule (Umbraco hits SQL on first boot). + firewall_rule_id = var.sql_firewall_rule_dependency + } + provisioner "local-exec" { - command = "./modules/umbraco/scripts/install-umbraco-cms-on-appservice.ps1 -rgName \"${var.resource_group_name}\" -appserviceName \"${azurerm_windows_web_app.appservice.name}\" -appserviceHostname \"${azurerm_windows_web_app.appservice.default_hostname}\" -umbracoVersion \"${var.umbraco_cms_version}\" -client_id \"${var.client_id}\" -client_secret \"${var.client_secret}\" -tenant_id \"${var.tenant_id}\"" - // Remember to change to "pwsh" for the pipeline - // "Powershell" for local + # SP credentials inherit from the parent terraform process env (set on the + # pipeline's Terraform Apply task). Not declared here on purpose - putting + # sensitive vars in `environment = {}` makes terraform suppress local-exec + # output, which hides the install script's progress. + # + # SeederResultPath is a per-test-case JSON file under /.seeder-results + # (path.root is the Terraform working dir = $(System.DefaultWorkingDirectory)/Terraform, + # so '../' resolves to the pipeline workspace root). The load-test job reads + # this file via the same path to surface seeder duration in the published metrics. + command = "./modules/umbraco/scripts/install-umbraco-cms-on-appservice.ps1 -ResourceGroupName \"${var.resource_group_name}\" -AppServiceName \"${azurerm_windows_web_app.app_service.name}\" -AppServiceHostname \"${azurerm_windows_web_app.app_service.default_hostname}\" -UmbracoVersion \"${var.umbraco_version}\" -Scenario \"${var.scenario}\" -SeederPreset \"${var.seeder_preset}\" -SeederResultPath \"${path.root}/../.seeder-results/${var.test_case_id}.json\"" interpreter = ["pwsh", "-Command"] } - depends_on = [azurerm_windows_web_app.appservice] -} \ No newline at end of file + + depends_on = [ + azurerm_windows_web_app.app_service, + azurerm_mssql_database.database, + ] +} diff --git a/Terraform/modules/umbraco/versions/output.tf b/Terraform/modules/umbraco/versions/output.tf index 948784e..771b7dd 100644 --- a/Terraform/modules/umbraco/versions/output.tf +++ b/Terraform/modules/umbraco/versions/output.tf @@ -1,7 +1,9 @@ -output "umbraco_version_values" { +output "test_case_values" { value = { - appserviceName = azurerm_windows_web_app.appservice.name - appserviceHostname = azurerm_windows_web_app.appservice.default_hostname - umbraco_cms_version = var.umbraco_cms_version + app_service_name = azurerm_windows_web_app.app_service.name + app_service_hostname = azurerm_windows_web_app.app_service.default_hostname + app_service_plan_id = var.service_plan_id + sql_database_name = azurerm_mssql_database.database.name + sql_database_id = azurerm_mssql_database.database.id } -} \ No newline at end of file +} diff --git a/Terraform/modules/umbraco/versions/variables.tf b/Terraform/modules/umbraco/versions/variables.tf index a652840..51a79c1 100644 --- a/Terraform/modules/umbraco/versions/variables.tf +++ b/Terraform/modules/umbraco/versions/variables.tf @@ -1,50 +1,94 @@ variable "resource_name_prefix" { - description = "This name will prefix all the created resources" + type = string + description = "Prefix for all created resources" } variable "resource_group_name" { - description = "Name of the resource group that the app service is gonna be located in" + type = string + description = "Resource group the App Service lives in" } variable "resource_group_location" { - description = "Location for the Azure resources" + type = string + description = "Azure region" } variable "service_plan_id" { - description = "ID of the service plan the App service is gonna use" + type = string + description = "ID of the App Service Plan this case attaches to" } variable "dotnet_version" { type = string - description = "The version of dotnet to use" + description = ".NET runtime version (e.g. v10.0)" +} + +variable "umbraco_version" { + type = string + description = "Umbraco CMS version (e.g. 17.0.0)" } -variable "umbraco_cms_version" { +variable "scenario" { type = string - description = "The version of Umbraco.Cms to add as PackageReference" + description = "Scenario name, surfaced in test_case_outputs for tagging." +} + +variable "test_case_id" { + type = string + description = "Unique case identifier ({umbraco}__{tier}__{scenario}); used as the per-case resource-name suffix." +} + +variable "app_settings_overlay" { + type = map(string) + description = "Already-flattened App Service app_settings overlay; overlay keys win over base keys." + default = {} } variable "admin_login" { type = string - description = "admin login" + description = "SQL Server admin login" + sensitive = true } variable "admin_password" { type = string - description = "admin password" + description = "SQL Server admin password" + sensitive = true } -variable "client_id" { - type = string - sensitive = true +variable "sql_server_name" { + type = string + description = "Name of the shared per-tier SQL server hosting this case's database" } -variable "client_secret" { - type = string - sensitive = true +variable "sql_server_id" { + type = string + description = "Resource ID of the shared per-tier SQL server" } -variable "tenant_id" { - type = string - sensitive = true -} \ No newline at end of file +variable "sql_server_fqdn" { + type = string + description = "Fully qualified domain name of the shared per-tier SQL server" +} + +variable "elastic_pool_id" { + type = string + description = "ID of the per-tier Elastic Pool this case's database joins" +} + +variable "sql_firewall_rule_dependency" { + type = string + description = "Pass-through ID of the parent module's firewall rule, used only to express the create-order dependency in the versions submodule." +} + +variable "seeder_preset" { + type = string + description = "Data seeder preset (Small, Medium, Large, Massive)" + default = "Medium" +} + +variable "common_tags" { + type = map(string) + description = "Common tags applied to every per-case resource (merged with case-specific tags)" + default = {} +} diff --git a/Terraform/output.tf b/Terraform/output.tf index 4198289..89a28ec 100644 --- a/Terraform/output.tf +++ b/Terraform/output.tf @@ -1,11 +1,4 @@ -output "hostnames" { - value = module.umbraco.hostnames -} - -output "cms_versions" { - value = module.umbraco.cms_versions -} - -output "app_service_name" { - value = module.umbraco.app_service_name +output "test_case_outputs" { + description = "Per-case Terraform outputs, keyed by testCaseId" + value = module.umbraco.test_case_outputs } diff --git a/Terraform/script/versionsToJson.ps1 b/Terraform/script/versionsToJson.ps1 deleted file mode 100644 index b2171cd..0000000 --- a/Terraform/script/versionsToJson.ps1 +++ /dev/null @@ -1,67 +0,0 @@ -[CmdletBinding()] -param ( - [Parameter()] - [string] - $firstDotNetVersion, - [Parameter()] - [string] - $firstUmbracoVersion, - [Parameter()] - [string] - $secondDotNetVersion, - [Parameter()] - [string] - $secondUmbracoVersion, - [Parameter()] - [string] - $thirdDotNetVersion, - [Parameter()] - [string] - $thirdUmbracoVersion, - [Parameter()] - [string] - $fourthDotNetVersion, - [Parameter()] - [string] - $fourthUmbracoVersion, - - [hashtable[]] - $Hashtables - ) - -$Hashtables = -@{"dotnet_version" = $firstDotNetVersion; - "umbraco_version" = $firstUmbracoVersion -}, -@{"dotnet_version" = $secondDotNetVersion; - "umbraco_version" = $secondUmbracoVersion -}, -@{"dotnet_version" = $thirdDotNetVersion; - "umbraco_version" = $thirdUmbracoVersion -}, -@{"dotnet_version" = $fourthDotNetVersion; - "umbraco_version" = $fourthUmbracoVersion -} - -$JsonTest - -for ($versions = 0; $versions -lt $Hashtables.count; $versions++) { - if ($Hashtables[$versions]["dotnet_version"] -ne "null" -and $Hashtables[$versions]["umbraco_version"] -ne "null") { - $replaceSpecialChars = $Hashtables[$versions]["umbraco_version"].Replace('.','_') - - if ($JsonTest) { - $JsonTest += ',"version' + $replaceSpecialChars + '":{"dotnet_version":\"' + $Hashtables[$versions]["dotnet_version"] + '\","umbraco_version":\"' + $Hashtables[$versions]["umbraco_version"] + '\"}' - } - elseif (!$JsonTest) { - $JsonTest += '{"version' + $replaceSpecialChars + '":{"dotnet_version":\"' + $Hashtables[$versions]["dotnet_version"] + '\","umbraco_version":\"' + $Hashtables[$versions]["umbraco_version"] + '\"}' - } - } -} - -if ($JsonTest) { - $JsonTest += '}' -} - -[Environment]::SetEnvironmentVariable('umbracoTFversions', $JsonTest) - -Get-ChildItem Env:umbracoTFversions \ No newline at end of file diff --git a/Terraform/terraform.tfvars.example b/Terraform/terraform.tfvars.example new file mode 100644 index 0000000..d4c2b36 --- /dev/null +++ b/Terraform/terraform.tfvars.example @@ -0,0 +1,44 @@ +# Hand-craftable test_cases for local `terraform plan`. The pipeline builds +# this via scripts/prepare-test-cases.ps1. + +resource_name_prefix = "umbraco-loadtest" +resource_group_name = "umbraco-loadtest-rg" +resource_group_location = "West Europe" +seeder_preset = "Medium" # Small / Medium / Large / Massive + +# Force every case onto a specific per-DB DTU cap. 0 = use each tier's value +# from loadtests/tiers.json. Set to 10/20/50/100/200 to isolate the app-side contribution. +pool_dtu_override = 0 + +# Force every case onto a specific App Service Plan SKU. "" = use each tier's +# value from loadtests/tiers.json. Set to P0v3/P1v3/P2v3/P3v3 to isolate +# the SQL-side contribution. +app_sku_override = "" + +# Tier values must exist in loadtests/tiers.json. app_settings_overlay normally +# comes from the validator; hand-write a small one for local plans. +test_cases = { + "17.0.0__Standard__Default" = { + dotnet_version = "v10.0" + umbraco_version = "17.0.0" + tier = "Standard" + scenario = "Default" + app_settings_overlay = {} + } + "17.0.0__Pro__Default" = { + dotnet_version = "v10.0" + umbraco_version = "17.0.0" + tier = "Pro" + scenario = "Default" + app_settings_overlay = {} + } +} + +# Azure credentials are read from env vars by the azurerm provider — they are NOT +# declared as Terraform variables here, so don't try to set them in tfvars. Export +# in your shell before `terraform plan`: +# export ARM_CLIENT_ID="..." +# export ARM_CLIENT_SECRET="..." # client-secret auth +# export ARM_TENANT_ID="..." +# export ARM_SUBSCRIPTION_ID="..." +# (Or use Azure CLI auth: `az login` and let the provider pick up the session.) diff --git a/Terraform/variables.tf b/Terraform/variables.tf index 7452f90..a2f7a37 100644 --- a/Terraform/variables.tf +++ b/Terraform/variables.tf @@ -1,43 +1,104 @@ variable "resource_name_prefix" { type = string - description = "This name will prefix all the created resources" + description = "Prefix for all created resources. Capped at 16 chars to fit Azure App Service's 60-char name limit once the per-case suffix is appended." + + validation { + condition = length(var.resource_name_prefix) <= 16 && can(regex("^[0-9a-z]([-0-9a-z]{0,14}[0-9a-z])?$", var.resource_name_prefix)) + error_message = "resource_name_prefix must be 1-16 chars, lowercase alphanumeric + hyphens, not starting or ending with hyphen." + } } variable "resource_group_location" { - type = string - default = "West Europe" + type = string + description = "Azure region for resource deployment" + default = "West Europe" } variable "resource_group_name" { - type = string + type = string + description = "Name of the Azure resource group" } -variable "client_id" { - type = string - default = "empty" - sensitive = true +variable "test_cases" { + type = map(object({ + dotnet_version = string + umbraco_version = string + tier = string + scenario = string + app_settings_overlay = map(string) + })) + description = "Map of test cases keyed by '{umbraco}__{tier}__{scenario}'. Built by scripts/prepare-test-cases.ps1 from the pipeline's testCases parameter." } -variable "client_secret" { - type = string - default = "empty" - sensitive = true +# Surfaces in resource tags. Defaults to "local" for hand runs. +variable "build_id" { + type = string + description = "Pipeline build ID (or 'local'). Used as a tag on every resource." + default = "local" } -variable "tenant_id" { - type = string - default = "empty" - sensitive = true +variable "seeder_preset" { + type = string + description = "Data seeder preset (Small, Medium, Large, Massive)" + default = "Medium" + validation { + condition = contains(["Small", "Medium", "Large", "Massive"], var.seeder_preset) + error_message = "seeder_preset must be one of: Small, Medium, Large, Massive" + } } -variable "app_service_plan_sku" { - type = string - default = "S3" +# Override the per-DB DTU cap for every case in the run. 0 = use the tier's +# built-in value from tiers.json. Lets you size SQL independently of the App +# Service tier — useful when comparison data shows the app tier scales but the +# database is the bottleneck. The pool's eDTU capacity is computed from this +# cap (smallest valid Standard pool size that can hold a DB at the cap). +variable "pool_dtu_override" { + type = number + description = "Override per-DB DTU cap for every case (0 = use each tier's default)." + default = 0 + validation { + condition = var.pool_dtu_override == 0 || contains([10, 20, 50, 100, 200], var.pool_dtu_override) + error_message = "pool_dtu_override must be 0 (auto) or one of the valid Standard per-DB DTU caps: 10, 20, 50, 100, 200" + } } -variable "umbraco_cms_versions" { - type = map(object({ - dotnet_version = string - umbraco_version = string - })) -} \ No newline at end of file +# Override the App Service Plan SKU for every case in the run. Empty = use +# the tier's built-in value from tiers.json. Counterpart to pool_dtu_override +# on the app side — useful for "is the app saturating before SQL does?" sweeps. +variable "app_sku_override" { + type = string + description = "Override App Service Plan SKU for every case ('' = use each tier's default)." + default = "" + validation { + condition = var.app_sku_override == "" || contains(["P0v3", "P1v3", "P2v3", "P3v3"], var.app_sku_override) + error_message = "app_sku_override must be empty (auto) or one of: P0v3, P1v3, P2v3, P3v3" + } +} + +# Cost guard on the ephemeral RG. Budget is monthly-grain (Azure's smallest); +# fires when accumulated MTD spend on the RG crosses the threshold percentage. +# Realistic numbers at our run size: a normal 60-min run is well under $1; a +# forgotten validation gate that lives 8 hours is ~$5; multiple stuck/runaway +# RGs in one month would compound. Empty -BudgetAlertEmails skips the budget +# entirely (default), so it's opt-in until a real email goes here. +variable "budget_alert_amount" { + type = number + description = "Monthly budget cap for the ephemeral RG (USD). Notifies when actual MTD spend exceeds budget_alert_threshold_pct." + default = 50 +} + +variable "budget_alert_threshold_pct" { + type = number + description = "Percentage of budget_alert_amount that triggers the email notification (0-100)." + default = 80 + validation { + condition = var.budget_alert_threshold_pct > 0 && var.budget_alert_threshold_pct <= 1000 + error_message = "budget_alert_threshold_pct must be between 1 and 1000." + } +} + +variable "budget_alert_emails" { + type = list(string) + description = "Recipients of the budget alert. Empty list disables the budget resource entirely." + default = [] +} diff --git a/azure-pipeline.yml b/azure-pipeline.yml index dee71ca..ee83064 100644 --- a/azure-pipeline.yml +++ b/azure-pipeline.yml @@ -1,425 +1,622 @@ -# File: azure-pipeline.yml - name: load_test_pipeline trigger: none +pr: none pool: vmImage: 'ubuntu-latest' parameters: -- name: appServicePlanSku - displayName: Sku for App Service Plan - type: string - default: S3 - values: - - S1 - - S2 - - S3 -- name: userAmount - displayName: Amount of virtual users - type: number - default: 100 - values: - - 50 - - 100 - - 150 - - 200 - - 250 - - 300 -- name: prefix - displayName: Prefix for all resources created in terraform - type: string - default: umbraco-azure-load-test-pipeline + - name: umbracoVersion + displayName: Umbraco version (e.g. 17.0.0, 17.0.1-rc.1, 17.1.0-beta.2) + type: string + default: '17.0.0' + + - name: scenario + displayName: Scenario (must match a folder under loadtests/scenarios/) + type: string + default: 'Default' + values: + - 'Default' + - 'DeliveryApi' + + - name: runStarter + displayName: Run Starter tier + type: boolean + default: true + + - name: runStandard + displayName: Run Standard tier + type: boolean + default: false + + - name: runPro + displayName: Run Pro tier + type: boolean + default: false + + - name: runEnterprise + displayName: Run Enterprise tier + type: boolean + default: false + + - name: loadProfile + displayName: 'Load profile (intensity) — smoke: 50 VUs / 60s, standard: 100 VUs / 300s, stress: 300 VUs / 600s' + type: string + default: 'standard' + values: + - 'smoke' # Small, 50 VUs, 60s, 1 engine -- quick "does it work" probe + - 'standard' # Medium, 100 VUs, 300s, 1 engine -- typical comparison run + - 'stress' # Large, 300 VUs, 600s, 2 engines -- saturate the lower tiers + + - name: azureRegion + displayName: Azure region + type: string + default: West Europe + values: + - West Europe + - North Europe + - East US + - West US 2 + + # Feeds into RG name + resource names. Concurrent runs with the same prefix collide. + - name: resourcePrefix + displayName: Resource name prefix (max 16 chars, lowercase alphanumeric + hyphens; must be unique per concurrent run) + type: string + default: umbraco-loadtest + + - name: skipWarmup + displayName: Skip warmup (test cold-start behaviour) + type: boolean + default: false + + - name: validationTimeoutMinutes + displayName: Manual validation timeout (minutes) - how long resources stay alive after tests + type: number + default: 60 + values: + - 15 + - 30 + - 60 + - 120 + - 240 + + # Override the per-DB DTU cap for every case so SQL can be sized independently + # of the tier's default — useful for "is SQL actually the bottleneck?" + # experiments. Defaults (per loadtests/tiers.json): Starter=20, Standard=50, + # Pro=100, Enterprise=200. 'Auto' = use each tier's built-in DTU cap. + - name: poolDtuOverride + displayName: SQL per-DB DTU override (Auto = match tier) + type: string + default: 'Auto' + values: + - 'Auto' + - '10' + - '20' + - '50' + - '100' + - '200' + + # Override the App Service Plan SKU for every case so app size is independent + # of the tier — counterpart to poolDtuOverride on the app side. Defaults + # (per loadtests/tiers.json): Starter=P0v3, Standard=P1v3, Pro=P2v3, + # Enterprise=P3v3. 'Auto' = use each tier's built-in SKU. + - name: appSkuOverride + displayName: App Service SKU override (Auto = match tier) + type: string + default: 'Auto' + values: + - 'Auto' + - 'P0v3' + - 'P1v3' + - 'P2v3' + - 'P3v3' + + # Override the TestDataSeeder preset so content size is independent of the + # load profile. 'Auto' couples preset to profile (smoke=Small, standard=Medium, + # stress=Large). Override unlocks the otherwise-unreachable Massive preset + # and enables off-diagonal cells (Small content + stress load, Massive content + # + smoke load, etc.). Approximate seeder times: Small ~10 min, Medium ~30 min, + # Large ~60 min, Massive ~120 min. + - name: seederPresetOverride + displayName: Seeder preset override (Auto = match load profile) + type: string + default: 'Auto' + values: + - 'Auto' + - 'Small' + - 'Medium' + - 'Large' + - 'Massive' variables: - serviceConnection: 'terraform-umbraco-load-testing-az-serviceconnection' - - # Azure Resource Variables - azurergname: '${{ parameters.prefix }}-rg' - loadTestResourceName: '${{ parameters.prefix }}-loadtest' - # Terraform settings - terraformWorkingDirectory: '$(System.DefaultWorkingDirectory)/Terraform' - # LoadTestLocation - loadTestFileLocation: '$(System.DefaultWorkingDirectory)/LoadTestVersions.yaml' - + # Variable group `umbraco-loadtest-history` provides all history* variables: + # historyResourceGroup, historyLocation, historyLoadTestName, + # historyStorageAccount, historyContainer. + - group: umbraco-loadtest-history + + - name: serviceConnection + value: 'terraform-umbraco-load-testing-az-connection' + - name: azureResourceGroup + value: '${{ parameters.resourcePrefix }}-rg' + - name: terraformWorkingDirectory + value: '$(System.DefaultWorkingDirectory)/Terraform' + + # Long-lived monitoring resources. Defaults are fine for a single team in a + # subscription; override in the variable group if multiple teams share one. + - name: historyWorkspaceName + value: 'umbraco-loadtest-laws' + - name: historyDceName + value: 'umbraco-loadtest-dce' + - name: historyDcrName + value: 'umbraco-loadtest-dcr' + stages: -- stage: terraformDeploy - displayName: Terraform Setup and Apply - jobs: - - job: checkResourceGroup - displayName: Checks if resource group with defined name already exists - steps: - - task: AzureCLI@2 - name: checkGroup - displayName: Checks if the resource group exists before running all the steps - inputs: - azureSubscription: '$(serviceConnection)' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: 'Write-Host "##vso[task.setvariable variable=resourceGroupExists;isOutput=true]$(az group exists -n $(azurergname))"' - - - job: getFormattedVersions - displayName: Formats versions - steps: - # If a version is empty it is changed to null, otherwise our powershell script would not be able to read the input - - script: echo "##vso[task.setvariable variable=firstDotNetVersion]null" - displayName: update value if firstDotNetVersion is empty - condition: eq(variables['firstDotNetVersion'],'') - - script: echo "##vso[task.setvariable variable=firstUmbracoVersion]null" - displayName: update value if firstUmbracoVersion is empty - condition: eq(variables['firstUmbracoVersion'],'') - - script: echo "##vso[task.setvariable variable=secondDotNetVersion]null" - displayName: update value if secondDotNetVersion is empty - condition: eq(variables['secondDotNetVersion'],'') - - script: echo "##vso[task.setvariable variable=secondUmbracoVersion]null" - displayName: update value if secondUmbracoVersion is empty - condition: eq(variables['secondUmbracoVersion'],'') - - script: echo "##vso[task.setvariable variable=thirdDotNetVersion]null" - displayName: update value if thirdDotNetVersion is empty - condition: eq(variables['thirdDotNetVersion'],'') - - script: echo "##vso[task.setvariable variable=thirdUmbracoVersion]null" - displayName: update value if thirdUmbracoVersion is empty - condition: eq(variables['thirdUmbracoVersion'],'') - - script: echo "##vso[task.setvariable variable=fourthDotNetVersion]null" - displayName: update value if fourthDotNetVersion is empty - condition: eq(variables['fourthDotNetVersion'],'') - - script: echo "##vso[task.setvariable variable=fourthUmbracoVersion]null" - displayName: update value if fourthUmbracoVersion is empty - condition: eq(variables['fourthUmbracoVersion'],'') - - - task: PowerShell@2 - name: versionsOutput - inputs: - filePath: '$(terraformWorkingDirectory)/script/versionsToJson.ps1' - arguments: | - -firstDotNetVersion $(firstDotNetVersion) -firstUmbracoVersion $(firstUmbracoVersion) -secondDotNetVersion $(secondDotNetVersion) -secondUmbracoVersion $(secondUmbracoVersion) -thirdDotNetVersion $(thirdDotNetVersion) -thirdUmbracoVersion $(thirdUmbracoVersion) -fourthDotNetVersion $(fourthDotNetVersion) -fourthUmbracoVersion $(fourthUmbracoVersion) - echo "##vso[task.setvariable variable=umbracoTFversions;isOutput=true]$env:umbracoTFversions" - - - job: setup - dependsOn: - - checkResourceGroup - - getFormattedVersions - condition: eq(dependencies.checkResourceGroup.outputs['checkGroup.resourceGroupExists'], false) - displayName: Terraform Setup - variables: - umbracoTFversions: $[dependencies.getFormattedVersions.outputs['versionsOutput.umbracoTFversions']] - steps: - - task: TerraformInstaller@0 - displayName: Install Terraform - inputs: - terraformVersion: 'latest' - - task: PowerShell@2 - displayName: Terraform Init - inputs: - targetType: 'inline' - script: 'terraform init -backend=false' - workingDirectory: '$(terraformWorkingDirectory)' - - task: TerraformTaskV4@4 - displayName: Validate - inputs: - provider: 'azurerm' - command: 'validate' - workingDirectory: '$(terraformWorkingDirectory)' - - - script: echo '$(umbracoTFversions)' - - task: TerraformTaskV4@4 - displayName: Terraform Plan - inputs: - provider: 'azurerm' - command: 'plan' - workingDirectory: '$(terraformWorkingDirectory)' - environmentServiceNameAzureRM: '$(serviceConnection)' - commandOptions: '-parallelism=1 -var resource_name_prefix=${{ parameters.prefix }} -var resource_group_name=$(azurergname) -var="umbraco_cms_versions=$(umbracoTFversions)"' - - - job: apply - displayName: Terraform Apply - dependsOn: - - getFormattedVersions - - setup - variables: - umbracoTFversions: $[dependencies.getFormattedVersions.outputs['versionsOutput.umbracoTFversions']] - steps: - # TODO: DO we really need to install terraform again? - - task: TerraformInstaller@0 - displayName: Install Terraform - inputs: - terraformVersion: 'latest' - # We need to initialize again, otherwise the pipeline will fail. - - task: PowerShell@2 - displayName: Terraform Init - inputs: - targetType: 'inline' - script: 'terraform init -backend=false' - workingDirectory: '$(terraformWorkingDirectory)' - - task: AzureCLI@2 - name: AzLoginInfo - displayName: Gets Azure Login Credentials - inputs: - azureSubscription: '$(serviceConnection)' - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$servicePrincipalId" - echo "##vso[task.setvariable variable=ARM_CLIENT_SECRET]$servicePrincipalKey" - echo "##vso[task.setvariable variable=ARM_TENANT_ID]$tenantId" - addSpnToEnvironment: true - - task: TerraformTaskV4@4 + - stage: validateTestCases + displayName: Validate test cases + jobs: + - job: prepareTestCases + displayName: Validate + resolve test cases + steps: + - task: PowerShell@2 + name: out + displayName: Resolve profile + validate scenario + inputs: + targetType: 'filePath' + filePath: '$(System.DefaultWorkingDirectory)/scripts/resolve-run-config.ps1' + arguments: > + -Profile '${{ parameters.loadProfile }}' + -UmbracoVersion '${{ parameters.umbracoVersion }}' + -Scenario '${{ parameters.scenario }}' + -RunStarter '${{ parameters.runStarter }}' + -RunStandard '${{ parameters.runStandard }}' + -RunPro '${{ parameters.runPro }}' + -RunEnterprise '${{ parameters.runEnterprise }}' + -PoolDtuOverride '${{ parameters.poolDtuOverride }}' + -AppSkuOverride '${{ parameters.appSkuOverride }}' + -SeederPresetOverride '${{ parameters.seederPresetOverride }}' + -WorkspaceRoot '$(System.DefaultWorkingDirectory)' + + - stage: ensureHistoryInfra + displayName: Ensure history infrastructure + dependsOn: validateTestCases + jobs: + - job: ensure + displayName: Ensure history RG, Azure Load Testing, storage + steps: + - task: AzureCLI@2 + displayName: Ensure history infra + inputs: + azureSubscription: '$(serviceConnection)' + scriptType: 'pscore' + scriptLocation: 'scriptPath' + scriptPath: '$(System.DefaultWorkingDirectory)/scripts/ensure-history-infra.ps1' + arguments: > + -HistoryResourceGroup "$(historyResourceGroup)" + -HistoryLocation "$(historyLocation)" + -LoadTestName "$(historyLoadTestName)" + -StorageAccountName "$(historyStorageAccount)" + -ContainerName "$(historyContainer)" + + - stage: ensureMonitoringInfra + displayName: Ensure monitoring infrastructure + dependsOn: ensureHistoryInfra + jobs: + - job: ensure + displayName: Ensure Log Analytics + custom table + DCR + Workbook + steps: + - task: AzureCLI@2 + name: ensureMon + displayName: Ensure monitoring infra (workspace, table, DCR, role grant) + inputs: + azureSubscription: '$(serviceConnection)' + scriptType: 'pscore' + # addSpnToEnvironment exposes the service connection's app ID via + # $env:servicePrincipalId; we resolve the SP's *object* ID from + # that for the role assignment. + addSpnToEnvironment: true + scriptLocation: 'inlineScript' + inlineScript: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + + if ([string]::IsNullOrWhiteSpace($env:servicePrincipalId)) { + Write-Error "servicePrincipalId env var not set. Verify addSpnToEnvironment: true on this task." + exit 1 + } + $spObjectId = az ad sp show --id $env:servicePrincipalId --query id -o tsv + if ([string]::IsNullOrWhiteSpace($spObjectId)) { + Write-Error "Couldn't resolve SP object ID from app ID $env:servicePrincipalId. Tenant may require explicit Directory Readers role on the SP." + exit 1 + } + & "$(System.DefaultWorkingDirectory)/scripts/ensure-monitoring-infra.ps1" ` + -HistoryResourceGroup "$(historyResourceGroup)" ` + -HistoryLocation "$(historyLocation)" ` + -WorkspaceName "$(historyWorkspaceName)" ` + -DceName "$(historyDceName)" ` + -DcrName "$(historyDcrName)" ` + -IngestPrincipalId $spObjectId ` + -EmitPipelineVars + + - task: AzureCLI@2 + displayName: Deploy Workbook (idempotent — re-applies dashboards/loadtest.workbook.json) + inputs: + azureSubscription: '$(serviceConnection)' + scriptType: 'pscore' + scriptLocation: 'scriptPath' + scriptPath: '$(System.DefaultWorkingDirectory)/scripts/deploy-workbook.ps1' + arguments: > + -HistoryResourceGroup "$(historyResourceGroup)" + -HistoryLocation "$(historyLocation)" + -WorkspaceName "$(historyWorkspaceName)" + + - stage: provision + displayName: Provision infrastructure + dependsOn: + - validateTestCases + - ensureHistoryInfra + variables: + testCasesJson: $[ stageDependencies.validateTestCases.prepareTestCases.outputs['out.testCasesJson'] ] + resolvedSeederPreset: $[ stageDependencies.validateTestCases.prepareTestCases.outputs['out.resolvedSeederPreset'] ] + resolvedPoolDtuOverride: $[ stageDependencies.validateTestCases.prepareTestCases.outputs['out.resolvedPoolDtuOverride'] ] + resolvedAppSkuOverride: $[ stageDependencies.validateTestCases.prepareTestCases.outputs['out.resolvedAppSkuOverride'] ] + resolvedSdkVersion: $[ stageDependencies.validateTestCases.prepareTestCases.outputs['out.resolvedSdkVersion'] ] + jobs: + - job: checkResourceGroup + displayName: Check if resource group exists + steps: + - task: AzureCLI@2 + displayName: Fail if resource group already exists + inputs: + azureSubscription: '$(serviceConnection)' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Fail if the RG already exists — either a stuck previous run or a prefix collision. + # If it's a stuck run you want to discard, delete it directly: + # az group delete -n --yes + # then re-queue this pipeline. + $exists = az group exists -n $(azureResourceGroup) + if ($exists -eq 'true') { + Write-Host "##vso[task.logissue type=error]Resource group '$(azureResourceGroup)' already exists." + Write-Host "##vso[task.logissue type=error] Delete it manually with 'az group delete -n $(azureResourceGroup) --yes' (if it's from a stuck run), or use a different prefix." + exit 1 + } else { + Write-Host "Resource group '$(azureResourceGroup)' does not exist - safe to proceed." + } + + - job: setup + dependsOn: checkResourceGroup + displayName: Terraform Setup + steps: + - task: TerraformInstaller@1 + displayName: Install Terraform + inputs: + terraformVersion: '1.13.3' + + - task: PowerShell@2 + displayName: Terraform Init + inputs: + targetType: 'inline' + script: 'terraform init -backend=false' + workingDirectory: '$(terraformWorkingDirectory)' + + - task: TerraformTaskV4@4 + displayName: Terraform Validate + inputs: + provider: 'azurerm' + command: 'validate' + workingDirectory: '$(terraformWorkingDirectory)' + + - task: TerraformTaskV4@4 + displayName: Terraform Plan + inputs: + provider: 'azurerm' + command: 'plan' + workingDirectory: '$(terraformWorkingDirectory)' + environmentServiceNameAzureRM: '$(serviceConnection)' + commandOptions: '-parallelism=1 -var resource_name_prefix=${{ parameters.resourcePrefix }} -var resource_group_name=$(azureResourceGroup) -var resource_group_location="${{ parameters.azureRegion }}" -var seeder_preset=$(resolvedSeederPreset) -var pool_dtu_override=$(resolvedPoolDtuOverride) -var app_sku_override=$(resolvedAppSkuOverride) -var build_id=$(Build.BuildId) -var="test_cases=$(testCasesJson)"' + + - job: apply displayName: Terraform Apply - inputs: - provider: 'azurerm' - command: 'apply' - workingDirectory: '$(terraformWorkingDirectory)' - environmentServiceNameAzureRM: '$(serviceConnection)' - commandOptions: '-parallelism=1 -var resource_name_prefix=${{ parameters.prefix }} -var resource_group_name=$(azurergname) -var client_id=$(ARM_CLIENT_ID) -var client_secret=$(ARM_CLIENT_SECRET) -var tenant_id=$(ARM_TENANT_ID) -var app_service_plan_sku=${{ parameters.appServicePlanSku }} -var="umbraco_cms_versions=$(umbracoTFversions)"' - - task: TerraformTaskV4@4 - displayName: Terraform Output - inputs: - provider: 'azurerm' - command: 'output' - workingDirectory: '$(terraformWorkingDirectory)' - environmentServiceNameAzureRM: '$(serviceConnection)' - - task: PowerShell@2 - name: outputVars - displayName: Sets the output as variables - inputs: - targetType: 'inline' - script: | - Write-Host "##vso[task.setvariable variable=firstHostName;isOutput=true]$(terraform output -json hostnames | jq -r '.[0]')" - Write-Host "##vso[task.setvariable variable=secondHostName;isOutput=true]$(terraform output -json hostnames | jq -r '.[1]')" - Write-Host "##vso[task.setvariable variable=thirdHostName;isOutput=true]$(terraform output -json hostnames | jq -r '.[2]')" - Write-Host "##vso[task.setvariable variable=fourthHostName;isOutput=true]$(terraform output -json hostnames | jq -r '.[3]')" - - Write-Host "##vso[task.setvariable variable=firstUmbracoVersion;isOutput=true]$(terraform output -json cms_versions | jq -r '.[0]')" - Write-Host "##vso[task.setvariable variable=secondUmbracoVersion;isOutput=true]$(terraform output -json cms_versions | jq -r '.[1]')" - Write-Host "##vso[task.setvariable variable=thirdUmbracoVersion;isOutput=true]$(terraform output -json cms_versions | jq -r '.[2]')" - Write-Host "##vso[task.setvariable variable=fourthUmbracoVersion;isOutput=true]$(terraform output -json cms_versions | jq -r '.[3]')" - - Write-Host "##vso[task.setvariable variable=firstAppServiceName;isOutput=true]$(terraform output -json app_service_name | jq -r '.[0]')" - Write-Host "##vso[task.setvariable variable=secondAppServiceName;isOutput=true]$(terraform output -json app_service_name | jq -r '.[1]')" - Write-Host "##vso[task.setvariable variable=thirdAppServiceName;isOutput=true]$(terraform output -json app_service_name | jq -r '.[2]')" - Write-Host "##vso[task.setvariable variable=fourthAppServiceName;isOutput=true]$(terraform output -json app_service_name | jq -r '.[3]')" - workingDirectory: '$(terraformWorkingDirectory)' - - - - job: runLoadTest - displayName: Runs the load test - dependsOn: apply - variables: - firstHostName: $[dependencies.apply.outputs['outputVars.firstHostName']] - secondHostName: $[dependencies.apply.outputs['outputVars.secondHostName']] - thirdHostName: $[dependencies.apply.outputs['outputVars.thirdHostName']] - fourthHostName: $[dependencies.apply.outputs['outputVars.fourthHostName']] - - firstUmbracoVersion: $[dependencies.apply.outputs['outputVars.firstUmbracoVersion']] - secondUmbracoVersion: $[dependencies.apply.outputs['outputVars.secondUmbracoVersion']] - thirdUmbracoVersion: $[dependencies.apply.outputs['outputVars.thirdUmbracoVersion']] - fourthUmbracoVersion: $[dependencies.apply.outputs['outputVars.fourthUmbracoVersion']] - - firstAppServiceName: $[dependencies.apply.outputs['outputVars.firstAppServiceName']] - secondAppServiceName: $[dependencies.apply.outputs['outputVars.secondAppServiceName']] - thirdAppServiceName: $[dependencies.apply.outputs['outputVars.thirdAppServiceName']] - fourthAppServiceName: $[dependencies.apply.outputs['outputVars.fourthAppServiceName']] - steps: - # The load test are set up so we will be able to load test up to 4 different hostnames. A load test is only gonna run if the hostName variable is not null - - task: AzureCLI@2 - name: startAppServiceOne - condition: or(ne(variables['firstHostName'], 'null'), ne(variables['firstUmbracoVersion'], 'null')) - displayName: Starts the first hostname - inputs: - azureSubscription: '$(serviceConnection)' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: az webapp start -n $(firstAppServiceName) -g $(azurergname) - - task: PowerShell@2 - condition: or(ne(variables['firstHostName'], 'null'), ne(variables['firstUmbracoVersion'], 'null')) - displayName: Sleeps for 3 minutes - inputs: - targetType: 'inline' - script: Start-Sleep -Seconds 180 - - task: AzureLoadTest@1 - condition: or(ne(variables['firstHostName'], 'null'), ne(variables['firstUmbracoVersion'], 'null')) - displayName: LoadTest 1 - inputs: - azureSubscription: '$(serviceConnection)' - loadTestConfigFile: '$(loadTestFileLocation)' - resourceGroup: '$(azurergname)' - loadTestResource: $(loadTestResourceName) - loadTestRunName: '$(firstUmbracoVersion)' - loadTestRunDescription: 'We are load testing different versions of Umbraco' - env: | - [ - { - "name": "hostName", - "value": "$(firstHostName)" - }, - { - "name": "users", - "value": "${{ parameters.userAmount }}" - } - ] - - task: AzureCLI@2 - name: startAppServiceTwo - condition: or(ne(variables['secondHostName'], 'null'), ne(variables['secondUmbracoVersion'], 'null')) - displayName: Starts the second hostname - inputs: - azureSubscription: '$(serviceConnection)' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - az webapp stop -n $(firstAppServiceName) -g $(azurergname) - az webapp start -n $(secondAppServiceName) -g $(azurergname) - - task: PowerShell@2 - condition: or(ne(variables['secondHostName'], 'null'), ne(variables['secondUmbracoVersion'], 'null')) - displayName: Sleeps for 3 minutes - inputs: - targetType: 'inline' - script: Start-Sleep -Seconds 180 - - task: AzureLoadTest@1 - condition: or(ne(variables['secondHostName'], 'null'), ne(variables['secondUmbracoVersion'], 'null')) - displayName: LoadTest 2 - inputs: - azureSubscription: '$(serviceConnection)' - loadTestConfigFile: '$(loadTestFileLocation)' - resourceGroup: '$(azurergname)' - loadTestResource: $(loadTestResourceName) - loadTestRunName: '$(secondUmbracoVersion)' - loadTestRunDescription: 'We are load testing different versions of Umbraco' - env: | - [ - { - "name": "hostName", - "value": "$(secondHostName)" - }, - { - "name": "users", - "value": "${{ parameters.userAmount }}" - } - ] - - task: AzureCLI@2 - name: startAppServiceThree - condition: or(ne(variables['thirdHostName'], 'null'), ne(variables['thirdUmbracoVersion'], 'null')) - displayName: Starts the third hostname - inputs: - azureSubscription: '$(serviceConnection)' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - az webapp stop -n $(secondAppServiceName) -g $(azurergname) - az webapp start -n $(thirdAppServiceName) -g $(azurergname) - - task: PowerShell@2 - condition: or(ne(variables['thirdHostName'], 'null'), ne(variables['thirdUmbracoVersion'], 'null')) - displayName: Sleeps for 3 minutes - inputs: - targetType: 'inline' - script: Start-Sleep -Seconds 180 - - task: AzureLoadTest@1 - condition: or(ne(variables['thirdHostName'], 'null'), ne(variables['thirdUmbracoVersion'], 'null')) - displayName: LoadTest 3 - inputs: - azureSubscription: '$(serviceConnection)' - loadTestConfigFile: '$(loadTestFileLocation)' - resourceGroup: '$(azurergname)' - loadTestResource: $(loadTestResourceName) - loadTestRunName: '$(thirdUmbracoVersion)' - loadTestRunDescription: 'We are load testing different versions of Umbraco' - env: | - [ - { - "name": "hostName", - "value": "$(thirdHostName)" - }, - { - "name": "users", - "value": "${{ parameters.userAmount }}" - } - ] - - task: AzureCLI@2 - name: startAppServiceFour - condition: or(ne(variables['fourthHostName'], 'null'), ne(variables['fourthUmbracoVersion'], 'null')) - displayName: Starts the fourth hostname - inputs: - azureSubscription: '$(serviceConnection)' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - az webapp stop -n $(thirdAppServiceName) -g $(azurergname) - az webapp start -n $(fourthAppServiceName) -g $(azurergname) - - task: PowerShell@2 - condition: or(ne(variables['fourthHostName'], 'null'), ne(variables['fourthUmbracoVersion'], 'null')) - displayName: Sleeps for 3 minutes - inputs: - targetType: 'inline' - script: Start-Sleep -Seconds 180 - - task: AzureLoadTest@1 - condition: or(ne(variables['fourthHostName'], 'null'), ne(variables['fourthUmbracoVersion'], 'null')) - displayName: LoadTest 4 - inputs: - azureSubscription: '$(serviceConnection)' - loadTestConfigFile: '$(loadTestFileLocation)' - resourceGroup: '$(azurergname)' - loadTestResource: $(loadTestResourceName) - loadTestRunName: '$(fourthUmbracoVersion)' - loadTestRunDescription: 'We are load testing different versions of Umbraco' - env: | - [ - { - "name": "hostName", - "value": "$(fourthHostName)" - }, - { - "name": "users", - "value": "${{ parameters.userAmount }}" - } - ] - - - - job: doesResourceGroupExists - dependsOn: runLoadTest - condition: succeededOrFailed() - displayName: Checks if the resource group exists - steps: - - task: AzureCLI@2 - name: checkResourceGroup - displayName: Checks if the resource group exists - inputs: - azureSubscription: '$(serviceConnection)' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: 'Write-Host "##vso[task.setvariable variable=doesExist;isOutput=true]$(az group exists -n $(azurergname))"' - - job: manualValidationResourceGroup - dependsOn: doesResourceGroupExists - condition: eq(dependencies.doesResourceGroupExists.outputs['checkResourceGroup.doesExist'], true) - displayName: Manual validation for deleting the resource group - pool: server - timeoutInMinutes: 1440 - steps: - - task: ManualValidation@0 - displayName: Wait for external validation - timeoutInMinutes: 1440 - inputs: - # notifyUsers: aze@umbraco.dk - instructions: 'Please press "Resume" if you want to keep the resource group $(azurergname). If you press "Reject" or do not pick an option after 24 hours, The resource group $(azurergname), and all the content inside of the resource group will be deleted' - - - - job: deleteResourceGroup - displayName: Deletes the resource group - dependsOn: manualValidationResourceGroup - condition: eq(dependencies.manualValidationResourceGroup.result, 'failed') - steps: - - task: AzureCLI@2 - name: checkResourceGroupAgain - displayName: Checks if the resource group exists before deleting it - inputs: - azureSubscription: '$(serviceConnection)' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: 'Write-Host "##vso[task.setvariable variable=doesResourceGroupExist;isOutput=true]$(az group exists -n $(azurergname))"' - - task: AzureCLI@2 - displayName: Deletes the resource group - retryCountOnTaskFailure: 5 - inputs: - azureSubscription: '$(serviceConnection)' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: 'az group delete -n $(azurergname) --yes' \ No newline at end of file + dependsOn: setup + # Apply runs the install script per case sequentially (parallelism=1). + # Massive preset can take ~120 min per case; budget generously. + timeoutInMinutes: 720 + steps: + - task: TerraformInstaller@1 + displayName: Install Terraform + inputs: + terraformVersion: '1.13.3' + + - task: UseDotNet@2 + displayName: 'Install .NET SDK $(resolvedSdkVersion)' + inputs: + packageType: sdk + version: '$(resolvedSdkVersion)' + + - task: PowerShell@2 + displayName: Terraform Init + inputs: + targetType: 'inline' + script: 'terraform init -backend=false' + workingDirectory: '$(terraformWorkingDirectory)' + + - task: AzureCLI@2 + name: AzLoginInfo + displayName: Get Azure credentials + inputs: + azureSubscription: '$(serviceConnection)' + scriptType: bash + scriptLocation: inlineScript + # Under WIF, $idToken is set and $servicePrincipalKey is empty. + # Under client-secret auth, the opposite. issecret=true masks all. + # Subscription ID isn't surfaced by addSpnToEnvironment, so read it + # explicitly — install-umbraco-cms-on-appservice.ps1 pins `az account` + # to it after `az login` to avoid multi-sub default-routing surprises. + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$tenantId" + echo "##vso[task.setvariable variable=ARM_CLIENT_SECRET;issecret=true]$servicePrincipalKey" + echo "##vso[task.setvariable variable=ARM_OIDC_TOKEN;issecret=true]$idToken" + echo "##vso[task.setvariable variable=ARM_SUBSCRIPTION_ID]$(az account show --query id -o tsv)" + addSpnToEnvironment: true + + - task: TerraformTaskV4@4 + displayName: Terraform Apply + # ARM_* env vars inherit into the install script via local-exec child-process + # inheritance. Kept out of terraform variables (which would suppress local-exec + # output) and out of the local-exec `environment` block (same reason). + env: + ARM_CLIENT_ID: $(ARM_CLIENT_ID) + ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET) + ARM_TENANT_ID: $(ARM_TENANT_ID) + ARM_OIDC_TOKEN: $(ARM_OIDC_TOKEN) + ARM_SUBSCRIPTION_ID: $(ARM_SUBSCRIPTION_ID) + inputs: + provider: 'azurerm' + command: 'apply' + workingDirectory: '$(terraformWorkingDirectory)' + environmentServiceNameAzureRM: '$(serviceConnection)' + commandOptions: '-parallelism=1 -var resource_name_prefix=${{ parameters.resourcePrefix }} -var resource_group_name=$(azureResourceGroup) -var resource_group_location="${{ parameters.azureRegion }}" -var seeder_preset=$(resolvedSeederPreset) -var pool_dtu_override=$(resolvedPoolDtuOverride) -var app_sku_override=$(resolvedAppSkuOverride) -var build_id=$(Build.BuildId) -var="test_cases=$(testCasesJson)"' + + - task: PowerShell@2 + name: outputVars + displayName: Extract Terraform outputs + inputs: + targetType: 'inline' + workingDirectory: '$(terraformWorkingDirectory)' + script: | + # Compact map keyed by testCaseId; downstream jobs read it as one variable. + $raw = terraform output -json test_case_outputs + $compact = ($raw | ConvertFrom-Json | ConvertTo-Json -Compress -Depth 10) + Write-Host "##vso[task.setvariable variable=testCaseOutputs;isOutput=true]$compact" + + # Aggregate per-case seeder results into a single output variable + # so the loadTest stage (different agent, no shared filesystem) + # doesn't need a pipeline-artifact handoff just for one field per + # case. install-umbraco-cms-on-appservice.ps1 wrote + # .seeder-results/.json during local-exec; we + # gather them all here and emit one compact JSON output var. + $seederResultsDir = Join-Path "$(System.DefaultWorkingDirectory)" ".seeder-results" + $seederMap = @{} + if (Test-Path $seederResultsDir) { + foreach ($f in Get-ChildItem -Path $seederResultsDir -Filter '*.json') { + try { + $seederMap[$f.BaseName] = Get-Content $f.FullName -Raw | ConvertFrom-Json + } catch { + Write-Warning "Couldn't parse seeder result $($f.FullName): $($_.Exception.Message)" + } + } + } + $seederJson = ($seederMap | ConvertTo-Json -Compress -Depth 5) + Write-Host "Aggregated $($seederMap.Count) seeder result(s)" + Write-Host "##vso[task.setvariable variable=seederResults;isOutput=true]$seederJson" + + - stage: loadTest + displayName: Run load tests + dependsOn: + - validateTestCases + - ensureMonitoringInfra + - provision + # Only run if provision completed; if provision failed we skip straight to cleanup. + condition: succeeded('provision') + variables: + # Stage-level fan-out: every job in this stage inherits these. + # Resolved values come from validateTestCases; testCaseOutputs from provision.apply. + testCaseOutputs: $[ stageDependencies.provision.apply.outputs['outputVars.testCaseOutputs'] ] + seederResults: $[ stageDependencies.provision.apply.outputs['outputVars.seederResults'] ] + resolvedTestCases: $[ stageDependencies.validateTestCases.prepareTestCases.outputs['out.resolvedTestCases'] ] + resolvedDotnetVersion: $[ stageDependencies.validateTestCases.prepareTestCases.outputs['out.resolvedDotnetVersion'] ] + resolvedSeederPreset: $[ stageDependencies.validateTestCases.prepareTestCases.outputs['out.resolvedSeederPreset'] ] + resolvedEngineInstances: $[ stageDependencies.validateTestCases.prepareTestCases.outputs['out.resolvedEngineInstances'] ] + monitoringDceUri: $[ stageDependencies.ensureMonitoringInfra.ensure.outputs['ensureMon.MonitoringDceUri'] ] + monitoringDcrImmutableId: $[ stageDependencies.ensureMonitoringInfra.ensure.outputs['ensureMon.MonitoringDcrImmutableId'] ] + monitoringStreamName: $[ stageDependencies.ensureMonitoringInfra.ensure.outputs['ensureMon.MonitoringStreamName'] ] + monitoringSeriesStreamName: $[ stageDependencies.ensureMonitoringInfra.ensure.outputs['ensureMon.MonitoringSeriesStreamName'] ] + jobs: + - job: runLoadTests + displayName: Run Load Tests + # Each case: warm-up (~5 min) + load test (up to 600s) + publish. Up to four + # cases per queue (one per selected tier); budget generously. + timeoutInMinutes: 720 + steps: + # Tier expansion driven by the runStarter/runStandard/runPro checkboxes. + # Profile is independent and only sets load intensity (resolved via the + # runtime variables resolvedSeederPreset / resolvedEngineInstances etc.). + # Keep testCaseId format in sync with the runtime construction in + # prepare-test-cases.ps1. + - ${{ if eq(parameters.runStarter, true) }}: + - template: templates/load-test-job.yml + parameters: + testCaseId: '${{ parameters.umbracoVersion }}__Starter__${{ parameters.scenario }}' + tier: 'Starter' + scenario: '${{ parameters.scenario }}' + umbracoVersion: '${{ parameters.umbracoVersion }}' + skipWarmup: ${{ parameters.skipWarmup }} + logAnalyticsDceUri: $(monitoringDceUri) + logAnalyticsDcrImmutableId: $(monitoringDcrImmutableId) + logAnalyticsStreamName: $(monitoringStreamName) + logAnalyticsSeriesStreamName: $(monitoringSeriesStreamName) + - ${{ if eq(parameters.runStandard, true) }}: + - template: templates/load-test-job.yml + parameters: + testCaseId: '${{ parameters.umbracoVersion }}__Standard__${{ parameters.scenario }}' + tier: 'Standard' + scenario: '${{ parameters.scenario }}' + umbracoVersion: '${{ parameters.umbracoVersion }}' + skipWarmup: ${{ parameters.skipWarmup }} + logAnalyticsDceUri: $(monitoringDceUri) + logAnalyticsDcrImmutableId: $(monitoringDcrImmutableId) + logAnalyticsStreamName: $(monitoringStreamName) + logAnalyticsSeriesStreamName: $(monitoringSeriesStreamName) + - ${{ if eq(parameters.runPro, true) }}: + - template: templates/load-test-job.yml + parameters: + testCaseId: '${{ parameters.umbracoVersion }}__Pro__${{ parameters.scenario }}' + tier: 'Pro' + scenario: '${{ parameters.scenario }}' + umbracoVersion: '${{ parameters.umbracoVersion }}' + skipWarmup: ${{ parameters.skipWarmup }} + logAnalyticsDceUri: $(monitoringDceUri) + logAnalyticsDcrImmutableId: $(monitoringDcrImmutableId) + logAnalyticsStreamName: $(monitoringStreamName) + logAnalyticsSeriesStreamName: $(monitoringSeriesStreamName) + - ${{ if eq(parameters.runEnterprise, true) }}: + - template: templates/load-test-job.yml + parameters: + testCaseId: '${{ parameters.umbracoVersion }}__Enterprise__${{ parameters.scenario }}' + tier: 'Enterprise' + scenario: '${{ parameters.scenario }}' + umbracoVersion: '${{ parameters.umbracoVersion }}' + skipWarmup: ${{ parameters.skipWarmup }} + logAnalyticsDceUri: $(monitoringDceUri) + logAnalyticsDcrImmutableId: $(monitoringDcrImmutableId) + logAnalyticsStreamName: $(monitoringStreamName) + logAnalyticsSeriesStreamName: $(monitoringSeriesStreamName) + + # Stop all apps so none run idle during the manual validation window. + - task: AzureCLI@2 + displayName: 'Stop all App Services' + condition: succeededOrFailed() + env: + TEST_CASE_OUTPUTS: $(testCaseOutputs) + inputs: + azureSubscription: '$(serviceConnection)' + scriptType: 'pscore' + scriptLocation: 'scriptPath' + scriptPath: '$(System.DefaultWorkingDirectory)/scripts/stop-all-app-services.ps1' + arguments: > + -ResourceGroupName "$(azureResourceGroup)" + + # Compare the just-published run against the baseline-median in history + # storage. Fails the pipeline only when a cell exceeds a threshold AND has + # >= 3 prior runs to compare against — early in a scenario's life, every + # cell reports "insufficient baseline" and the script exits 0. Once + # baselines accrue per (version, tier, sampler), this becomes a real gate. + - stage: regression + displayName: Regression check + dependsOn: + - ensureMonitoringInfra + - loadTest + condition: succeeded('loadTest') + variables: + monitoringDceUri: $[ stageDependencies.ensureMonitoringInfra.ensure.outputs['ensureMon.MonitoringDceUri'] ] + monitoringDcrImmutableId: $[ stageDependencies.ensureMonitoringInfra.ensure.outputs['ensureMon.MonitoringDcrImmutableId'] ] + monitoringStreamName: $[ stageDependencies.ensureMonitoringInfra.ensure.outputs['ensureMon.MonitoringStreamName'] ] + jobs: + - job: regressionCheck + displayName: Regression check + steps: + - task: AzureCLI@2 + displayName: Compare run against baseline + inputs: + azureSubscription: '$(serviceConnection)' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + . "$(System.DefaultWorkingDirectory)/scripts/_helpers.ps1" + $major = Get-UmbracoMajor '${{ parameters.umbracoVersion }}' + & "$(System.DefaultWorkingDirectory)/scripts/check-regression.ps1" ` + -Scenario '${{ parameters.scenario }}' ` + -Major $major ` + -HistoryResourceGroup '$(historyResourceGroup)' ` + -StorageAccountName '$(historyStorageAccount)' ` + -ContainerName '$(historyContainer)' ` + -OutputPath "$(System.DefaultWorkingDirectory)/regression-report.md" ` + -LogAnalyticsDceUri '$(monitoringDceUri)' ` + -LogAnalyticsDcrImmutableId '$(monitoringDcrImmutableId)' ` + -LogAnalyticsStreamName '$(monitoringStreamName)' + - task: PublishBuildArtifacts@1 + displayName: Publish regression report + condition: succeededOrFailed() + continueOnError: true + inputs: + PathtoPublish: '$(System.DefaultWorkingDirectory)/regression-report.md' + ArtifactName: 'regression-report' + publishLocation: 'Container' + + # Cleanup must run on every outcome — including pipeline cancel. `always()` + # covers cancel; `succeededOrFailed()` would skip cleanup when the user + # cancels mid-run, orphaning the ephemeral RG until someone manually deletes + # it. Depends on provision so the chain still fires if loadTest/regression + # were skipped or failed; depends on loadTest+regression so cleanup doesn't + # race with them. + - stage: cleanup + displayName: Cleanup + dependsOn: + - provision + - loadTest + - regression + condition: always() + jobs: + - job: checkResourceGroupForCleanup + displayName: Check resource group for cleanup + steps: + - task: AzureCLI@2 + name: checkResourceGroup + displayName: Check if resource group exists + inputs: + azureSubscription: '$(serviceConnection)' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $exists = az group exists -n $(azureResourceGroup) + Write-Host "##vso[task.setvariable variable=doesExist;isOutput=true]$exists" + + - job: manualValidation + dependsOn: checkResourceGroupForCleanup + condition: eq(dependencies.checkResourceGroupForCleanup.outputs['checkResourceGroup.doesExist'], 'true') + displayName: Manual validation - keep resources? + pool: server + timeoutInMinutes: ${{ parameters.validationTimeoutMinutes }} + steps: + - task: ManualValidation@0 + displayName: Approve to keep resources + timeoutInMinutes: ${{ parameters.validationTimeoutMinutes }} + inputs: + instructions: | + Press "Resume" to KEEP the resource group $(azureResourceGroup). + Press "Reject" (or wait ${{ parameters.validationTimeoutMinutes }} minutes) to DELETE all resources. + + # User pressed Reject, the validation timed out, or the chain skipped: delete the RG. + - job: deleteResourceGroup + displayName: Delete resource group + dependsOn: manualValidation + condition: in(dependencies.manualValidation.result, 'Failed', 'Canceled', 'Skipped') + steps: + - task: AzureCLI@2 + displayName: Delete resource group + retryCountOnTaskFailure: 5 + inputs: + azureSubscription: '$(serviceConnection)' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $exists = az group exists -n $(azureResourceGroup) + if ($exists -eq 'true') { + Write-Host "Deleting resource group $(azureResourceGroup)..." + az group delete -n $(azureResourceGroup) --yes + } else { + Write-Host "Resource group $(azureResourceGroup) does not exist, skipping deletion." + } diff --git a/dashboards/loadtest.workbook.json b/dashboards/loadtest.workbook.json new file mode 100644 index 0000000..b9f1562 --- /dev/null +++ b/dashboards/loadtest.workbook.json @@ -0,0 +1,963 @@ +{ + "version": "Notebook/1.0", + "items": [ + { + "type": 1, + "content": { + "json": "# Umbraco Load Test Dashboard\nData source: `LoadTestSummary_CL` in the workspace below, populated by `scripts/publish-load-test-results.ps1` on every pipeline run. Open the **Glossary** tab for the full vocabulary." + }, + "name": "title" + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "parameters": [ + { + "id": "param-workspace", + "version": "KqlParameterItem/1.0", + "name": "Workspace", + "label": "Workspace", + "type": 5, + "isRequired": true, + "value": "__WORKSPACE_ID__", + "typeSettings": { + "additionalResourceOptions": [], + "includeAll": false, + "showDefault": true + }, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "param-timerange", + "version": "KqlParameterItem/1.0", + "name": "TimeRange", + "label": "Time range", + "type": 4, + "isRequired": true, + "value": { "durationMs": 604800000 }, + "typeSettings": { + "selectableValues": [ + { "durationMs": 86400000 }, + { "durationMs": 604800000 }, + { "durationMs": 2592000000 }, + { "durationMs": 7776000000 }, + { "durationMs": 31536000000 } + ] + } + }, + { + "id": "param-scenario", + "version": "KqlParameterItem/1.0", + "name": "Scenario", + "label": "Scenario", + "type": 2, + "isRequired": false, + "value": "*", + "query": "LoadTestSummary_CL\n| distinct scenario\n| where isnotempty(scenario)\n| order by scenario asc", + "crossComponentResources": ["{Workspace}"], + "typeSettings": { "additionalResourceOptions": [], "includeAll": true, "showDefault": false }, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "param-version", + "version": "KqlParameterItem/1.0", + "name": "Version", + "label": "Version", + "type": 2, + "isRequired": false, + "value": "*", + "query": "LoadTestSummary_CL\n| distinct umbraco_version\n| where isnotempty(umbraco_version)\n| order by parse_version(umbraco_version) asc", + "crossComponentResources": ["{Workspace}"], + "typeSettings": { "additionalResourceOptions": [], "includeAll": true, "showDefault": false }, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "param-tier", + "version": "KqlParameterItem/1.0", + "name": "Tier", + "label": "Tier", + "type": 2, + "isRequired": false, + "value": "*", + "query": "LoadTestSummary_CL\n| distinct infra_tier\n| where isnotempty(infra_tier)\n| order by infra_tier asc", + "crossComponentResources": ["{Workspace}"], + "typeSettings": { "additionalResourceOptions": [], "includeAll": true, "showDefault": false }, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + } + ], + "style": "pills", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "global-filters" + }, + { + "type": 1, + "content": { + "json": "_Filters above scope **Trends**, **Compare**, and **Runs**. **Tiers** and **Versions** have their own pickers; they default to the matching global values when set._" + }, + "name": "global-filters-scope-note" + }, + + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// One-row health banner. Always renders; green ✓ when data is reachable in\n// the selected Workspace, amber ⚠ when nothing matches — so a fresh deploy\n// with the wrong Workspace param or an empty table is obvious instead of\n// silently rendering every panel blank.\nprint n = toscalar(LoadTestSummary_CL | summarize count())\n| extend ['Data source'] = iif(n > 0, strcat('✓ Connected — ', tostring(n), ' record(s) in LoadTestSummary_CL'), '⚠ No data in LoadTestSummary_CL — check the Workspace parameter above, widen Time range, or run the pipeline first')\n| project ['Data source']", + "size": 4, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "Data source", "formatter": 18, "formatOptions": { "thresholdsOptions": "icons", "thresholdsGrid": [ + { "operator": "contains", "thresholdValue": "✓", "representation": "success" }, + { "operator": "contains", "thresholdValue": "⚠", "representation": "warning" }, + { "operator": "Default", "representation": "info" } + ] } } + ] + } + }, + "name": "workspace-health" + }, + + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "parameters": [ + { + "id": "param-selected-tab", + "version": "KqlParameterItem/1.0", + "name": "SelectedTab", + "label": "", + "type": 10, + "isRequired": true, + "value": "Trends", + "typeSettings": { "additionalResourceOptions": [], "showDefault": false }, + "jsonData": "[\n { \"value\": \"Trends\", \"label\": \"Trends\", \"selected\": true },\n { \"value\": \"Tiers\", \"label\": \"Tiers\" },\n { \"value\": \"Versions\", \"label\": \"Versions\" },\n { \"value\": \"Compare\", \"label\": \"Compare\" },\n { \"value\": \"Runs\", \"label\": \"Runs\" },\n { \"value\": \"Glossary\", \"label\": \"Glossary\" }\n]" + } + ], + "style": "pills" + }, + "name": "tab-bar" + }, + + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Latest-run summary card. Top-of-page glance answer to 'is the latest\n// run OK?' Pulls one row per (scenario × version × tier) bounded by the\n// global filter. Capacity status flags infra saturation (Server CPU /\n// memory / Database load peaks at 85/95% or error rate at 1%);\n// Regression surfaces the verdict posted by scripts/check-regression.ps1\n// for the same case.\n// publish-load-test-results.ps1 writes one row per sampler with the\n// run-level resource metrics (plan_*, sql_*) duplicated across every\n// row of the same run. arg_max picks one row per case and its resource\n// fields ARE the run-level values — no need to filter to an empty\n// scenario_name (which would exclude everything).\nlet runs = LoadTestSummary_CL\n | where TimeGenerated {TimeRange}\n | where '{Scenario}' == '*' or scenario == '{Scenario}'\n | where '{Version}' == '*' or umbraco_version == '{Version}'\n | where '{Tier}' == '*' or infra_tier == '{Tier}'\n | where parse_status == 'ok' and coalesce(cold_start, false) == false\n | extend run_key = tolower(trim(@'\\s+', tostring(run_id))), scenario_key = tolower(trim(@'\\s+', scenario)), version_key = tolower(trim(@'\\s+', umbraco_version)), tier_key = tolower(trim(@'\\s+', infra_tier))\n | summarize arg_max(TimeGenerated, *) by run_key, scenario_key, version_key, tier_key\n | top 5 by TimeGenerated desc;\nlet regression = LoadTestSummary_CL\n | where parse_status == 'regression_check'\n | extend run_key = tolower(trim(@'\\s+', tostring(run_id))), scenario_key = tolower(trim(@'\\s+', scenario)), version_key = tolower(trim(@'\\s+', umbraco_version)), tier_key = tolower(trim(@'\\s+', infra_tier))\n | summarize regression_status = take_any(regression_status), reg_ts = max(TimeGenerated) by run_key, scenario_key, version_key, tier_key;\nruns\n| join kind=leftouter regression on run_key, scenario_key, version_key, tier_key\n| extend run_error_rate = todouble(coalesce(error_rate, real(0)))\n| extend bn_peak = coalesce(max_of(plan_CpuPercentage_max, plan_MemoryPercentage_max, sql_dtu_consumption_percent_max, sql_cpu_percent_max, sql_log_write_percent_max, sql_physical_data_read_percent_max), real(0))\n// Decision-first column order: identifier, verdict triple (Capacity/Bottleneck/\n// Regression), then the supporting metric columns.\n| project Started = TimeGenerated,\n Scenario = scenario,\n Version = umbraco_version,\n Tier = infra_tier,\n ['Capacity status'] = case(\n run_error_rate >= 0.01\n or plan_CpuPercentage_max >= 95\n or plan_MemoryPercentage_max >= 95\n or sql_dtu_consumption_percent_max >= 95\n or sql_cpu_percent_max >= 95, 'Saturated',\n plan_CpuPercentage_max >= 85\n or plan_MemoryPercentage_max >= 85\n or sql_dtu_consumption_percent_max >= 85\n or sql_cpu_percent_max >= 85, 'Stressed',\n 'OK'),\n Bottleneck = case(\n bn_peak < 50, '—',\n plan_CpuPercentage_max == bn_peak, strcat('App CPU ', round(bn_peak, 0), '%'),\n plan_MemoryPercentage_max == bn_peak, strcat('Server memory ', round(bn_peak, 0), '%'),\n sql_dtu_consumption_percent_max == bn_peak, strcat('Database load ', round(bn_peak, 0), '%'),\n sql_cpu_percent_max == bn_peak, strcat('Database CPU ', round(bn_peak, 0), '%'),\n sql_log_write_percent_max == bn_peak, strcat('Database log-write ', round(bn_peak, 0), '%'),\n sql_physical_data_read_percent_max == bn_peak, strcat('Database physical reads ', round(bn_peak, 0), '%'),\n '—'),\n Regression = case(\n regression_status == 'regress', 'regression detected',\n regression_status == 'pass', 'no regression',\n regression_status == 'insufficient', 'not enough history',\n isnotempty(regression_status), regression_status,\n TimeGenerated > ago(15min), 'checking',\n 'not run'),\n ['Error %'] = round(run_error_rate * 100, 2),\n ['Run #'] = run_id,\n ['Server CPU avg %'] = round(plan_CpuPercentage_avg, 1),\n ['Server CPU peak %'] = round(plan_CpuPercentage_max, 1),\n ['Database load avg %'] = round(sql_dtu_consumption_percent_avg, 1),\n ['Database load peak %'] = round(sql_dtu_consumption_percent_max, 1)\n| order by Started desc", + "size": 0, + "title": "Latest 5 runs (chronological) — quick status check on the most recent activity across the global filter. Complements the issues panel above (which is severity-sorted and only shows what's flagged).", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "Server CPU avg %", "formatter": 8, "formatOptions": { "palette": "yellow", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Server CPU peak %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Database load avg %", "formatter": 8, "formatOptions": { "palette": "yellow", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Database load peak %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Error %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 5 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 2 } } }, + { "columnMatch": "Capacity status", "formatter": 18, "formatOptions": { "thresholdsOptions": "icons", "thresholdsGrid": [ + { "operator": "==", "thresholdValue": "Saturated", "representation": "critical" }, + { "operator": "==", "thresholdValue": "Stressed", "representation": "3" }, + { "operator": "Default", "representation": "success" } + ] } }, + { "columnMatch": "Regression", "formatter": 18, "formatOptions": { "thresholdsOptions": "icons", "thresholdsGrid": [ + { "operator": "==", "thresholdValue": "regression detected", "representation": "critical" }, + { "operator": "==", "thresholdValue": "not enough history", "representation": "3" }, + { "operator": "==", "thresholdValue": "checking", "representation": "info" }, + { "operator": "==", "thresholdValue": "not run", "representation": "unknown" }, + { "operator": "==", "thresholdValue": "no regression", "representation": "success" }, + { "operator": "Default", "representation": "unknown" } + ] } } + ] + } + }, + "name": "latest-run-card", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Trends" } + }, + + { + "type": 1, + "content": { + "json": "## Trends\nOne line per `(sampler × version × tier)` across recent runs. The latest-run card above is the at-a-glance capacity check; the chart below is the longitudinal view, with a foldout matrix showing median ± spread per cell. **Cold-start runs (queued with `skipWarmup: true`) are excluded by default — see the Runs tab to view them, or the Compare tab to pick one explicitly.**" + }, + "name": "trends-header", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Trends" } + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "parameters": [ + { + "id": "param-metric", + "version": "KqlParameterItem/1.0", + "name": "Metric", + "label": "Metric", + "type": 2, + "isRequired": true, + "value": "p95 (ms)", + "typeSettings": { + "additionalResourceOptions": [], + "showDefault": false + }, + "jsonData": "[\n { \"value\": \"p95 (ms)\", \"label\": \"p95 (ms)\" },\n { \"value\": \"p99 (ms)\", \"label\": \"p99 (ms)\" },\n { \"value\": \"avg (ms)\", \"label\": \"avg (ms)\" },\n { \"value\": \"error rate\", \"label\": \"error rate\" },\n { \"value\": \"Requests/sec\", \"label\": \"Requests/sec\" },\n { \"value\": \"Server CPU peak %\", \"label\": \"Server CPU peak %\" },\n { \"value\": \"Database load peak %\", \"label\": \"Database load peak %\" }\n]" + }, + { + "id": "param-sampler", + "version": "KqlParameterItem/1.0", + "name": "Sampler", + "label": "Sampler", + "type": 2, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "isRequired": true, + "value": ["*"], + "query": "// Dropdown is scoped by the global Scenario filter so picking a scenario\n// trims the sampler list to just that scenario's tasks. parse_status filter\n// keeps metadata-only / regression-check rows out — they have empty\n// scenario_name and would surface as an empty option.\nLoadTestSummary_CL\n| where '{Scenario}' == '*' or scenario == '{Scenario}'\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| distinct scenario_name\n| order by scenario_name asc", + "crossComponentResources": ["{Workspace}"], + "typeSettings": { "additionalResourceOptions": [], "includeAll": true, "showDefault": false, "selectAllValue": "*" }, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + } + ], + "style": "pills" + }, + "name": "trends-params", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Trends" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Run-indexed x-axis (NOT time): each Run # is a discrete categorical\n// position so runs are equally spaced regardless of how long elapsed\n// between them. Time-based axis crammed bursts and isolated long-gap\n// runs together making run-to-run trends unreadable. Chronological order\n// is preserved via the upstream `order by TimeGenerated asc`.\n//\n// Pivoted into wide-form (one numeric column per Sampler) so Workbook's\n// linechart treats each Sampler as its own series automatically. The\n// long-form (Run, Sampler, value) shape causes Workbook to collapse\n// samplers via Sum at each x-axis position.\n//\n// Metric translation inlined as a case() because KQL `column_ifexists`\n// requires a string literal — a let-bound variable doesn't compile.\nLoadTestSummary_CL\n| where TimeGenerated {TimeRange}\n| where '{Scenario}' == '*' or scenario == '{Scenario}'\n| where '{Version}' == '*' or umbraco_version == '{Version}'\n| where '{Tier}' == '*' or infra_tier == '{Tier}'\n| where '*' in ({Sampler}) or scenario_name in ({Sampler})\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| extend value = case(\n '{Metric}' == 'p95 (ms)', toreal(p95_ms),\n '{Metric}' == 'p99 (ms)', toreal(p99_ms),\n '{Metric}' == 'avg (ms)', toreal(avg_ms),\n '{Metric}' == 'error rate', toreal(error_rate),\n '{Metric}' == 'Requests/sec', toreal(requests_per_sec),\n '{Metric}' == 'Server CPU peak %', toreal(plan_CpuPercentage_max),\n '{Metric}' == 'Database load peak %', toreal(sql_dtu_consumption_percent_max),\n real(null))\n| where isnotnull(value)\n| extend Sampler = strcat(scenario_name, ' | ', umbraco_version, ' | ', infra_tier)\n| extend Run = strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd HH:mm'), ' #', run_id)\n| project Run, Sampler, value\n| evaluate pivot(Sampler, any(value))\n| order by Run asc", + "size": 0, + "title": "{Metric} per run — one line per (sampler · version · tier). Ignore the (Sum) totals next to each series in the legend; that's the chart engine summing values across runs, which isn't a meaningful number — read each series line directly.", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "linechart", + "chartSettings": { + "xAxis": "Run", + "showLegend": true + } + }, + "customWidth": "50", + "name": "trends-chart", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Trends" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Server-side resource pressure per run. Read alongside the latency chart\n// to the LEFT: if latency climbs as Server CPU / Database load does, you're\n// infra-bound; if they diverge, the regression is code-side. Run-indexed\n// x-axis matches the chart on the left and prevents Workbook's time-bin\n// aggregation from summing peak-percentages across rows (a 'Sum' line\n// showing 300%+ is meaningless).\n//\n// Unpivoted to long-form so the chart can render one line per\n// (metric × tier). Otherwise a multi-tier sweep zigzags between the\n// tiers' values on a single 'Server CPU' line.\n// Resource metrics are duplicated across all per-sampler rows of the\n// same run by the publish script, so arg_max by run_id returns valid\n// run-level values — picking any sampler row is fine.\nlet pressure = LoadTestSummary_CL\n | where TimeGenerated {TimeRange}\n | where '{Scenario}' == '*' or scenario == '{Scenario}'\n | where '{Version}' == '*' or umbraco_version == '{Version}'\n | where '{Tier}' == '*' or infra_tier == '{Tier}'\n | where parse_status == 'ok' and coalesce(cold_start, false) == false\n | summarize arg_max(TimeGenerated, *) by run_id, umbraco_version, infra_tier;\nunion\n (pressure | extend metric='Server CPU peak %', value=round(todouble(plan_CpuPercentage_max), 1)),\n (pressure | extend metric='Server memory peak %', value=round(todouble(plan_MemoryPercentage_max), 1)),\n (pressure | extend metric='Database load peak %', value=round(todouble(sql_dtu_consumption_percent_max), 1)),\n (pressure | extend metric='Database CPU peak %', value=round(todouble(sql_cpu_percent_max), 1))\n| where isnotnull(value)\n| extend Run = strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd HH:mm'), ' #', run_id), Series = strcat(metric, ' | ', infra_tier)\n| project Run, Series, value\n| evaluate pivot(Series, any(value))\n| order by Run asc", + "size": 0, + "title": "Resource pressure per run — Server CPU, Server memory, Database load, Database CPU peaks. (Ignore the (Sum) totals in the legend — chart artifact, not meaningful.)", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "linechart", + "chartSettings": { + "xAxis": "Run", + "showLegend": true, + "ySettings": { "min": 0, "max": 100, "numberFormatSettings": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 0 } } } + } + }, + "customWidth": "50", + "name": "trends-resources", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Trends" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// 'error rate' is 0..1 and rounds to '0 ± 0' at integer precision — use 3\n// decimals for fractional metrics, integers for everything else. Metric\n// translation inlined because KQL column_ifexists requires a literal.\nlet precision = iif('{Metric}' == 'error rate', 3, 0);\nLoadTestSummary_CL\n| where TimeGenerated {TimeRange}\n| where '{Scenario}' == '*' or scenario == '{Scenario}'\n| where '{Version}' == '*' or umbraco_version == '{Version}'\n| where '{Tier}' == '*' or infra_tier == '{Tier}'\n| where '*' in ({Sampler}) or scenario_name in ({Sampler})\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| extend value = case(\n '{Metric}' == 'p95 (ms)', toreal(p95_ms),\n '{Metric}' == 'p99 (ms)', toreal(p99_ms),\n '{Metric}' == 'avg (ms)', toreal(avg_ms),\n '{Metric}' == 'error rate', toreal(error_rate),\n '{Metric}' == 'Requests/sec', toreal(requests_per_sec),\n '{Metric}' == 'Server CPU peak %', toreal(plan_CpuPercentage_max),\n '{Metric}' == 'Database load peak %', toreal(sql_dtu_consumption_percent_max),\n real(null))\n| where isnotnull(value)\n| summarize med_raw = percentile(value, 50), sd_raw = stdev(value), n = count() by scenario_name, umbraco_version, infra_tier\n| extend med = round(med_raw, precision), sd = round(sd_raw, precision)\n// Stability is the coefficient of variation (stdev/median × 100) bucketed into\n// plain-language labels. Computed from pre-round values so a 3-decimal metric\n// like error rate doesn't lose precision to rounding. Thresholds align with the\n// default 10% regression-check sensitivity — cells noisier than that need\n// wider thresholds before a regression call is meaningful.\n| extend cv = iif(n < 5 or med_raw == 0, real(null), sd_raw / med_raw * 100.0)\n| extend Stability = case(\n isnull(cv), 'few runs',\n cv < 10, 'stable',\n cv < 25, 'moderate',\n 'noisy')\n| project Sampler = scenario_name, Version = umbraco_version, Tier = infra_tier, ['median ± spread (runs)'] = strcat(med, ' ± ', sd, ' (', n, ' runs)'), Stability\n| order by Sampler asc, Version asc, Tier asc", + "size": 0, + "title": "Matrix view — median {Metric} ± spread across runs, plus Stability per cell. Noisy cells mean run-to-run spread is wide enough that a small regression threshold won't be meaningful.", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "Stability", "formatter": 18, "formatOptions": { "thresholdsOptions": "icons", "thresholdsGrid": [ + { "operator": "==", "thresholdValue": "noisy", "representation": "critical" }, + { "operator": "==", "thresholdValue": "moderate", "representation": "3" }, + { "operator": "==", "thresholdValue": "stable", "representation": "success" }, + { "operator": "==", "thresholdValue": "few runs", "representation": "unknown" }, + { "operator": "Default", "representation": "info" } + ] } } + ] + } + }, + "name": "trends-matrix", + "styleSettings": { "showBorder": true }, + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Trends" } + }, + + { + "type": 1, + "content": { + "json": "## Tiers\nCompare tiers for one `(scenario, version)` combination. Answers \"what do I get for upgrading the tier?\" Pick the scenario and version below; the tables and chart aggregate across every matching run — medians for latency/throughput, plus a worst-case column on each resource peak so a single saturated run still flags Saturated. The **Runs** column shows how many runs back each median; cells with 1 run are single-run snapshots, 3+ runs is statistically meaningful. When the global filter at the top is narrowed, the pickers default to the matching scenario/version automatically. **Cold-start runs are excluded; see the Runs tab for those.**" + }, + "name": "tiers-header", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Tiers" } + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "parameters": [ + { + "id": "param-tiers-scenario", + "version": "KqlParameterItem/1.0", + "name": "TiersScenario", + "label": "Scenario", + "type": 2, + "isRequired": true, + "query": "// Inherits the global Scenario filter when set: with a global selection the\n// dropdown lists only that scenario and showDefault auto-picks it; with `*` it\n// shows the full universe. parse_status filter keeps metadata-only rows from\n// surfacing scenarios that never produced metrics — otherwise picking them\n// would render an empty Tiers chart.\nLoadTestSummary_CL\n| where '{Scenario}' == '*' or scenario == '{Scenario}'\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| distinct scenario\n| where isnotempty(scenario)\n| order by scenario asc", + "crossComponentResources": ["{Workspace}"], + "typeSettings": { "additionalResourceOptions": [], "showDefault": true }, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "param-tiers-version", + "version": "KqlParameterItem/1.0", + "name": "TiersVersion", + "label": "Version", + "type": 2, + "isRequired": true, + "query": "// Show all versions; if the global Version filter is set, sort the matching\n// version to the top so showDefault picks it. Otherwise newest-first.\nLoadTestSummary_CL\n| where scenario == '{TiersScenario}'\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| distinct umbraco_version\n| where isnotempty(umbraco_version)\n| order by iif(umbraco_version == '{Version}', 0, 1) asc, parse_version(umbraco_version) desc", + "crossComponentResources": ["{Workspace}"], + "typeSettings": { "additionalResourceOptions": [], "showDefault": true }, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "param-tiers-metric", + "version": "KqlParameterItem/1.0", + "name": "TiersMetric", + "label": "Metric", + "type": 2, + "isRequired": true, + "value": "p95 (ms)", + "jsonData": "[\n { \"value\": \"p95 (ms)\", \"label\": \"p95 (ms)\" },\n { \"value\": \"p99 (ms)\", \"label\": \"p99 (ms)\" },\n { \"value\": \"avg (ms)\", \"label\": \"avg (ms)\" }\n]" + } + ], + "style": "pills" + }, + "name": "tiers-params", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Tiers" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Resource metrics (plan_*, sql_*) and requests_per_sec are duplicated across\n// every per-sampler row of the same run by the publish script. Reduce to one\n// row per (tier, run_id) first, then aggregate by tier across runs.\n//\n// Peaks show worst-case (max across runs). Capacity status, Headroom, and\n// Bottleneck all derive from worst so a single saturated run flags the tier\n// — and the resource columns on the right tell you why.\nLoadTestSummary_CL\n| where scenario == '{TiersScenario}' and umbraco_version == '{TiersVersion}'\n| where parse_status == 'ok' and coalesce(cold_start, false) == false\n| summarize arg_max(TimeGenerated, *) by infra_tier, run_id\n| extend run_error_rate = todouble(coalesce(error_rate, real(0)))\n| summarize\n Runs = count(),\n ['Requests/sec'] = round(percentile(todouble(requests_per_sec), 50), 1),\n ['Server CPU avg %'] = round(percentile(plan_CpuPercentage_avg, 50), 1),\n ['Server CPU peak %'] = round(max(plan_CpuPercentage_max), 1),\n ['Server memory peak %'] = round(max(plan_MemoryPercentage_max), 1),\n ['Database load avg %'] = round(percentile(sql_dtu_consumption_percent_avg, 50), 1),\n ['Database load peak %'] = round(max(sql_dtu_consumption_percent_max), 1),\n ['Database CPU peak %'] = round(max(sql_cpu_percent_max), 1),\n ['Database physical reads peak %'] = round(max(sql_physical_data_read_percent_max), 1),\n ['Database log-write peak %'] = round(max(sql_log_write_percent_max), 1),\n ['Error %'] = round(max(run_error_rate) * 100, 2),\n // Seeder duration (s) — null on Skipped/Failed/TimedOut/pre-feature runs;\n // percentile ignores nulls so those don't drag the median to 0.\n // Drop zero/negative readings from the median - the seeder takes minutes\n // on every preset, so a 0 means the publish path captured ElapsedMs wrong\n // (or a historical bug). Letting it through would drag the median to 0.\n ['Seeder duration (s)'] = round(percentile(iif(seeder_duration_seconds > 0, seeder_duration_seconds, real(null)), 50), 0),\n // Helpers below feed Capacity status, Headroom, Bottleneck — dropped from\n // the projection at the end so they don't show in the table.\n cpu_worst = max(plan_CpuPercentage_max),\n mem_worst = max(plan_MemoryPercentage_max),\n sqldtu_worst = max(sql_dtu_consumption_percent_max),\n sqlcpu_worst = max(sql_cpu_percent_max),\n sqllog_worst = max(sql_log_write_percent_max),\n sqlread_worst = max(sql_physical_data_read_percent_max),\n err_worst = max(run_error_rate)\n by Tier = infra_tier\n| extend ['Capacity status'] = case(\n err_worst >= 0.01\n or cpu_worst >= 95\n or mem_worst >= 95\n or sqldtu_worst >= 95\n or sqlcpu_worst >= 95, 'Saturated',\n cpu_worst >= 85\n or mem_worst >= 85\n or sqldtu_worst >= 85\n or sqlcpu_worst >= 85, 'Stressed',\n 'OK')\n// bn_peak is the highest peak across ALL six resources (the four used for\n// Capacity status plus log-write and physical-reads). Headroom and Bottleneck\n// derive from it so the 'how much room' and 'what saturated' answers agree.\n| extend bn_peak = coalesce(max_of(cpu_worst, mem_worst, sqldtu_worst, sqlcpu_worst, sqllog_worst, sqlread_worst), real(0))\n| extend Headroom = round(100 - bn_peak, 0)\n| extend Bottleneck = case(\n bn_peak < 50, '—',\n cpu_worst == bn_peak, strcat('App CPU ', round(bn_peak, 0), '%'),\n mem_worst == bn_peak, strcat('Server memory ', round(bn_peak, 0), '%'),\n sqldtu_worst == bn_peak, strcat('Database load ', round(bn_peak, 0), '%'),\n sqlcpu_worst == bn_peak, strcat('Database CPU ', round(bn_peak, 0), '%'),\n sqllog_worst == bn_peak, strcat('Database log-write ', round(bn_peak, 0), '%'),\n sqlread_worst == bn_peak, strcat('Database physical reads ', round(bn_peak, 0), '%'),\n '—')\n// Decision-first column order: verdict, then context, then supporting evidence.\n| project Tier, ['Capacity status'], Headroom, Bottleneck,\n Runs, ['Requests/sec'],\n ['Server CPU avg %'], ['Server CPU peak %'],\n ['Server memory peak %'],\n ['Database load avg %'], ['Database load peak %'],\n ['Database CPU peak %'],\n ['Database physical reads peak %'],\n ['Database log-write peak %'],\n ['Error %'],\n ['Seeder duration (s)']\n// Sort by tier capacity rank, not alphabetical - upgrade progression should\n// read top-to-bottom (Starter -> Standard -> Pro -> Enterprise).\n| extend tier_rank = case(Tier == 'Starter', 1, Tier == 'Standard', 2, Tier == 'Pro', 3, Tier == 'Enterprise', 4, 99)\n| order by tier_rank asc\n| project-away tier_rank", + "size": 0, + "title": "Capacity verdict per tier — verdict columns on the left, supporting resource peaks on the right. Peaks are worst-case across runs (a single saturated run flags the tier).", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "Capacity status", "formatter": 18, "formatOptions": { "thresholdsOptions": "icons", "thresholdsGrid": [ + { "operator": "==", "thresholdValue": "Saturated", "representation": "critical" }, + { "operator": "==", "thresholdValue": "Stressed", "representation": "3" }, + { "operator": "Default", "representation": "success" } + ] } }, + { "columnMatch": "Headroom", "formatter": 8, "formatOptions": { "palette": "redToGreen", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 0 } } }, + { "columnMatch": "Runs", "formatter": 8, "formatOptions": { "palette": "blue", "min": 1, "max": 10 } }, + { "columnMatch": "Requests/sec", "formatter": 8, "formatOptions": { "palette": "blue", "min": 0, "max": 500 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Server CPU avg %", "formatter": 8, "formatOptions": { "palette": "yellow", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Server CPU peak %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Server memory peak %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Database load avg %", "formatter": 8, "formatOptions": { "palette": "yellow", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Database load peak %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Database CPU peak %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Database physical reads peak %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Database log-write peak %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 100 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "Error %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 5 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 2 } } }, + { "columnMatch": "Seeder duration (s)", "formatter": 8, "formatOptions": { "palette": "blue", "min": 0, "max": 3600 },"numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 0 } } } + ] + } + }, + "name": "tiers-saturation", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Tiers" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Long-form (Sampler, Tier, value) + `group: Tier` in chartSettings.\n// Pivoted-wide-form bars stack by default in Workbook; the explicit group\n// column renders them side-by-side. Median across all matching runs per\n// (sampler, tier).\nLoadTestSummary_CL\n| where scenario == '{TiersScenario}' and umbraco_version == '{TiersVersion}'\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| extend value = case(\n '{TiersMetric}' == 'p95 (ms)', toreal(p95_ms),\n '{TiersMetric}' == 'p99 (ms)', toreal(p99_ms),\n '{TiersMetric}' == 'avg (ms)', toreal(avg_ms),\n real(null))\n| where isnotnull(value)\n| summarize ['{TiersMetric}'] = round(percentile(value, 50), 0) by Sampler = scenario_name, Tier = infra_tier\n| order by Sampler asc, Tier asc", + "size": 0, + "title": "Median {TiersMetric} per sampler — one bar per tier. Ignore the (Sum) totals in the legend; that's the chart engine summing medians across samplers, which isn't a meaningful number — compare individual bars.", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "barchart", + "chartSettings": { + "xAxis": "Sampler", + "group": "Tier", + "createOtherGroup": 0, + "showLegend": true + } + }, + "name": "tiers-bar", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Tiers" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Per-sampler detail aggregated across runs. Run-level resource metrics\n// (Server CPU peak, Database load peak) are NOT shown here — they're\n// identical for every sampler in the same run, so they'd just repeat the\n// same number on every row. Those values live in the Capacity verdict\n// table above. median + p95 spread (stdev) give the typical reading and\n// how much the runs disagree; worst Error % surfaces a single bad run that\n// the median would otherwise hide.\nLoadTestSummary_CL\n| where scenario == '{TiersScenario}' and umbraco_version == '{TiersVersion}'\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| summarize\n Runs = count(),\n Requests = toint(percentile(request_count, 50)),\n ['Requests/sec'] = round(percentile(todouble(requests_per_sec), 50), 1),\n ['avg (ms)'] = round(percentile(avg_ms, 50), 0),\n ['p95 (ms)'] = round(percentile(p95_ms, 50), 0),\n ['p95 spread (ms)'] = round(stdev(p95_ms), 0),\n ['p99 (ms)'] = round(percentile(p99_ms, 50), 0),\n ['max (ms)'] = round(max(max_ms), 0),\n ['median Error %'] = round(percentile(error_rate, 50) * 100, 2),\n ['worst Error %'] = round(max(error_rate) * 100, 2)\n by Sampler = scenario_name, Tier = infra_tier\n| order by Sampler asc, Tier asc", + "size": 0, + "title": "Per-sampler detail per tier — medians across all matching runs.", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "Runs", "formatter": 8, "formatOptions": { "palette": "blue", "min": 1, "max": 10 } }, + { "columnMatch": "Requests", "formatter": 8, "formatOptions": { "palette": "blue", "min": 0, "max": 10000 } }, + { "columnMatch": "Requests/sec", "formatter": 8, "formatOptions": { "palette": "blue", "min": 0, "max": 200 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "p95 (ms)", "formatter": 8, "formatOptions": { "palette": "yellow", "min": 0, "max": 2000 } }, + { "columnMatch": "p95 spread (ms)", "formatter": 8, "formatOptions": { "palette": "yellow", "min": 0, "max": 500 } }, + { "columnMatch": "p99 (ms)", "formatter": 8, "formatOptions": { "palette": "orange", "min": 0, "max": 2000 } }, + { "columnMatch": "max (ms)", "formatter": 8, "formatOptions": { "palette": "redBright", "min": 0, "max": 10000 } }, + { "columnMatch": "median Error %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 5 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 2 } } }, + { "columnMatch": "worst Error %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 25 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 2 } } } + ] + } + }, + "name": "tiers-table", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Tiers" } + }, + + { + "type": 1, + "content": { + "json": "## Versions\nCompare Umbraco versions for one `(scenario, tier)` combination. Answers \"how does v17.3.0 compare to v17.0.0 / v18.0.0 on the same hardware running the same workload?\" Aggregates across every matching run in the table — the **Runs** column shows how many runs back each median: cells with 1 run are single-run snapshots; 3+ runs is statistically meaningful. When the global filter is narrowed, the pickers default to the matching scenario/tier automatically. **Cold-start runs are excluded; see the Runs tab for those.**" + }, + "name": "versions-header", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Versions" } + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "parameters": [ + { + "id": "param-versions-scenario", + "version": "KqlParameterItem/1.0", + "name": "VersionsScenario", + "label": "Scenario", + "type": 2, + "isRequired": true, + "query": "LoadTestSummary_CL\n| where '{Scenario}' == '*' or scenario == '{Scenario}'\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| distinct scenario\n| where isnotempty(scenario)\n| order by scenario asc", + "crossComponentResources": ["{Workspace}"], + "typeSettings": { "additionalResourceOptions": [], "showDefault": true }, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "param-versions-tier", + "version": "KqlParameterItem/1.0", + "name": "VersionsTier", + "label": "Tier", + "type": 2, + "isRequired": true, + "query": "// Show all tiers; if the global Tier filter is set, sort the matching\n// tier first so showDefault picks it. Otherwise alphabetical.\nLoadTestSummary_CL\n| where scenario == '{VersionsScenario}'\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| distinct infra_tier\n| where isnotempty(infra_tier)\n| extend sort_key = iif(infra_tier == '{Tier}', '0', strcat('1_', infra_tier))\n| order by sort_key asc\n| project infra_tier", + "crossComponentResources": ["{Workspace}"], + "typeSettings": { "additionalResourceOptions": [], "showDefault": true }, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "param-versions-baseline", + "version": "KqlParameterItem/1.0", + "name": "VersionsBaseline", + "label": "Baseline version", + "type": 2, + "isRequired": true, + "query": "LoadTestSummary_CL\n| where scenario == '{VersionsScenario}' and infra_tier == '{VersionsTier}'\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| distinct umbraco_version\n| order by parse_version(umbraco_version) asc", + "crossComponentResources": ["{Workspace}"], + "typeSettings": { "additionalResourceOptions": [], "showDefault": true }, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "param-versions-metric", + "version": "KqlParameterItem/1.0", + "name": "VersionsMetric", + "label": "Metric", + "type": 2, + "isRequired": true, + "value": "p95 (ms)", + "jsonData": "[\n { \"value\": \"p95 (ms)\", \"label\": \"p95 (ms)\" },\n { \"value\": \"p99 (ms)\", \"label\": \"p99 (ms)\" },\n { \"value\": \"avg (ms)\", \"label\": \"avg (ms)\" }\n]" + }, + { + "id": "param-versions-threshold", + "version": "KqlParameterItem/1.0", + "name": "VersionsThreshold", + "label": "Significant change (%)", + "type": 1, + "isRequired": true, + "value": "10" + } + ], + "style": "pills" + }, + "name": "versions-params", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Versions" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Long-form output (Sampler, Version, value) and `group: Version` in\n// chartSettings — pivoted-wide-form would stack the bars; long-form\n// with an explicit grouping column renders side-by-side reliably.\nLoadTestSummary_CL\n| where scenario == '{VersionsScenario}' and infra_tier == '{VersionsTier}'\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| extend value = case(\n '{VersionsMetric}' == 'p95 (ms)', toreal(p95_ms),\n '{VersionsMetric}' == 'p99 (ms)', toreal(p99_ms),\n '{VersionsMetric}' == 'avg (ms)', toreal(avg_ms),\n real(null))\n| where isnotnull(value)\n| summarize ['{VersionsMetric}'] = round(percentile(value, 50), 0) by Sampler = scenario_name, Version = umbraco_version\n| order by Sampler asc, parse_version(Version) asc", + "size": 0, + "title": "Median {VersionsMetric} per sampler — one bar per version. Ignore the (Sum) totals in the legend; that's the chart engine summing medians across samplers, which isn't a meaningful number — compare individual bars.", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "barchart", + "chartSettings": { + "xAxis": "Sampler", + "group": "Version", + "createOtherGroup": 0, + "showLegend": true, + "ySettings": { "numberFormatSettings": { "unit": 0, "options": { "style": "decimal", "useGrouping": false } } } + } + }, + "name": "versions-bar", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Versions" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Overall version-vs-baseline delta. For each non-baseline version,\n// compute median p95/p99 per sampler, take the average % delta against\n// the baseline version's median, then summarize across samplers as a\n// single per-version row. 'faster_p95' counts how many samplers got\n// better in the candidate; 'slower_p95' the opposite.\nlet baseline = LoadTestSummary_CL\n | where scenario == '{VersionsScenario}' and infra_tier == '{VersionsTier}' and umbraco_version == '{VersionsBaseline}'\n | where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n | summarize base_p95 = percentile(p95_ms, 50), base_p99 = percentile(p99_ms, 50) by scenario_name;\nLoadTestSummary_CL\n| where scenario == '{VersionsScenario}' and infra_tier == '{VersionsTier}' and umbraco_version != '{VersionsBaseline}'\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| summarize cand_p95 = percentile(p95_ms, 50), cand_p99 = percentile(p99_ms, 50), runs = count_distinct(run_id) by scenario_name, umbraco_version\n| join kind=inner baseline on scenario_name\n| extend p95_delta = (cand_p95 - base_p95) / base_p95 * 100,\n p99_delta = (cand_p99 - base_p99) / base_p99 * 100\n| summarize\n ['Samplers'] = count(),\n runs_min = min(runs),\n runs_max = max(runs),\n ['Avg p95 change %'] = round(avg(p95_delta), 0),\n ['Avg p99 change %'] = round(avg(p99_delta), 0),\n ['Samplers faster'] = countif(p95_delta < 0),\n ['Samplers slower'] = countif(p95_delta > 0)\n by Version = umbraco_version\n| extend ['Runs (per sampler)'] = iif(runs_min == runs_max, tostring(runs_min), strcat(runs_min, '–', runs_max))\n| project Version, Samplers, ['Runs (per sampler)'], ['Avg p95 change %'], ['Avg p99 change %'], ['Samplers faster'], ['Samplers slower']\n| order by Version asc", + "size": 0, + "title": "Overall — each version vs baseline {VersionsBaseline}", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "Avg p95 change %", "formatter": 18, "formatOptions": { "thresholdsOptions": "colors", "thresholdsGrid": [ + { "operator": "<", "thresholdValue": "-{VersionsThreshold:value}", "representation": "green" }, + { "operator": ">", "thresholdValue": "{VersionsThreshold:value}", "representation": "redBright" }, + { "operator": "Default", "representation": "transparent" } + ] } }, + { "columnMatch": "Avg p99 change %", "formatter": 18, "formatOptions": { "thresholdsOptions": "colors", "thresholdsGrid": [ + { "operator": "<", "thresholdValue": "-{VersionsThreshold:value}", "representation": "green" }, + { "operator": ">", "thresholdValue": "{VersionsThreshold:value}", "representation": "redBright" }, + { "operator": "Default", "representation": "transparent" } + ] } }, + { "columnMatch": "Samplers faster", "formatter": 8, "formatOptions": { "palette": "green", "min": 0, "max": 10 } }, + { "columnMatch": "Samplers slower", "formatter": 8, "formatOptions": { "palette": "redBright", "min": 0, "max": 10 } } + ] + } + }, + "name": "versions-overall", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Versions" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// median + worst Error %: median represents typical behavior, worst surfaces\n// catastrophic single-run outliers that the median would otherwise hide\n// (e.g. one run at 100% error rate in a 5-run sample → median 0%, worst 100%).\nLoadTestSummary_CL\n| where scenario == '{VersionsScenario}' and infra_tier == '{VersionsTier}'\n| where isnotempty(scenario_name) and parse_status == 'ok' and coalesce(cold_start, false) == false\n| summarize\n Runs = count(),\n ['median p95 (ms)'] = round(percentile(p95_ms, 50), 0),\n ['p95 spread (ms)'] = round(stdev(p95_ms), 0),\n ['median p99 (ms)'] = round(percentile(p99_ms, 50), 0),\n ['median Error %'] = round(percentile(error_rate, 50) * 100, 1),\n ['worst Error %'] = round(max(error_rate) * 100, 1)\n by Sampler = scenario_name, Version = umbraco_version\n| order by Sampler asc, Version asc", + "size": 0, + "title": "Per-sampler stats per version", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "Runs", "formatter": 8, "formatOptions": { "palette": "blue", "min": 1, "max": 10 } }, + { "columnMatch": "median Error %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 5 } }, + { "columnMatch": "worst Error %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 25 } } + ] + } + }, + "name": "versions-table", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Versions" } + }, + + { + "type": 1, + "content": { + "json": "## Compare\nPick two runs to see per-sampler + server-side changes. The dropdowns honour the global **Time range** but ignore the other global filters — each entry is one `(run × scenario × version × tier)`, so a multi-case pipeline run shows up as several picks." + }, + "name": "compare-header", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Compare" } + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "parameters": [ + { + "id": "param-compare-baseline", + "version": "KqlParameterItem/1.0", + "name": "Baseline", + "label": "Baseline run", + "type": 2, + "isRequired": true, + "typeSettings": { "additionalResourceOptions": [], "showDefault": true }, + "query": "// One row per (run_id × scenario × version × tier) so a pipeline run that\n// swept multiple cases gets one dropdown entry per case — picking the wrong\n// one would otherwise mix cases via scenario_name joins. value is a composite\n// 'run_id|scenario|version|tier' the comparison queries split on.\n// Baseline default = second-most-recent (the comparison reference); the\n// most-recent run is hidden so showDefault picks the previous one, giving\n// 'previous vs newest' as the default comparison instead of 'newest vs\n// newest' which would render an identity-delta table.\n// parse_status filter excludes failed runs (no_metrics, no_results_dir) - they\n// have a metadata-only row but no sampler data, so picking one as baseline or\n// candidate yields an empty comparison.\nlet pool = LoadTestSummary_CL\n | where TimeGenerated {TimeRange}\n | where parse_status == 'ok'\n | where isnotempty(scenario) and isnotempty(umbraco_version) and isnotempty(infra_tier)\n | summarize arg_max(TimeGenerated, *) by run_id, scenario, umbraco_version, infra_tier;\nlet newest_ts = toscalar(pool | summarize max(TimeGenerated));\npool\n| where TimeGenerated < newest_ts\n| project value = strcat(run_id, '|', scenario, '|', umbraco_version, '|', infra_tier),\n label = strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd HH:mm'), ' · ', scenario, ' / ', umbraco_version, ' / ', infra_tier, ' (#', run_id, ')'),\n TimeGenerated\n| order by TimeGenerated desc\n| project value, label", + "crossComponentResources": ["{Workspace}"], + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "param-compare-candidate", + "version": "KqlParameterItem/1.0", + "name": "Candidate", + "label": "Candidate run", + "type": 2, + "isRequired": true, + "typeSettings": { "additionalResourceOptions": [], "showDefault": true }, + "query": "// Candidate is the run under test. Default = most recent, so 'previous vs\n// newest' is the out-of-the-box comparison (Baseline excludes the most\n// recent for exactly this reason). parse_status filter mirrors the Baseline\n// dropdown - failed runs aren't comparable.\nLoadTestSummary_CL\n| where TimeGenerated {TimeRange}\n| where parse_status == 'ok'\n| where isnotempty(scenario) and isnotempty(umbraco_version) and isnotempty(infra_tier)\n| summarize arg_max(TimeGenerated, *) by run_id, scenario, umbraco_version, infra_tier\n| project value = strcat(run_id, '|', scenario, '|', umbraco_version, '|', infra_tier),\n label = strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd HH:mm'), ' · ', scenario, ' / ', umbraco_version, ' / ', infra_tier, ' (#', run_id, ')'),\n TimeGenerated\n| order by TimeGenerated desc\n| project value, label", + "crossComponentResources": ["{Workspace}"], + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "param-compare-threshold", + "version": "KqlParameterItem/1.0", + "name": "Threshold", + "label": "Significant change (%)", + "type": 1, + "isRequired": true, + "value": "10" + }, + { + "id": "param-repo-compare-url", + "version": "KqlParameterItem/1.0", + "name": "RepoCompareUrl", + "label": "Repo compare URL prefix", + "description": "Optional URL prefix that turns baseline+candidate commit SHAs into a clickable diff. GitHub format: `https://github.com///compare/`. AzDO format: `https://dev.azure.com///_git//branchCompare?baseVersion=GC&targetVersion=GC&_a=files&baseVersionType=GC&targetVersionType=GC&baseVersion=`. The SHAs are appended as `...` (GitHub style); leave blank to disable.", + "type": 1, + "isRequired": false, + "value": "" + } + ], + "style": "pills" + }, + "name": "compare-params", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Compare" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Header rows showing exactly which case is on each side, so the reviewer\n// can confirm they're comparing like-for-like. Commit comes from the per-\n// run summary row. The 'Diff' column is populated only when RepoCompareUrl\n// is set, producing a clickable baseline..candidate URL.\nlet b_parts = split('{Baseline}', '|');\nlet c_parts = split('{Candidate}', '|');\nlet b_commit = toscalar(LoadTestSummary_CL\n | where run_id == tostring(b_parts[0]) and scenario == tostring(b_parts[1]) and umbraco_version == tostring(b_parts[2]) and infra_tier == tostring(b_parts[3])\n | extend safe_commit = tostring(column_ifexists(\"commit\", \"\"))\n | summarize arg_max(TimeGenerated, safe_commit)\n | project safe_commit);\nlet c_commit = toscalar(LoadTestSummary_CL\n | where run_id == tostring(c_parts[0]) and scenario == tostring(c_parts[1]) and umbraco_version == tostring(c_parts[2]) and infra_tier == tostring(c_parts[3])\n | extend safe_commit = tostring(column_ifexists(\"commit\", \"\"))\n | summarize arg_max(TimeGenerated, safe_commit)\n | project safe_commit);\nlet diff_url = iif(strlen('{RepoCompareUrl}') > 0 and strlen(b_commit) > 0 and strlen(c_commit) > 0, strcat('{RepoCompareUrl}', substring(b_commit, 0, 8), '...', substring(c_commit, 0, 8)), '');\nprint Side = 'Baseline', ['Run #'] = tostring(b_parts[0]), Scenario = tostring(b_parts[1]), Version = tostring(b_parts[2]), Tier = tostring(b_parts[3]), Commit = substring(b_commit, 0, 8), Diff = ''\n| union (print Side = 'Candidate', ['Run #'] = tostring(c_parts[0]), Scenario = tostring(c_parts[1]), Version = tostring(c_parts[2]), Tier = tostring(c_parts[3]), Commit = substring(c_commit, 0, 8), Diff = diff_url)", + "size": 4, + "title": "Comparing", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "Diff", "formatter": 7, "formatOptions": { "linkTarget": "Url", "linkIsContextBlade": false } } + ] + } + }, + "name": "compare-summary", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Compare" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// {Baseline}/{Candidate} values are composite keys 'run_id|scenario|version|tier'\n// produced by the dropdown queries above. Split here so a single dropdown\n// pick targets exactly one case of one run, not 'all cases of run_id'.\nlet b_parts = split('{Baseline}', '|');\nlet c_parts = split('{Candidate}', '|');\nlet baseline = LoadTestSummary_CL\n | where run_id == tostring(b_parts[0]) and scenario == tostring(b_parts[1]) and umbraco_version == tostring(b_parts[2]) and infra_tier == tostring(b_parts[3])\n | where isnotempty(scenario_name);\nlet candidate = LoadTestSummary_CL\n | where run_id == tostring(c_parts[0]) and scenario == tostring(c_parts[1]) and umbraco_version == tostring(c_parts[2]) and infra_tier == tostring(c_parts[3])\n | where isnotempty(scenario_name);\nlet delta = (a:real, b:real) { iif(a == 0 or isnull(a) or isnull(b), real(null), round((b - a) / a * 100.0, 0)) };\nbaseline\n| join kind=fullouter (candidate) on scenario_name\n// Columns grouped per metric (raw baseline, raw candidate, change %) so each\n// row reads as 'for this sampler: avg story, p95 story, p99 story' instead\n// of jumping between far-apart raw and % blocks.\n| project Sampler = coalesce(scenario_name, scenario_name1),\n ['Reqs baseline'] = request_count, ['Reqs candidate'] = request_count1,\n ['avg baseline (ms)'] = round(avg_ms, 0), ['avg candidate (ms)'] = round(avg_ms1, 0), ['avg change %'] = delta(avg_ms, avg_ms1),\n ['p95 baseline (ms)'] = round(p95_ms, 0), ['p95 candidate (ms)'] = round(p95_ms1, 0), ['p95 change %'] = delta(p95_ms, p95_ms1),\n ['p99 baseline (ms)'] = round(p99_ms, 0), ['p99 candidate (ms)'] = round(p99_ms1, 0), ['p99 change %'] = delta(p99_ms, p99_ms1)\n| order by Sampler asc", + "size": 0, + "title": "Per-sampler change", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "avg change %", "formatter": 18, "formatOptions": { "thresholdsOptions": "colors", "thresholdsGrid": [ + { "operator": "<", "thresholdValue": "-{Threshold:value}", "representation": "green" }, + { "operator": ">", "thresholdValue": "{Threshold:value}", "representation": "redBright" }, + { "operator": "Default", "representation": "transparent" } + ] } }, + { "columnMatch": "p95 change %", "formatter": 18, "formatOptions": { "thresholdsOptions": "colors", "thresholdsGrid": [ + { "operator": "<", "thresholdValue": "-{Threshold:value}", "representation": "green" }, + { "operator": ">", "thresholdValue": "{Threshold:value}", "representation": "redBright" }, + { "operator": "Default", "representation": "transparent" } + ] } }, + { "columnMatch": "p99 change %", "formatter": 18, "formatOptions": { "thresholdsOptions": "colors", "thresholdsGrid": [ + { "operator": "<", "thresholdValue": "-{Threshold:value}", "representation": "green" }, + { "operator": ">", "thresholdValue": "{Threshold:value}", "representation": "redBright" }, + { "operator": "Default", "representation": "transparent" } + ] } } + ] + } + }, + "name": "compare-table", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Compare" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Long-form output (Sampler, Series, p95 ms) instead of pivoting Series\n// into columns. Workbook bar charts render pivoted multi-numeric columns\n// as stacked by default; long-form with a categorical Series column\n// renders side-by-side bars reliably.\nlet b_parts = split('{Baseline}', '|');\nlet c_parts = split('{Candidate}', '|');\nlet baseline = LoadTestSummary_CL\n | where run_id == tostring(b_parts[0]) and scenario == tostring(b_parts[1]) and umbraco_version == tostring(b_parts[2]) and infra_tier == tostring(b_parts[3])\n | where isnotempty(scenario_name);\nlet candidate = LoadTestSummary_CL\n | where run_id == tostring(c_parts[0]) and scenario == tostring(c_parts[1]) and umbraco_version == tostring(c_parts[2]) and infra_tier == tostring(c_parts[3])\n | where isnotempty(scenario_name);\nunion\n (baseline | project Sampler = scenario_name, Series = 'Baseline', ['p95 (ms)'] = todouble(p95_ms)),\n (candidate | project Sampler = scenario_name, Series = 'Candidate', ['p95 (ms)'] = todouble(p95_ms))\n| where isnotnull(['p95 (ms)'])\n| order by Sampler asc, Series asc", + "size": 0, + "title": "p95 per sampler — baseline vs candidate. Ignore the (Sum) totals in the legend; that's the chart engine summing p95 values across samplers, which isn't a meaningful number — compare individual bars.", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "barchart", + "chartSettings": { + "xAxis": "Sampler", + "group": "Series", + "createOtherGroup": 0, + "showLegend": true, + "ySettings": { "numberFormatSettings": { "unit": 0, "options": { "style": "decimal", "useGrouping": false } } } + } + }, + "name": "compare-bar", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Compare" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Resource metrics live on every per-sampler row of a given run (the\n// publish script duplicates them), so arg_max picks any sampler row and\n// the run-level resource fields are correct.\nlet b_parts = split('{Baseline}', '|');\nlet c_parts = split('{Candidate}', '|');\nlet baseline = LoadTestSummary_CL\n | where run_id == tostring(b_parts[0]) and scenario == tostring(b_parts[1]) and umbraco_version == tostring(b_parts[2]) and infra_tier == tostring(b_parts[3])\n | where parse_status == 'ok' and coalesce(cold_start, false) == false\n | summarize arg_max(TimeGenerated, *);\nlet candidate = LoadTestSummary_CL\n | where run_id == tostring(c_parts[0]) and scenario == tostring(c_parts[1]) and umbraco_version == tostring(c_parts[2]) and infra_tier == tostring(c_parts[3])\n | where parse_status == 'ok' and coalesce(cold_start, false) == false\n | summarize arg_max(TimeGenerated, *);\nunion\n (baseline | project Metric='Server CPU peak %', Baseline=plan_CpuPercentage_max, Candidate=toscalar(candidate | project plan_CpuPercentage_max)),\n (baseline | project Metric='Server memory peak %', Baseline=plan_MemoryPercentage_max, Candidate=toscalar(candidate | project plan_MemoryPercentage_max)),\n (baseline | project Metric='Database load peak %', Baseline=sql_dtu_consumption_percent_max, Candidate=toscalar(candidate | project sql_dtu_consumption_percent_max)),\n (baseline | project Metric='Database CPU peak %', Baseline=sql_cpu_percent_max, Candidate=toscalar(candidate | project sql_cpu_percent_max)),\n (baseline | project Metric='Database log-write peak %', Baseline=sql_log_write_percent_max, Candidate=toscalar(candidate | project sql_log_write_percent_max)),\n (baseline | project Metric='Database physical reads peak %', Baseline=sql_physical_data_read_percent_max, Candidate=toscalar(candidate | project sql_physical_data_read_percent_max)),\n (baseline | project Metric='Server HTTP 4xx peak', Baseline=app_Http4xx_max, Candidate=toscalar(candidate | project app_Http4xx_max)),\n (baseline | project Metric='Server HTTP 5xx peak', Baseline=app_Http5xx_max, Candidate=toscalar(candidate | project app_Http5xx_max))\n| extend Baseline = round(Baseline, 1), Candidate = round(Candidate, 1)\n| extend ['change %'] = iif(Baseline == 0 or isnull(Baseline) or isnull(Candidate), real(null), round((Candidate - Baseline) / Baseline * 100.0, 0)),\n Note = case(\n isnull(Baseline) and isnull(Candidate), 'no data either side',\n isnull(Baseline), 'no baseline data',\n isnull(Candidate), 'no candidate data',\n Baseline == 0 and Candidate == 0, 'both zero',\n Baseline == 0, 'baseline zero',\n '')", + "size": 0, + "title": "Server-side change", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "change %", "formatter": 18, "formatOptions": { "thresholdsOptions": "colors", "thresholdsGrid": [ + { "operator": "<", "thresholdValue": "-{Threshold:value}", "representation": "green" }, + { "operator": ">", "thresholdValue": "{Threshold:value}", "representation": "redBright" }, + { "operator": "Default", "representation": "transparent" } + ] } } + ] + } + }, + "name": "compare-server", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Compare" } + }, + + { + "type": 1, + "content": { + "json": "## Runs\nAll runs matching the global filter. To inspect a specific run's per-sampler detail, pick it from the **Drill into run #** dropdown below — the detail panel only renders when one is selected. The two **URL prefix** fields are optional: set them once to turn Run # → AzDO build links and Commit short-SHAs → repo commit links in the table." + }, + "name": "runs-header", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Runs" } + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "parameters": [ + { + "id": "param-drill-run", + "version": "KqlParameterItem/1.0", + "name": "DrillRun", + "label": "Drill into run #", + "type": 2, + "isRequired": false, + "value": "", + "typeSettings": { "additionalResourceOptions": [], "showDefault": false }, + "query": "// Dropdown sourced from the same runs the table above shows, in the same\n// time range and global filter scope. value = run_id (string) so the drill\n// queries below can compare directly; label adds context so the right run\n// is easy to spot. Includes failed runs so the per-sampler detail panel\n// can still surface 'no rows' clearly for them.\nLoadTestSummary_CL\n| where TimeGenerated {TimeRange}\n| where '{Scenario}' == '*' or scenario == '{Scenario}'\n| where '{Version}' == '*' or umbraco_version == '{Version}'\n| where '{Tier}' == '*' or infra_tier == '{Tier}'\n| where parse_status in ('ok', 'no_metrics', 'no_results_dir')\n| summarize arg_max(TimeGenerated, scenario, umbraco_version, infra_tier) by run_id\n| project value = tostring(run_id),\n label = strcat(format_datetime(TimeGenerated, 'yyyy-MM-dd HH:mm'), ' #', run_id, ' · ', scenario, ' / ', umbraco_version, ' / ', infra_tier),\n TimeGenerated\n| order by TimeGenerated desc\n| project value, label", + "crossComponentResources": ["{Workspace}"], + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "param-azdo-org-url", + "version": "KqlParameterItem/1.0", + "name": "AzdoOrgUrl", + "label": "AzDO build URL prefix (ends in ?buildId=)", + "description": "Optional URL prefix — NOT a run number. Paste your pipeline's full result URL up to and including `?buildId=`, e.g. `https://dev.azure.com/myorg/myproject/_build/results?buildId=`. The run number from each row is appended automatically, turning Run # cells in the table below into clickable links. Leave blank to disable.", + "type": 1, + "isRequired": false, + "value": "" + }, + { + "id": "param-repo-commit-url", + "version": "KqlParameterItem/1.0", + "name": "RepoCommitUrl", + "label": "Repo commit URL prefix", + "description": "Optional URL prefix that turns the Commit short-SHA into a clickable link. GitHub format: `https://github.com///commit/`. AzDO format: `https://dev.azure.com///_git//commit/`. The commit SHA is appended automatically. Leave blank to disable.", + "type": 1, + "isRequired": false, + "value": "" + } + ], + "style": "pills" + }, + "name": "runs-params", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Runs" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Group by (run_id × scenario × version × tier) so a pipeline run that\n// covered multiple cases shows one row per case. parse_status filter\n// excludes the regression_check rows from the load-test summary, then we\n// leftouter-join them back to surface the regression verdict per case.\nlet runs = LoadTestSummary_CL\n | where TimeGenerated {TimeRange}\n | where '{Scenario}' == '*' or scenario == '{Scenario}'\n | where '{Version}' == '*' or umbraco_version == '{Version}'\n | where '{Tier}' == '*' or infra_tier == '{Tier}'\n | where parse_status in ('ok', 'no_metrics', 'no_results_dir')\n | extend run_key = tolower(trim(@'\\s+', tostring(run_id))), scenario_key = tolower(trim(@'\\s+', scenario)), version_key = tolower(trim(@'\\s+', umbraco_version)), tier_key = tolower(trim(@'\\s+', infra_tier))\n | summarize arg_max(TimeGenerated, *) by run_key, scenario_key, version_key, tier_key;\nlet regression = LoadTestSummary_CL\n | where parse_status == 'regression_check'\n | extend run_key = tolower(trim(@'\\s+', tostring(run_id))), scenario_key = tolower(trim(@'\\s+', scenario)), version_key = tolower(trim(@'\\s+', umbraco_version)), tier_key = tolower(trim(@'\\s+', infra_tier))\n | summarize regression_status = take_any(regression_status), regressed_count = take_any(regressed_count), reg_ts = max(TimeGenerated) by run_key, scenario_key, version_key, tier_key;\nruns\n| join kind=leftouter regression on run_key, scenario_key, version_key, tier_key\n| extend ['Build URL'] = iif(strlen('{AzdoOrgUrl}') > 0, strcat('{AzdoOrgUrl}', run_id), '')\n| extend bn_peak = coalesce(max_of(plan_CpuPercentage_max, plan_MemoryPercentage_max, sql_dtu_consumption_percent_max, sql_cpu_percent_max, sql_log_write_percent_max, sql_physical_data_read_percent_max), real(0))\n// Decision-first column order: row identifier, then verdict triple\n// (Regression / Bottleneck / Regressed #), then reference / metadata columns.\n| project Started=TimeGenerated,\n Scenario=scenario,\n Version=umbraco_version,\n Tier=infra_tier,\n ['Cold start']=coalesce(cold_start, false),\n Regression=case(\n regression_status == 'regress', 'regression detected',\n regression_status == 'pass', 'no regression',\n regression_status == 'insufficient', 'not enough history',\n isnotempty(regression_status), regression_status,\n TimeGenerated > ago(15min), 'checking',\n 'not run'),\n Bottleneck=case(\n bn_peak < 50, '—',\n plan_CpuPercentage_max == bn_peak, strcat('App CPU ', round(bn_peak, 0), '%'),\n plan_MemoryPercentage_max == bn_peak, strcat('Server memory ', round(bn_peak, 0), '%'),\n sql_dtu_consumption_percent_max == bn_peak, strcat('Database load ', round(bn_peak, 0), '%'),\n sql_cpu_percent_max == bn_peak, strcat('Database CPU ', round(bn_peak, 0), '%'),\n sql_log_write_percent_max == bn_peak, strcat('Database log-write ', round(bn_peak, 0), '%'),\n sql_physical_data_read_percent_max == bn_peak, strcat('Database physical reads ', round(bn_peak, 0), '%'),\n '—'),\n ['Regressed #']=coalesce(regressed_count, 0),\n ['Run #']=run_id,\n ['Build URL']=['Build URL'],\n Commit=substring(column_ifexists(\"commit\", \"\"), 0, 8),\n ['Parse status']=parse_status,\n ['Commit URL'] = iif(strlen('{RepoCommitUrl}') > 0 and strlen(column_ifexists(\"commit\", \"\")) > 0, strcat('{RepoCommitUrl}', column_ifexists(\"commit\", \"\")), '')\n| order by Started desc", + "size": 0, + "title": "Runs", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "rowLimit": 500, + "formatters": [ + { "columnMatch": "Build URL", "formatter": 7, "formatOptions": { "linkTarget": "Url", "linkIsContextBlade": false } }, + { "columnMatch": "Commit URL", "formatter": 7, "formatOptions": { "linkTarget": "Url", "linkIsContextBlade": false } }, + { "columnMatch": "Regression", "formatter": 18, "formatOptions": { "thresholdsOptions": "icons", "thresholdsGrid": [ + { "operator": "==", "thresholdValue": "regression detected", "representation": "critical" }, + { "operator": "==", "thresholdValue": "not enough history", "representation": "3" }, + { "operator": "==", "thresholdValue": "checking", "representation": "info" }, + { "operator": "==", "thresholdValue": "not run", "representation": "unknown" }, + { "operator": "==", "thresholdValue": "no regression", "representation": "success" }, + { "operator": "Default", "representation": "unknown" } + ] } }, + { "columnMatch": "Regressed #", "formatter": 8, "formatOptions": { "palette": "redBright", "min": 0, "max": 10 } } + ] + } + }, + "name": "runs-table", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Runs" } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Regression detail for the drilled run — which specific samplers regressed\n// per case, sourced from the regression_check rows posted by scripts/check-\n// regression.ps1. Empty result = no regression row exists for this run\n// (regression stage skipped, hasn't completed, or this run predates the\n// regression-LA integration).\nLoadTestSummary_CL\n| where run_id == '{DrillRun}' and parse_status == 'regression_check'\n| project Scenario=scenario, Version=umbraco_version, Tier=infra_tier,\n Verdict=case(\n regression_status == 'regress', 'regression detected',\n regression_status == 'pass', 'no regression',\n regression_status == 'insufficient', 'not enough history',\n regression_status),\n ['Regressed samplers']=replace_string(coalesce(regressed_samplers, ''), ',', ', '),\n ['# regressed']=coalesce(regressed_count, 0)\n| order by Verdict asc, Scenario asc, Version asc, Tier asc", + "size": 0, + "title": "Regression breakdown for selected run", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "Verdict", "formatter": 18, "formatOptions": { "thresholdsOptions": "icons", "thresholdsGrid": [ + { "operator": "==", "thresholdValue": "regression detected", "representation": "critical" }, + { "operator": "==", "thresholdValue": "not enough history", "representation": "3" }, + { "operator": "==", "thresholdValue": "no regression", "representation": "success" }, + { "operator": "Default", "representation": "unknown" } + ] } }, + { "columnMatch": "# regressed", "formatter": 8, "formatOptions": { "palette": "redBright", "min": 0, "max": 10 } } + ] + } + }, + "name": "runs-regression", + "conditionalVisibilities": [ + { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Runs" }, + { "parameterName": "DrillRun", "comparison": "isNotEqualTo", "value": "" } + ] + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Drill-down: paste a run_id into the parameter to see per-sampler detail for\n// that run. Tier column is required because a multi-tier pipeline run produces\n// one row per (sampler, tier) — without Tier the table shows duplicate sampler\n// names with no way to tell Standard from Starter.\nLoadTestSummary_CL\n| where run_id == '{DrillRun}' and isnotempty(scenario_name)\n| project Tier=infra_tier, Sampler=scenario_name,\n Reqs=request_count, Fails=failure_count,\n ['Error %']=round(error_rate*100, 1),\n Avg=round(avg_ms, 0), p50=round(p50_ms, 0), p90=round(p90_ms, 0), p95=round(p95_ms, 0), p99=round(p99_ms, 0), Max=round(max_ms, 0)\n| order by Sampler asc, Tier asc", + "size": 0, + "title": "Per-sampler detail for selected run", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "table", + "gridSettings": { + "formatters": [ + { "columnMatch": "Error %", "formatter": 8, "formatOptions": { "palette": "yellowOrangeRed", "min": 0, "max": 5 }, "numberFormat": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 1 } } }, + { "columnMatch": "p95", "formatter": 8, "formatOptions": { "palette": "yellow", "min": 0, "max": 2000 } }, + { "columnMatch": "p99", "formatter": 8, "formatOptions": { "palette": "orange", "min": 0, "max": 2000 } }, + { "columnMatch": "Max", "formatter": 8, "formatOptions": { "palette": "redBright", "min": 0, "max": 10000 } } + ] + } + }, + "name": "runs-detail", + "conditionalVisibilities": [ + { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Runs" }, + { "parameterName": "DrillRun", "comparison": "isNotEqualTo", "value": "" } + ] + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Per-minute resource pressure (% metrics only) for the drilled run.\n// Sourced from LoadTestSeries_CL. Split from the HTTP-error chart below\n// because percentages share a 0-100 scale where errors would visually flatten.\nLoadTestSeries_CL\n| where run_id == '{DrillRun}'\n| where metric_name !startswith 'app_Http'\n| project TimeGenerated, metric_name, value\n| evaluate pivot(metric_name, any(value))\n| order by TimeGenerated asc", + "size": 0, + "title": "Per-minute resource pressure (% metrics) — App / SQL CPU, DTU, log-write, physical reads.", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "linechart", + "chartSettings": { + "xAxis": "TimeGenerated", + "showLegend": true, + "ySettings": { "min": 0, "max": 100, "numberFormatSettings": { "unit": 0, "options": { "style": "decimal", "maximumFractionDigits": 0 } } } + } + }, + "name": "runs-series-percentages", + "conditionalVisibilities": [ + { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Runs" }, + { "parameterName": "DrillRun", "comparison": "isNotEqualTo", "value": "" } + ] + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "// Per-minute HTTP error counts (4xx / 5xx) for the drilled run. Separate\n// chart from the % metrics above so the count Y-axis auto-scales to the\n// natural range (usually 0-10 except during spikes) instead of being crushed\n// against a 0-100 floor shared with CPU/DTU percentages.\nLoadTestSeries_CL\n| where run_id == '{DrillRun}'\n| where metric_name startswith 'app_Http'\n| project TimeGenerated, metric_name, value\n| evaluate pivot(metric_name, any(value))\n| order by TimeGenerated asc", + "size": 0, + "title": "Per-minute HTTP error counts — 4xx (client errors) + 5xx (server errors). Auto-scaled Y-axis.", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "visualization": "linechart", + "chartSettings": { + "xAxis": "TimeGenerated", + "showLegend": true + } + }, + "name": "runs-series-errors", + "conditionalVisibilities": [ + { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Runs" }, + { "parameterName": "DrillRun", "comparison": "isNotEqualTo", "value": "" } + ] + }, + { + "type": 1, + "content": { + "json": "## Glossary\n\n### Entities\n- **Run** — one pipeline build. Identified by `Run #` (the AzDO build ID).\n- **Case** — one (scenario × version × tier) slice. A multi-case pipeline run produces several rows per `Run #`.\n- **Scenario** — the workload YAML under `loadtests/scenarios/` (e.g. `BackofficeOnly`).\n- **Sampler** — one Locust task inside a scenario (an individual endpoint or operation). Shown as `Sampler` in tables (underlying column: `scenario_name`).\n- **Tier** — the Azure App Service + SQL Database pairing the case ran on (e.g. Starter, Standard, Premium).\n\n### Metrics\n- **p95 / p99** — 95th / 99th percentile of per-request latency in milliseconds. *5 in 100 / 1 in 100* requests were slower than this.\n- **Max** — the single slowest request observed for that sampler during the run. One observation, not an aggregate. Useful for spotting outliers; noisy run-to-run.\n- **Requests/sec** — throughput the case sustained.\n- **Error %** — percentage of requests that failed.\n- **Server CPU avg / peak %** — average and peak Azure App Service Plan CPU during the run.\n- **Database load avg / peak %** — average and peak Azure SQL Database load (Azure's bundled CPU + IO measure, internally called DTU).\n- **Database CPU peak %** — peak Azure SQL Database CPU during the run (the CPU portion only, distinct from `Database load` which bundles CPU + IO).\n- **Database physical reads peak %** — peak fraction of database reads that hit disk instead of memory cache. Rises *before* DTU saturates — a leading indicator that SQL is memory-constrained.\n- **Database log-write peak %** — peak fraction of the database's transaction-log throughput budget consumed during the run. Surfaces write-heavy bottlenecks (inserts/updates/deletes); pairs with `Database CPU peak %` to separate write pressure from read/compute pressure.\n- **Server HTTP 4xx peak / Server HTTP 5xx peak** — peak count of client errors (4xx — auth failures, validation, missing endpoints) and server errors (5xx — crashes, unhandled exceptions) seen during the run.\n- **spread** — one standard deviation across the runs that produced the median. Rough indicator of variability; latency is skewed so treat as indicative, not symmetric.\n- **change %** — percentage difference vs. the chosen baseline. *Negative* = candidate is faster than baseline, *positive* = slower.\n\n### Verdicts\n- **Capacity status** — verdict on a run's infrastructure: *Saturated* (any resource ≥ 95% or error rate ≥ 1%), *Stressed* (any resource ≥ 85%), or *OK*.\n- **Headroom** — `100 − highest peak %` across all monitored resources. The buffer before the most-saturated resource hits ceiling; higher = more room to grow.\n- **Bottleneck** — names the hottest resource and its peak (e.g. `Database load 92%`). Tells you *what* saturated. `—` means everything was below 50%.\n- **Stability** — how consistent the metric was across runs in this cell. *stable* (CV < 10%), *moderate* (10–25%), *noisy* (≥ 25%), *few runs* (< 5). Noisy cells need wider regression thresholds — a 10% regression in a 30%-noise cell is indistinguishable from run-to-run variance.\n- **Regression** — verdict from `scripts/check-regression.ps1` comparing this run's p95/p99/error against the cell's baseline median: *regression detected*, *no regression*, *not enough history*, *checking*, *not run*.\n- **Cold start** — `true` if the run skipped warmup (queue parameter `skipWarmup: true`). Cold-start runs are excluded from Trends/Tiers/Versions by default; visible on Runs.\n\n### Layout\nThe global filter scopes **Trends**, **Compare**, and **Runs**. The **Tiers** and **Versions** tabs have their own pickers but inherit the global filter when one is set." + }, + "name": "glossary", + "conditionalVisibility": { "parameterName": "SelectedTab", "comparison": "isEqualTo", "value": "Glossary" } + } + ], + "fallbackResourceIds": [], + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" +} diff --git a/loadtests/JmeterTest.jmx b/loadtests/JmeterTest.jmx deleted file mode 100644 index a4a5463..0000000 --- a/loadtests/JmeterTest.jmx +++ /dev/null @@ -1,132 +0,0 @@ - - - - - - false - true - true - - - - - - - - stopthread - - false - -1 - - ${__BeanShell( System.getenv("users") )} - 150 - true - 300 - 5 - false - - - - - - - ${__BeanShell( System.getenv("hostName") )} - - https - - - 6 - - - - - - - - - - - - - / - GET - true - false - true - false - - - - - - - - - - - - - - /blog/ - GET - true - false - true - false - - - - - - - - - - - - - - /about/ - GET - true - false - true - false - - - - - - - - - - - - - - /blog/popular-blogs/ - GET - true - false - true - false - - - - - - - - - host - ${__P(host)} - = - - - - - - - - diff --git a/loadtests/JmeterTest/v13/MemberLogin.jmx b/loadtests/JmeterTest/v13/MemberLogin.jmx new file mode 100644 index 0000000..45fc31f --- /dev/null +++ b/loadtests/JmeterTest/v13/MemberLogin.jmx @@ -0,0 +1,511 @@ + + + + + + + + + + + + + numberOfThread + 1 + = + + + duration + 120 + = + + + server + localhost + = + + + protocol + https + = + + + port + 44325 + = + + + backoffice_username + hnd@acceptance.test + = + + + backoffice_password + 0123456789 + = + + + totalOfMember + 30 + = + + + member_password + Test1234! + = + + + + + + ${server} + ${port} + ${protocol} + + + + + + + + + true + false + + + + true + false + false + + + + false + + saveConfig + + + true + true + true + + true + true + true + false + false + true + false + false + false + false + true + false + false + true + true + 0 + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + ${numberOfThread} + ${numberOfThread} + ${duration} + true + true + stopthread + + -1 + false + + + + + groovy + + + true + int total = (vars.get("totalOfMember") ?: "1") as int +int idx = new Random().nextInt(total) + 1 +vars.put("member_username", "TestMember_" + idx) + + + + true + false + + + + ${server} + ${port} + ${protocol} + /testmember_member-login/ + true + GET + true + false + + + + + + + + + Sec-Fetch-Mode + navigate + + + Sec-Fetch-Site + none + + + Accept-Language + en-US,en;q=0.9 + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + true + false + + + + Detected the start of a redirect chain + ${server} + ${port} + ${protocol} + /umbraco/api/memberlogin/login + true + POST + true + false + + + + true + RedirectUrl + /testmember_member-area/ + = + true + + + true + LoginUrl + /testmember_member-login/ + = + true + + + false + Username + ${member_username} + = + true + + + true + Password + ${member_password} + = + true + + + + + + + + + Sec-Fetch-Mode + navigate + + + Referer + ${protocol}://${server}:${port}/testmember_member-login/ + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Origin + ${protocol}://${server}:${port} + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Content-Type + application/x-www-form-urlencoded + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + true + false + + + + ${server} + ${port} + ${protocol} + /testmember_member-area/ + true + GET + true + false + + + + + + + + + Sec-Fetch-Mode + navigate + + + Referer + ${protocol}://${server}:${port}/testmember_member-login/ + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + true + false + + + + Detected the start of a redirect chain + ${server} + ${port} + ${protocol} + /umbraco/api/memberlogin/logout + true + POST + true + false + + + + true + RedirectUrl + /testmember_member-login/ + = + true + + + + + + + + + Sec-Fetch-Mode + navigate + + + Referer + ${protocol}://${server}:${port}/testmember_member-area/ + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Origin + ${protocol}://${server}:${port} + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Content-Type + application/x-www-form-urlencoded + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + + + diff --git a/loadtests/JmeterTest/v13/SaveAndPublishContent.jmx b/loadtests/JmeterTest/v13/SaveAndPublishContent.jmx new file mode 100644 index 0000000..cf6f11e --- /dev/null +++ b/loadtests/JmeterTest/v13/SaveAndPublishContent.jmx @@ -0,0 +1,1910 @@ + + + + + + + + false + false + + + + + + numberOfThread + 1 + = + + + duration + 120 + = + + + server + localhost + = + + + protocol + https + = + + + port + 44325 + = + + + backoffice_username + hnd@acceptance.test + = + + + backoffice_password + 0123456789 + = + + + + + + ${server} + ${port} + ${protocol} + + + + + + + + + true + false + + + + true + false + false + + + + false + + saveConfig + + + true + true + true + + true + true + true + false + false + true + false + false + false + false + true + false + false + true + true + 0 + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + ${numberOfThread} + ${numberOfThread} + ${duration} + true + true + continue + + -1 + false + + + + + true + false + + + + /umbraco + true + GET + true + false + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + Accept-Language + en-US,en;q=0.9 + + + + + + + /umbraco/login + true + GET + true + false + + + + + + + /umbraco/localizedtext + true + GET + true + false + + + + + + + + true + false + + + + /umbraco/backoffice/umbracoapi/authentication/postlogin + true + POST + true + true + + + + false + {"username":"${backoffice_username}","password":"${backoffice_password}","rememberMe":false} + = + + + + + + + + + Content-Type + application/json + + + Accept + */* + + + X-Requested-With + XMLHttpRequest + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + Origin + ${protocol}://${server}:${port} + + + Referer + ${protocol}://${server}:${port}/umbraco/login + + + + + + true + xsrf_token + UMB-XSRF-TOKEN=([^;]+) + $1$ + TOKEN_NOT_FOUND + 1 + false + + + + + + false + props.put("xsrf_token", vars.get("xsrf_token")); + + + + + /umbraco + true + GET + true + false + + + + + + + /umbraco/backoffice/umbracoapi/authentication/IsAuthenticated + true + GET + true + false + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracoapi/section/GetSections + true + GET + true + false + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracotrees/applicationtree/GetApplicationTrees + true + GET + true + false + + + + false + application + content + = + true + + + false + tree + + = + true + + + false + use + main + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-UMB-SEGMENT + null + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Accept-Language + en-US,en;q=0.9 + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + + + + + + true + false + + + + /umbraco/backoffice/umbracotrees/applicationtree/GetApplicationTrees + true + GET + true + false + + + + false + application + content + = + true + + + false + tree + + = + true + + + false + use + main + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-UMB-SEGMENT + null + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Accept-Language + en-US,en;q=0.9 + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + + + + groovy + + + false + import groovy.json.JsonSlurper +def raw = prev.getResponseDataAsString() +// Strip Umbraco XSSI prefix ")]}',\n" if present +def body = raw.startsWith(")]}',") ? raw.substring(raw.indexOf('\n') + 1) : raw +try { + def json = new JsonSlurper().parseText(body) + def nodes = json.children?.findAll { it.id != "-20" } + if (nodes && nodes.size() > 0) { + def node = nodes[new Random().nextInt(nodes.size()-4)] + def id = node.id.toString() + def cssClasses = node.cssClasses + def isNotPublished = false + if (cssClasses instanceof List) { + isNotPublished = cssClasses.any { it?.toString()?.contains("not-published") } + } else if (cssClasses != null) { + isNotPublished = cssClasses.toString().contains("not-published") + } + def published = isNotPublished ? "false" : "true" + vars.put("root_content_id", id) + vars.put("target_document_id", id) + vars.put("published_root", published) + log.info("root_content_id=" + id + " published_root=" + published + " cssClasses=" + cssClasses) + } else { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("target_document_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("published_root", "false") + log.warn("No children found under root") + } +} catch(Exception e) { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("target_document_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("published_root", "false") + log.error("Extract root_content_id failed: " + e.getMessage()) +} + + false + + + + + /umbraco/backoffice/umbracoapi/dashboard/GetDashboard + true + GET + true + false + + + + false + section + content + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracoapi/currentuser/GetUserTours + true + GET + true + false + + + + + + + + + X-UMB-SEGMENT + null + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracoapi/dashboard/GetRemoteDashboardContent + true + GET + true + false + + + + false + section + content + = + true + + + + + + + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + + + + + + true + false + + + + ${__groovy(vars.get("root_content_id") != "ROOT_CONTENT_ID_NOT_FOUND" && vars.get("published_root") == "true")} + false + true + + + + /umbraco/backoffice/umbracoapi/content/GetChildren + true + GET + true + false + + + + false + ${root_content_id} + = + true + id + + + false + updateDate,owner + = + true + includeProperties + + + false + 1 + = + true + pageNumber + + + false + 100 + = + true + pageSize + + + false + updateDate + = + true + orderBy + + + false + Descending + = + true + orderDirection + + + false + true + = + true + orderBySystemField + + + false + + = + true + filter + + + false + en-US + = + true + cultureName + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + groovy + + + false + +import groovy.json.JsonSlurper + +def raw = prev.getResponseDataAsString() ?: '' +def bracketIdx = raw.indexOf('[') +def braceIdx = raw.indexOf('{') +def startIdx +if (bracketIdx < 0) startIdx = braceIdx +else if (braceIdx < 0) startIdx = bracketIdx +else startIdx = Math.min(bracketIdx, braceIdx) +def body = (startIdx >= 0) ? raw.substring(startIdx) : raw + +def asNodeList = { obj -> + if (obj instanceof List) return obj + if (obj instanceof Map) { + if (obj.items instanceof List) return obj.items + if (obj.Items instanceof List) return obj.Items + if (obj.children instanceof List) return obj.children + if (obj.Children instanceof List) return obj.Children + } + return [] +} + +try { + def parsed = body?.trim() ? new JsonSlurper().parseText(body) : null + def nodes = asNodeList(parsed) + if (nodes && nodes.size() > 0) { + def node = nodes[new Random().nextInt(nodes.size())] + def id = node.id.toString() + def published = (node.published != null) ? node.published.toString() : "false" + vars.put("node_id_l1", id) + vars.put("target_document_id", id) + vars.put("parent_target_document_id", vars.get("root_content_id")) + vars.put("published_l1", published) + log.info("node_id_l1=" + id + " published_l1=" + published + " (of " + nodes.size() + " candidates)") + } else { + vars.put("node_id_l1", "NODE_L1_NOT_FOUND") + vars.put("published_l1", "false") + if (!vars.get("target_document_id")) { + vars.put("target_document_id", vars.get("root_content_id") ?: "") + } + log.warn("L1: no items found under root_content_id=" + vars.get("root_content_id") + + " (body starts with: " + (body ? body.take(120) : '<empty>') + ")") + } +} catch(Exception e) { + vars.put("node_id_l1", "NODE_L1_NOT_FOUND") + vars.put("published_l1", "false") + if (!vars.get("target_document_id")) { + vars.put("target_document_id", vars.get("root_content_id") ?: "") + } + log.error("Extract node_id_l1 failed: " + e.getMessage() + + " (body starts with: " + (body ? body.take(120) : '<empty>') + ")", e) +} + + false + + + + + ${__groovy(vars.get("node_id_l1") != "NODE_L1_NOT_FOUND" && vars.get("published_l1") == "true")} + false + true + + + + /umbraco/backoffice/umbracoapi/content/GetChildren + true + GET + true + false + + + + false + ${node_id_l1} + = + true + id + + + false + updateDate,owner + = + true + includeProperties + + + false + 1 + = + true + pageNumber + + + false + 100 + = + true + pageSize + + + false + updateDate + = + true + orderBy + + + false + Descending + = + true + orderDirection + + + false + true + = + true + orderBySystemField + + + false + + = + true + filter + + + false + en-US + = + true + cultureName + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + groovy + + + false + +import groovy.json.JsonSlurper + +def raw = prev.getResponseDataAsString() ?: '' +def bracketIdx = raw.indexOf('[') +def braceIdx = raw.indexOf('{') +def startIdx +if (bracketIdx < 0) startIdx = braceIdx +else if (braceIdx < 0) startIdx = bracketIdx +else startIdx = Math.min(bracketIdx, braceIdx) +def body = (startIdx >= 0) ? raw.substring(startIdx) : raw + +def asNodeList = { obj -> + if (obj instanceof List) return obj + if (obj instanceof Map) { + if (obj.items instanceof List) return obj.items + if (obj.Items instanceof List) return obj.Items + if (obj.children instanceof List) return obj.children + if (obj.Children instanceof List) return obj.Children + } + return [] +} + +try { + def parsed = body?.trim() ? new JsonSlurper().parseText(body) : null + def nodes = asNodeList(parsed) + if (nodes && nodes.size() > 0) { + def node = nodes[new Random().nextInt(nodes.size())] + def id = node.id.toString() + def published = (node.published != null) ? node.published.toString() : "false" + vars.put("node_id_l2", id) + vars.put("target_document_id", id) + vars.put("parent_target_document_id", vars.get("node_id_l1")) + vars.put("published_l2", published) + log.info("node_id_l2=" + id + " published_l2=" + published + " (of " + nodes.size() + " candidates)") + } else { + vars.put("node_id_l2", "NODE_L2_NOT_FOUND") + vars.put("published_l2", "false") + log.warn("L2: no items - target_document_id stays at L1=" + vars.get("target_document_id")) + } +} catch(Exception e) { + vars.put("node_id_l2", "NODE_L2_NOT_FOUND") + vars.put("published_l2", "false") + log.error("Extract node_id_l2 failed: " + e.getMessage(), e) +} + + false + + + + + ${__groovy(vars.get("node_id_l2") != "NODE_L2_NOT_FOUND" && vars.get("published_l2") == "true")} + false + true + + + + /umbraco/backoffice/umbracoapi/content/GetChildren + true + GET + true + false + + + + false + ${node_id_l2} + = + true + id + + + false + updateDate,owner + = + true + includeProperties + + + false + 1 + = + true + pageNumber + + + false + 100 + = + true + pageSize + + + false + updateDate + = + true + orderBy + + + false + Descending + = + true + orderDirection + + + false + true + = + true + orderBySystemField + + + false + + = + true + filter + + + false + en-US + = + true + cultureName + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + groovy + + + false + +import groovy.json.JsonSlurper + +def raw = prev.getResponseDataAsString() ?: '' +def bracketIdx = raw.indexOf('[') +def braceIdx = raw.indexOf('{') +def startIdx +if (bracketIdx < 0) startIdx = braceIdx +else if (braceIdx < 0) startIdx = bracketIdx +else startIdx = Math.min(bracketIdx, braceIdx) +def body = (startIdx >= 0) ? raw.substring(startIdx) : raw + +def asNodeList = { obj -> + if (obj instanceof List) return obj + if (obj instanceof Map) { + if (obj.items instanceof List) return obj.items + if (obj.Items instanceof List) return obj.Items + if (obj.children instanceof List) return obj.children + if (obj.Children instanceof List) return obj.Children + } + return [] +} + +try { + def parsed = body?.trim() ? new JsonSlurper().parseText(body) : null + def nodes = asNodeList(parsed) + if (nodes && nodes.size() > 0) { + def node = nodes[new Random().nextInt(nodes.size())] + def id = node.id.toString() + def published = (node.published != null) ? node.published.toString() : "false" + vars.put("node_id_l3", id) + vars.put("target_document_id", id) + vars.put("parent_target_document_id", vars.get("node_id_l2")) + vars.put("published_l3", published) + log.info("target_document_id (L3)=" + id + " published_l3=" + published + " (of " + nodes.size() + " candidates)") + } else { + vars.put("node_id_l3", "NODE_L3_NOT_FOUND") + vars.put("published_l3", "false") + log.warn("L3: no items - target_document_id stays at L2=" + vars.get("target_document_id")) + } +} catch(Exception e) { + vars.put("node_id_l3", "NODE_L3_NOT_FOUND") + vars.put("published_l3", "false") + log.error("L3 failed: " + e.getMessage(), e) +} + + false + + + + + + + + /umbraco/backoffice/umbracoapi/content/GetById + true + GET + true + false + + + + false + id + ${target_document_id} + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + groovy + + + false + +import groovy.json.JsonSlurper +import groovy.json.JsonOutput +import java.text.SimpleDateFormat +import java.util.Date + +def raw = prev.getResponseDataAsString() +def body = raw.startsWith(")]}',") ? raw.substring(raw.indexOf('\n') + 1) : raw + +try { + def content = new JsonSlurper().parseText(body) + def ts = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()) + + def postVariants = content.variants.withIndex().collect { variant, idx -> + def props = variant.tabs?.collectMany { tab -> + tab.properties?.collect { prop -> + [ + id : prop.id, + alias : prop.alias, + value : prop.value, + culture: prop.culture, + segment: prop.segment + ] + } ?: [] + } ?: [] + + def variantName = (idx == 0) ? ("LoadTest_" + ts) : variant.name + + [ + name : variantName, + properties : props, + culture : variant.language?.culture, + segment : variant.segment, + publish : true, + save : true, + releaseDate: null, + expireDate : null + ] + } + + def postSaveBody = [ + id : content.id, + contentTypeAlias: content.contentTypeAlias, + parentId : content.parentId, + action : "publish", + variants : postVariants, + templateAlias : content.template, + releaseDate : null, + expireDate : null + ] + + vars.put("post_save_body", JsonOutput.toJson(postSaveBody)) + log.info("PostSave body built for id=" + content.id + " cultures=" + content.variants.size() + " ts=" + ts) +} catch(Exception e) { + vars.put("post_save_body", "{}") + log.error("Build PostSave body failed: " + e.getMessage()) +} + + false + + + + + /umbraco/backoffice/umbracotrees/contenttree/GetMenu + true + GET + true + false + + + + false + id + ${target_document_id} + = + true + + + false + application + content + = + true + + + false + tree + content + = + true + + + false + use + main + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracoapi/entity/GetAncestors + true + GET + true + false + + + + false + id + ${target_document_id} + = + true + + + false + type + document + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracoapi/entity/GetAll + true + GET + true + false + + + + false + type + Template + = + true + + + false + postFilter + + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + + true + false + + + + /umbraco/backoffice/umbracoapi/content/PostSave + true + POST + true + true + true + false + + + + false + contentItem + ${post_save_body} + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + groovy + + + false + import groovy.json.JsonSlurper +def raw = prev.getResponseDataAsString() +def body = raw.startsWith(")]}',") ? raw.substring(raw.indexOf('\n') + 1) : raw +try { + def json = new JsonSlurper().parseText(body) + def urls = json?.urls ?: [] + def enUrl = urls.find { it?.culture == 'en-US' && it?.isUrl == true } + if (enUrl) { + def fullUrl = enUrl.text?.toString() ?: "" + def path = fullUrl + def schemeIdx = fullUrl.indexOf("://") + if (schemeIdx >= 0) { + def afterScheme = fullUrl.substring(schemeIdx + 3) + def slashIdx = afterScheme.indexOf("/") + path = (slashIdx >= 0) ? afterScheme.substring(slashIdx) : "/" + } + if (!path.startsWith("/")) path = "/" + path + vars.put("content_url", path) + log.info("content_url=" + path + " (from " + fullUrl + ")") + } else { + vars.put("content_url", "CONTENT_URL_NOT_FOUND") + log.warn("No en-US URL with isUrl=true found in PostSave response") + } +} catch(Exception e) { + vars.put("content_url", "CONTENT_URL_NOT_FOUND") + log.error("Extract content_url failed: " + e.getMessage()) +} + + false + + + + + /umbraco/backoffice/umbracotrees/contenttree/GetNodes + true + GET + true + false + + + + false + id + ${parent_target_document_id} + = + true + + + false + application + content + = + true + + + false + tree + content + = + true + + + false + use + main + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracoapi/entity/GetAncestors + true + GET + true + false + + + + false + id + ${target_document_id} + = + true + + + false + type + document + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + + true + false + + + + ${content_url} + true + GET + true + false + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Language + en-US,en;q=0.9 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + + true + false + + + + /umbraco/backoffice/umbracoapi/authentication/PostLogout + true + POST + true + false + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/login + true + GET + true + false + + + + false + logout + true + = + true + + + + + + + + + + diff --git a/loadtests/JmeterTest/v13/SaveContent.jmx b/loadtests/JmeterTest/v13/SaveContent.jmx new file mode 100644 index 0000000..d021a19 --- /dev/null +++ b/loadtests/JmeterTest/v13/SaveContent.jmx @@ -0,0 +1,1841 @@ + + + + + + + + false + false + + + + + + numberOfThread + 1 + = + + + duration + 120 + = + + + server + localhost + = + + + protocol + https + = + + + port + 44325 + = + + + backoffice_username + hnd@acceptance.test + = + + + backoffice_password + 0123456789 + = + + + + + + ${server} + ${port} + ${protocol} + + + + + + + + + true + false + + + + true + false + false + + + + false + + saveConfig + + + true + true + true + + true + true + true + false + false + true + false + false + false + false + true + false + false + true + true + 0 + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + ${numberOfThread} + ${numberOfThread} + ${duration} + true + true + continue + + -1 + false + + + + + true + false + + + + /umbraco + true + GET + true + false + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + Accept-Language + en-US,en;q=0.9 + + + + + + + /umbraco/login + true + GET + true + false + + + + + + + /umbraco/localizedtext + true + GET + true + false + + + + + + + + true + false + + + + /umbraco/backoffice/umbracoapi/authentication/postlogin + true + POST + true + true + + + + false + {"username":"${backoffice_username}","password":"${backoffice_password}","rememberMe":false} + = + + + + + + + + + Content-Type + application/json + + + Accept + */* + + + X-Requested-With + XMLHttpRequest + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + Origin + ${protocol}://${server}:${port} + + + Referer + ${protocol}://${server}:${port}/umbraco/login + + + + + + true + xsrf_token + UMB-XSRF-TOKEN=([^;]+) + $1$ + TOKEN_NOT_FOUND + 1 + false + + + + + + false + props.put("xsrf_token", vars.get("xsrf_token")); + + + + + /umbraco + true + GET + true + false + + + + + + + /umbraco/backoffice/umbracoapi/authentication/IsAuthenticated + true + GET + true + false + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracoapi/section/GetSections + true + GET + true + false + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracotrees/applicationtree/GetApplicationTrees + true + GET + true + false + + + + false + application + content + = + true + + + false + tree + + = + true + + + false + use + main + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-UMB-SEGMENT + null + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Accept-Language + en-US,en;q=0.9 + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + + + + + + true + false + + + + /umbraco/backoffice/umbracotrees/applicationtree/GetApplicationTrees + true + GET + true + false + + + + false + application + content + = + true + + + false + tree + + = + true + + + false + use + main + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-UMB-SEGMENT + null + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Accept-Language + en-US,en;q=0.9 + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + + + + groovy + + + false + import groovy.json.JsonSlurper +def raw = prev.getResponseDataAsString() +// Strip Umbraco XSSI prefix ")]}',\n" if present +def body = raw.startsWith(")]}',") ? raw.substring(raw.indexOf('\n') + 1) : raw +try { + def json = new JsonSlurper().parseText(body) + def nodes = json.children?.findAll { it.id != "-20" } + if (nodes && nodes.size() > 0) { + def node = nodes[new Random().nextInt(nodes.size()-4)] + def id = node.id.toString() + def cssClasses = node.cssClasses + def isNotPublished = false + if (cssClasses instanceof List) { + isNotPublished = cssClasses.any { it?.toString()?.contains("not-published") } + } else if (cssClasses != null) { + isNotPublished = cssClasses.toString().contains("not-published") + } + def published = isNotPublished ? "false" : "true" + vars.put("root_content_id", id) + vars.put("target_document_id", id) + vars.put("published_root", published) + log.info("root_content_id=" + id + " published_root=" + published + " cssClasses=" + cssClasses) + } else { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("target_document_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("published_root", "false") + log.warn("No children found under root") + } +} catch(Exception e) { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("target_document_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("published_root", "false") + log.error("Extract root_content_id failed: " + e.getMessage()) +} + + false + + + + + /umbraco/backoffice/umbracoapi/dashboard/GetDashboard + true + GET + true + false + + + + false + section + content + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracoapi/currentuser/GetUserTours + true + GET + true + false + + + + + + + + + X-UMB-SEGMENT + null + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracoapi/dashboard/GetRemoteDashboardContent + true + GET + true + false + + + + false + section + content + = + true + + + + + + + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + + + + + + true + false + + + + ${__groovy(vars.get("root_content_id") != "ROOT_CONTENT_ID_NOT_FOUND" && vars.get("published_root") == "true")} + false + true + + + + /umbraco/backoffice/umbracoapi/content/GetChildren + true + GET + true + false + + + + false + ${root_content_id} + = + true + id + + + false + updateDate,owner + = + true + includeProperties + + + false + 1 + = + true + pageNumber + + + false + 100 + = + true + pageSize + + + false + updateDate + = + true + orderBy + + + false + Descending + = + true + orderDirection + + + false + true + = + true + orderBySystemField + + + false + + = + true + filter + + + false + en-US + = + true + cultureName + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + groovy + + + false + +import groovy.json.JsonSlurper + +def raw = prev.getResponseDataAsString() ?: '' +def bracketIdx = raw.indexOf('[') +def braceIdx = raw.indexOf('{') +def startIdx +if (bracketIdx < 0) startIdx = braceIdx +else if (braceIdx < 0) startIdx = bracketIdx +else startIdx = Math.min(bracketIdx, braceIdx) +def body = (startIdx >= 0) ? raw.substring(startIdx) : raw + +def asNodeList = { obj -> + if (obj instanceof List) return obj + if (obj instanceof Map) { + if (obj.items instanceof List) return obj.items + if (obj.Items instanceof List) return obj.Items + if (obj.children instanceof List) return obj.children + if (obj.Children instanceof List) return obj.Children + } + return [] +} + +try { + def parsed = body?.trim() ? new JsonSlurper().parseText(body) : null + def nodes = asNodeList(parsed) + if (nodes && nodes.size() > 0) { + def node = nodes[new Random().nextInt(nodes.size())] + def id = node.id.toString() + def published = (node.published != null) ? node.published.toString() : "false" + vars.put("node_id_l1", id) + vars.put("target_document_id", id) + vars.put("parent_target_document_id", vars.get("root_content_id")) + vars.put("published_l1", published) + log.info("node_id_l1=" + id + " published_l1=" + published + " (of " + nodes.size() + " candidates)") + } else { + vars.put("node_id_l1", "NODE_L1_NOT_FOUND") + vars.put("published_l1", "false") + if (!vars.get("target_document_id")) { + vars.put("target_document_id", vars.get("root_content_id") ?: "") + } + log.warn("L1: no items found under root_content_id=" + vars.get("root_content_id") + + " (body starts with: " + (body ? body.take(120) : '<empty>') + ")") + } +} catch(Exception e) { + vars.put("node_id_l1", "NODE_L1_NOT_FOUND") + vars.put("published_l1", "false") + if (!vars.get("target_document_id")) { + vars.put("target_document_id", vars.get("root_content_id") ?: "") + } + log.error("Extract node_id_l1 failed: " + e.getMessage() + + " (body starts with: " + (body ? body.take(120) : '<empty>') + ")", e) +} + + false + + + + + ${__groovy(vars.get("node_id_l1") != "NODE_L1_NOT_FOUND" && vars.get("published_l1") == "true")} + false + true + + + + /umbraco/backoffice/umbracoapi/content/GetChildren + true + GET + true + false + + + + false + ${node_id_l1} + = + true + id + + + false + updateDate,owner + = + true + includeProperties + + + false + 1 + = + true + pageNumber + + + false + 100 + = + true + pageSize + + + false + updateDate + = + true + orderBy + + + false + Descending + = + true + orderDirection + + + false + true + = + true + orderBySystemField + + + false + + = + true + filter + + + false + en-US + = + true + cultureName + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + groovy + + + false + +import groovy.json.JsonSlurper + +def raw = prev.getResponseDataAsString() ?: '' +def bracketIdx = raw.indexOf('[') +def braceIdx = raw.indexOf('{') +def startIdx +if (bracketIdx < 0) startIdx = braceIdx +else if (braceIdx < 0) startIdx = bracketIdx +else startIdx = Math.min(bracketIdx, braceIdx) +def body = (startIdx >= 0) ? raw.substring(startIdx) : raw + +def asNodeList = { obj -> + if (obj instanceof List) return obj + if (obj instanceof Map) { + if (obj.items instanceof List) return obj.items + if (obj.Items instanceof List) return obj.Items + if (obj.children instanceof List) return obj.children + if (obj.Children instanceof List) return obj.Children + } + return [] +} + +try { + def parsed = body?.trim() ? new JsonSlurper().parseText(body) : null + def nodes = asNodeList(parsed) + if (nodes && nodes.size() > 0) { + def node = nodes[new Random().nextInt(nodes.size())] + def id = node.id.toString() + def published = (node.published != null) ? node.published.toString() : "false" + vars.put("node_id_l2", id) + vars.put("target_document_id", id) + vars.put("parent_target_document_id", vars.get("node_id_l1")) + vars.put("published_l2", published) + log.info("node_id_l2=" + id + " published_l2=" + published + " (of " + nodes.size() + " candidates)") + } else { + vars.put("node_id_l2", "NODE_L2_NOT_FOUND") + vars.put("published_l2", "false") + log.warn("L2: no items - target_document_id stays at L1=" + vars.get("target_document_id")) + } +} catch(Exception e) { + vars.put("node_id_l2", "NODE_L2_NOT_FOUND") + vars.put("published_l2", "false") + log.error("Extract node_id_l2 failed: " + e.getMessage(), e) +} + + false + + + + + ${__groovy(vars.get("node_id_l2") != "NODE_L2_NOT_FOUND" && vars.get("published_l2") == "true")} + false + true + + + + /umbraco/backoffice/umbracoapi/content/GetChildren + true + GET + true + false + + + + false + ${node_id_l2} + = + true + id + + + false + updateDate,owner + = + true + includeProperties + + + false + 1 + = + true + pageNumber + + + false + 100 + = + true + pageSize + + + false + updateDate + = + true + orderBy + + + false + Descending + = + true + orderDirection + + + false + true + = + true + orderBySystemField + + + false + + = + true + filter + + + false + en-US + = + true + cultureName + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + groovy + + + false + +import groovy.json.JsonSlurper + +def raw = prev.getResponseDataAsString() ?: '' +def bracketIdx = raw.indexOf('[') +def braceIdx = raw.indexOf('{') +def startIdx +if (bracketIdx < 0) startIdx = braceIdx +else if (braceIdx < 0) startIdx = bracketIdx +else startIdx = Math.min(bracketIdx, braceIdx) +def body = (startIdx >= 0) ? raw.substring(startIdx) : raw + +def asNodeList = { obj -> + if (obj instanceof List) return obj + if (obj instanceof Map) { + if (obj.items instanceof List) return obj.items + if (obj.Items instanceof List) return obj.Items + if (obj.children instanceof List) return obj.children + if (obj.Children instanceof List) return obj.Children + } + return [] +} + +try { + def parsed = body?.trim() ? new JsonSlurper().parseText(body) : null + def nodes = asNodeList(parsed) + if (nodes && nodes.size() > 0) { + def node = nodes[new Random().nextInt(nodes.size())] + def id = node.id.toString() + def published = (node.published != null) ? node.published.toString() : "false" + vars.put("node_id_l3", id) + vars.put("target_document_id", id) + vars.put("parent_target_document_id", vars.get("node_id_l2")) + vars.put("published_l3", published) + log.info("target_document_id (L3)=" + id + " published_l3=" + published + " (of " + nodes.size() + " candidates)") + } else { + vars.put("node_id_l3", "NODE_L3_NOT_FOUND") + vars.put("published_l3", "false") + log.warn("L3: no items - target_document_id stays at L2=" + vars.get("target_document_id")) + } +} catch(Exception e) { + vars.put("node_id_l3", "NODE_L3_NOT_FOUND") + vars.put("published_l3", "false") + log.error("L3 failed: " + e.getMessage(), e) +} + + false + + + + + + + + /umbraco/backoffice/umbracoapi/content/GetById + true + GET + true + false + + + + false + id + ${target_document_id} + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + groovy + + + false + +import groovy.json.JsonSlurper +import groovy.json.JsonOutput +import java.text.SimpleDateFormat +import java.util.Date + +def raw = prev.getResponseDataAsString() +def body = raw.startsWith(")]}',") ? raw.substring(raw.indexOf('\n') + 1) : raw + +try { + def content = new JsonSlurper().parseText(body) + def ts = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()) + + // Build variants list. Only the first (active/primary) variant is actually + // saved; the rest are included for completeness but with save=false. This + // mirrors what the Umbraco backoffice JS sends when the user clicks the + // plain "Save" button (no publish). + def postVariants = content.variants.withIndex().collect { variant, idx -> + def props = variant.tabs?.collectMany { tab -> + tab.properties?.collect { prop -> + [ + id : prop.id, + alias : prop.alias, + value : prop.value, + culture: prop.culture, + segment: prop.segment + ] + } ?: [] + } ?: [] + + // Only bump the name on the active (first) variant. + def variantName = (idx == 0) ? ("LoadTest_" + ts) : variant.name + + [ + name : variantName, + properties : props, + culture : variant.language?.culture, + segment : variant.segment, + publish : false, // SAVE-only never publishes + save : (idx == 0), // only the active variant is saved + releaseDate: null, + expireDate : null + ] + } + + def postSaveBody = [ + id : content.id, + contentTypeAlias: content.contentTypeAlias, + parentId : content.parentId, + action : "save", // <-- was "publish" + variants : postVariants, + templateAlias : content.template, + releaseDate : null, + expireDate : null + ] + + vars.put("post_save_body", JsonOutput.toJson(postSaveBody)) + log.info("PostSave (save-only) body built for id=" + content.id + + " variants=" + content.variants.size() + " ts=" + ts) +} catch(Exception e) { + vars.put("post_save_body", "{}") + log.error("Build PostSave body failed: " + e.getMessage(), e) +} + + false + + + + + /umbraco/backoffice/umbracotrees/contenttree/GetMenu + true + GET + true + false + + + + false + id + ${target_document_id} + = + true + + + false + application + content + = + true + + + false + tree + content + = + true + + + false + use + main + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracoapi/entity/GetAncestors + true + GET + true + false + + + + false + id + ${target_document_id} + = + true + + + false + type + document + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracoapi/entity/GetAll + true + GET + true + false + + + + false + type + Template + = + true + + + false + postFilter + + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + + true + false + + + + /umbraco/backoffice/umbracoapi/content/PostSave + true + POST + true + true + true + false + + + + false + contentItem + ${post_save_body} + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracotrees/contenttree/GetNodes + true + GET + true + false + + + + false + id + ${parent_target_document_id} + = + true + + + false + application + content + = + true + + + false + tree + content + = + true + + + false + use + main + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracoapi/entity/GetAncestors + true + GET + true + false + + + + false + id + ${target_document_id} + = + true + + + false + type + document + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + X-UMB-CULTURE + en-US + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + + true + false + + + + /umbraco/backoffice/umbracoapi/authentication/PostLogout + true + POST + true + false + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/login + true + GET + true + false + + + + false + logout + true + = + true + + + + + + + + + + diff --git a/loadtests/JmeterTest/v13/SaveDocumentType.jmx b/loadtests/JmeterTest/v13/SaveDocumentType.jmx new file mode 100644 index 0000000..2d28539 --- /dev/null +++ b/loadtests/JmeterTest/v13/SaveDocumentType.jmx @@ -0,0 +1,2123 @@ + + + + + + + + + + + + + numberOfThread + 1 + = + + + duration + 120 + = + + + server + localhost + = + + + protocol + https + = + + + port + 44325 + = + + + backoffice_username + hnd@acceptance.test + = + + + backoffice_password + 0123456789 + = + + + root_document_type + Variant Document Types + = + + + + + + ${server} + ${port} + ${protocol} + + + + + + + + + true + false + + + + true + false + false + + + + false + + saveConfig + + + true + true + true + + true + true + true + false + false + true + false + false + false + false + true + false + false + true + true + 0 + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + ${numberOfThread} + ${numberOfThread} + ${duration} + true + true + continue + + -1 + false + + + + + true + false + + + + /umbraco + true + GET + true + false + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + Accept-Language + en-US,en;q=0.9 + + + + + + + /umbraco/login + true + GET + true + false + + + + + + + /umbraco/localizedtext + true + GET + true + false + + + + + + + + true + false + + + + /umbraco/backoffice/umbracoapi/authentication/postlogin + true + POST + true + true + + + + false + {"username":"${backoffice_username}","password":"${backoffice_password}","rememberMe":false} + = + + + + + + + + + Content-Type + application/json + + + Accept + */* + + + X-Requested-With + XMLHttpRequest + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + Origin + ${protocol}://${server}:${port} + + + Referer + ${protocol}://${server}:${port}/umbraco/login + + + + + + true + xsrf_token + UMB-XSRF-TOKEN=([^;]+) + $1$ + TOKEN_NOT_FOUND + 1 + false + + + + + + false + props.put("xsrf_token", vars.get("xsrf_token")); + + + + + /umbraco + true + GET + true + false + + + + + + + /umbraco/backoffice/umbracoapi/authentication/IsAuthenticated + true + GET + true + false + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracoapi/section/GetSections + true + GET + true + false + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/backoffice/umbracotrees/applicationtree/GetApplicationTrees + true + GET + true + false + + + + false + application + content + = + true + + + false + tree + + = + true + + + false + use + main + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-UMB-SEGMENT + null + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Accept-Language + en-US,en;q=0.9 + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + + + + groovy + + + false + +import groovy.json.JsonSlurper +def raw = prev.getResponseDataAsString() +// Strip Umbraco XSSI prefix ")]}',\n" if present +def body = raw.startsWith(")]}',") ? raw.substring(raw.indexOf('\n') + 1) : raw +try { + def json = new JsonSlurper().parseText(body) + def nodes = json.children?.findAll { it.id != "-20" } + if (nodes && nodes.size() > 0) { + def id = nodes[new Random().nextInt(nodes.size())].id.toString() + vars.put("root_content_id", id) + log.info("root_content_id=" + id) + } else { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + log.warn("No children found under root") + } +} catch(Exception e) { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + log.error("Extract root_content_id failed: " + e.getMessage()) +} + + false + + + + + + true + false + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracoapi/dashboard/GetDashboard + true + GET + true + false + + + + false + section + settings + = + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Priority + u=0 + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracotrees/applicationtree/GetApplicationTrees + true + GET + true + false + + + + false + application + settings + = + + + false + tree + + = + + + false + use + main + = + + + false + culture + en-US + = + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Priority + u=0 + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + + true + false + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracotrees/contenttypetree/GetNodes + true + GET + true + false + + + + false + id + -1 + = + + + false + application + settings + = + + + false + tree + documentTypes + = + + + false + use + main + = + + + false + culture + en-US + = + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Priority + u=0 + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + groovy + + + false + import groovy.json.JsonSlurper +def raw = prev.getResponseDataAsString() +def body = raw.startsWith(")]}',") ? raw.substring(raw.indexOf('\n') + 1) : raw +try { + def json = new JsonSlurper().parseText(body) + def items = (json instanceof List) ? json : (json?.children ?: []) + def target = items.find { it?.name == '${root_document_type}' } + if (target?.id != null) { + vars.put('variantDocTypeId', target.id.toString()) + log.info('variantDocTypeId=' + target.id) + } else { + vars.put('variantDocTypeId', 'variantDocTypeId_NOT_FOUND') + log.warn('Variant Document Types not found in response') + } +} catch(Exception e) { + vars.put('variantDocTypeId', 'variantDocTypeId_NOT_FOUND') + log.error('Find Variant Document Types failed: ' + e.getMessage()) +} + + false + + + + + ${server} + ${port} + ${protocol} + utf-8 + /umbraco/backoffice/umbracotrees/contenttypetree/GetNodes + true + GET + true + false + + + + false + id + ${variantDocTypeId} + = + + + false + application + settings + = + + + false + tree + documentTypes + = + + + false + use + main + = + + + false + culture + en-US + = + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Priority + u=0 + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + groovy + + + false + import groovy.json.JsonSlurper +def raw = prev.getResponseDataAsString() +def body = raw.startsWith(")]}',") ? raw.substring(raw.indexOf('\n') + 1) : raw +try { + def json = new JsonSlurper().parseText(body) + def items = (json instanceof List) ? json : (json?.children ?: []) + if (items && items.size() > 0) { + def picked = items[new Random().nextInt(items.size())] + vars.put('randomDocTypeId', picked.id.toString()) + log.info('randomDocTypeId=' + picked.id) + } else { + vars.put('randomDocTypeId', 'randomDocTypeId_NOT_FOUND') + log.warn('No children to pick from') + } +} catch(Exception e) { + vars.put('randomDocTypeId', 'randomDocTypeId_NOT_FOUND') + log.error('Pick random doctype failed: ' + e.getMessage()) +} + + false + + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/api/modelsbuilderdashboard/GetModelsOutOfDateStatus + true + GET + true + false + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracoapi/contenttype/GetById + true + GET + true + false + + + + false + id + ${randomDocTypeId} + = + true + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + groovy + + + false + import groovy.json.JsonSlurper +def raw = prev.getResponseDataAsString() +def body = raw.startsWith(")]}',") ? raw.substring(raw.indexOf('\n') + 1) : raw +// Clear leftovers from previous iteration +def clearVars = { String prefix -> + int i = 1 + while (vars.get(prefix + i) != null) { vars.remove(prefix + i); i++ } +} +clearVars('mediatype_aliases_') +clearVars('block_keys_') +vars.put('mediatype_count', '0') +vars.put('block_count', '0') + +def getConfigValue = { config, String key -> + if (config == null) return null + if (config instanceof Map) return config[key] + if (config instanceof List) { + def entry = config.find { it instanceof Map && (it.key == key || it.alias == key) } + return entry?.value + } + return null +} + +try { + def ct = new JsonSlurper().parseText(body) + def groups = ct?.groups ?: [] + def mediaList = [] + def blockList = [] + def MEDIA_EDITORS = ['Umbraco.MediaPicker3', 'Umbraco.MediaPicker', 'Umbraco.ImageCropper'] as Set + def BLOCK_EDITORS = ['Umbraco.BlockList', 'Umbraco.BlockGrid'] as Set + groups.each { g -> + (g?.properties ?: []).each { p -> + def editor = p?.editor?.toString() ?: '' + if (MEDIA_EDITORS.contains(editor)) { + def filter = getConfigValue(p.config, 'filter') + def aliases = '' + if (filter instanceof String) aliases = filter + else if (filter instanceof List) aliases = filter.collect { it?.toString() }.findAll { it }.join(',') + if (aliases == null || aliases.trim() == '') aliases = 'Image' + mediaList << aliases + } else if (BLOCK_EDITORS.contains(editor)) { + def blocks = getConfigValue(p.config, 'blocks') + def keys = [] + if (blocks instanceof List) { + blocks.each { b -> + def k = (b instanceof Map) ? (b.contentElementTypeKey ?: b.key) : null + if (k) keys << k.toString() + } + } + if (keys.size() > 0) { + def jsonArr = '["' + keys.join('","') + '"]' + blockList << jsonArr + } + } + } + } + mediaList.eachWithIndex { val, i -> vars.put('mediatype_aliases_' + (i + 1), val) } + vars.put('mediatype_count', mediaList.size().toString()) + blockList.eachWithIndex { val, i -> vars.put('block_keys_' + (i + 1), val) } + vars.put('block_count', blockList.size().toString()) + log.info('media properties=' + mediaList.size() + ', block properties=' + blockList.size()) +} catch(Exception e) { + log.error('Scan content type properties failed: ' + e.getMessage()) +} + + false + + + + groovy + + + false + // Capture the raw doctype JSON for later reuse in PostSave. +// Strip the Umbraco XSSI prefix ")]}',\n" if present. +def raw = prev.getResponseDataAsString() +def body = raw.startsWith(")]}',") ? raw.substring(raw.indexOf('\n') + 1) : raw +vars.put('doctype_body_template', body) + + false + + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracoapi/contenttype/HasContentNodes + true + GET + true + false + + + + false + id + ${randomDocTypeId} + = + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracoapi/entity/GetAll + true + GET + true + false + + + + false + type + Template + = + + + false + postFilter + + = + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracoapi/contenttype/GetAll + true + GET + true + false + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracoapi/datatype/GetByName + true + GET + true + false + + + + true + name + List View - Content + = + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + mediatype_aliases + mediatype_aliases + true + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracoapi/mediatype/getAllFiltered + true + GET + true + false + + + + false + aliases + ${mediatype_aliases} + = + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + + block_keys + block_keys + true + + + + ${server} + ${port} + ${protocol} + utf-8 + /umbraco/backoffice/umbracoapi/content/GetEmptyByKeys + true + POST + true + true + + + + false + {"contentTypeKeys":${block_keys},"parentId":-20} + = + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Origin + ${protocol}://${server}:${port} + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Content-Type + application/json;charset=utf-8 + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + + + true + false + + + + ${server} + ${port} + ${protocol} + utf-8 + /umbraco/backoffice/umbracoapi/contenttype/PostSave + true + POST + true + true + + + + false + ${doctype_body} + = + + + + + + + groovy + + + false + // Build PostSave body in the slim DocumentTypeSave schema that Umbraco +// backoffice JS actually POSTs. We start from the captured GetById response +// (DocumentTypeDisplay schema) and transform it down. +import groovy.json.JsonSlurper +import groovy.json.JsonOutput +import java.text.SimpleDateFormat + +def template = vars.get('doctype_body_template') +log.info('[BuildBody] template length=' + (template ? template.length() : 0)) +if (!template || template.trim().isEmpty()) { + log.error('[BuildBody] doctype_body_template MISSING - step 04 GetById did not capture') + vars.put('doctype_body', '') + return +} + +try { + def src = new JsonSlurper().parseText(template) + + // ---- groups + properties ------------------------------------------- + def newGroups = [] + (src.groups ?: []).each { g -> + def newProps = [] + (g.properties ?: []).each { p -> + newProps << [ + id : p.id, + alias : p.alias, + description : p.description, + validation : p.validation, + label : p.label, + sortOrder : p.sortOrder, + dataTypeId : p.dataTypeId, + groupId : p.groupId, + allowCultureVariant: p.allowCultureVariant, + allowSegmentVariant: p.allowSegmentVariant, + labelOnTop : p.labelOnTop, + ] + } + newGroups << [ + id : g.id, + sortOrder : g.sortOrder, + name : g.name, + key : g.key, + alias : g.alias, + type : g.type, + properties: newProps, + ] + } + + // ---- templates: object/list-of-objects -> alias strings ------------ + def defaultTemplateAlias = null + if (src.defaultTemplate instanceof Map) { + defaultTemplateAlias = src.defaultTemplate.alias + } else if (src.defaultTemplate instanceof String) { + defaultTemplateAlias = src.defaultTemplate + } + def allowedTemplateAliases = [] + (src.allowedTemplates ?: []).each { t -> + if (t instanceof Map) allowedTemplateAliases << t.alias + else if (t instanceof String) allowedTemplateAliases << t + } + + // ---- bump name ----------------------------------------------------- + def ts = new SimpleDateFormat('yyyyMMdd_HHmmss').format(new Date()) + def newName = 'LoadTest_' + ts + + // ---- build the slim body in roughly the same field order the UI uses + def body = [ + compositeContentTypes: src.compositeContentTypes ?: [], + isContainer : src.isContainer, + allowAsRoot : src.allowAsRoot, + allowedTemplates : allowedTemplateAliases, + allowedContentTypes : src.allowedContentTypes ?: [], + alias : src.alias, + description : src.description, + thumbnail : src.thumbnail, + name : newName, + id : src.id, + icon : src.icon, + trashed : src.trashed, + key : src.key, + parentId : src.parentId, + path : src.path, + allowCultureVariant : src.allowCultureVariant, + allowSegmentVariant : src.allowSegmentVariant, + isElement : src.isElement, + historyCleanup : src.historyCleanup, + defaultTemplate : defaultTemplateAlias, + groups : newGroups, + ] + + def json = JsonOutput.toJson(body) + vars.put('doctype_body', json) + log.info('[BuildBody] body length=' + json.length() + ' name=' + newName + + ' groups=' + newGroups.size() + + ' defaultTemplate=' + defaultTemplateAlias + + ' allowedTemplates=' + allowedTemplateAliases) +} catch(Exception e) { + vars.put('doctype_body', '') + log.error('[BuildBody] FAILED: ' + e.getClass().getName() + ' - ' + e.getMessage(), e) +} + + false + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Origin + ${protocol}://${server}:${port} + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Priority + u=0 + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Content-Type + application/json;charset=utf-8 + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracotrees/contenttypetree/GetNodes + true + GET + true + false + + + + false + id + ${randomDocTypeId} + = + true + + + false + application + settings + = + true + + + false + tree + documentTypes + = + true + + + false + use + main + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/backoffice/umbracotrees/templatestree/GetNodes + true + GET + true + false + + + + false + id + -1 + = + true + + + false + application + settings + = + true + + + false + tree + templates + = + true + + + false + use + main + = + true + + + false + culture + en-US + = + true + + + + + + + + + X-UMB-SEGMENT + null + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Accept + application/json, text/plain, */* + + + X-Requested-With + XMLHttpRequest + + + X-UMB-CULTURE + en-US + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + + true + false + + + + /umbraco/backoffice/umbracoapi/authentication/PostLogout + true + POST + true + false + + + + + + + + + X-Requested-With + XMLHttpRequest + + + Accept + application/json, text/plain, */* + + + X-UMB-XSRF-TOKEN + ${xsrf_token} + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Encoding + gzip, deflate, br, zstd + + + + + + + /umbraco/login + true + GET + true + false + + + + false + logout + true + = + true + + + + + + + + + + diff --git a/loadtests/JmeterTest/v13/ViewHomePage.jmx b/loadtests/JmeterTest/v13/ViewHomePage.jmx new file mode 100644 index 0000000..0222f28 --- /dev/null +++ b/loadtests/JmeterTest/v13/ViewHomePage.jmx @@ -0,0 +1,236 @@ + + + + + + + + + + + + + numberOfThread + 1 + = + + + duration + 120 + = + + + server + localhost + = + + + protocol + https + = + + + port + 44322 + = + + + backoffice_username + hnd@acceptance.test + = + + + backoffice_password + 0123456789 + = + + + totalOfMember + 30 + = + + + member_password + Test1234! + = + + + + + + ${server} + ${port} + ${protocol} + + + + + + + + + true + false + + + + true + false + false + + + + false + + saveConfig + + + true + true + true + + true + true + true + false + false + true + false + false + false + false + true + false + false + true + true + 0 + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + ${numberOfThread} + ${numberOfThread} + ${duration} + true + true + stopthread + + -1 + false + + + + + true + false + + + + ${server} + ${port} + ${protocol} + / + true + GET + true + false + + + + + + + + + Sec-Fetch-Mode + navigate + + + Sec-Fetch-Site + none + + + Accept-Language + en-US,en;q=0.9 + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + + + diff --git a/loadtests/JmeterTest/v17/MemberLogin.jmx b/loadtests/JmeterTest/v17/MemberLogin.jmx new file mode 100644 index 0000000..7de2e2d --- /dev/null +++ b/loadtests/JmeterTest/v17/MemberLogin.jmx @@ -0,0 +1,513 @@ + + + + + + + + false + false + + + + + + numberOfThread + 1 + = + + + duration + 120 + = + + + server + localhost + = + + + protocol + https + = + + + port + 44322 + = + + + backoffice_username + hnd@acceptance.test + = + + + backoffice_password + 0123456789 + = + + + totalOfMember + 30 + = + + + member_password + Test1234! + = + + + + + + ${server} + ${port} + ${protocol} + + + + + + + + + true + false + + + + true + false + false + + + + false + + saveConfig + + + true + true + true + + true + true + true + false + false + true + false + false + false + false + true + false + false + true + true + 0 + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + ${numberOfThread} + ${numberOfThread} + ${duration} + true + true + stopthread + + -1 + false + + + + + groovy + + + true + int total = (vars.get("totalOfMember") ?: "1") as int +int idx = new Random().nextInt(total) + 1 +vars.put("member_username", "TestMember_" + idx) + + + + true + false + + + + ${server} + ${port} + ${protocol} + /testmember_member-login/ + true + GET + true + false + + + + + + + + + Sec-Fetch-Mode + navigate + + + Sec-Fetch-Site + none + + + Accept-Language + en-US,en;q=0.9 + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + true + false + + + + Detected the start of a redirect chain + ${server} + ${port} + ${protocol} + /umbraco/api/memberlogin/login + true + POST + true + false + + + + true + RedirectUrl + /testmember_member-area/ + = + true + + + true + LoginUrl + /testmember_member-login/ + = + true + + + false + Username + ${member_username} + = + true + + + true + Password + ${member_password} + = + true + + + + + + + + + Sec-Fetch-Mode + navigate + + + Referer + ${protocol}://${server}:${port}/testmember_member-login/ + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Origin + ${protocol}://${server}:${port} + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Content-Type + application/x-www-form-urlencoded + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + true + false + + + + ${server} + ${port} + ${protocol} + /testmember_member-area/ + true + GET + true + false + + + + + + + + + Sec-Fetch-Mode + navigate + + + Referer + ${protocol}://${server}:${port}/testmember_member-login/ + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + true + false + + + + Detected the start of a redirect chain + ${server} + ${port} + ${protocol} + /umbraco/api/memberlogin/logout + true + POST + true + false + + + + true + RedirectUrl + /testmember_member-login/ + = + true + + + + + + + + + Sec-Fetch-Mode + navigate + + + Referer + ${protocol}://${server}:${port}/testmember_member-area/ + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Origin + ${protocol}://${server}:${port} + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Content-Type + application/x-www-form-urlencoded + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + + + diff --git a/loadtests/JmeterTest/v17/PublishContent.jmx b/loadtests/JmeterTest/v17/PublishContent.jmx new file mode 100644 index 0000000..df170b1 --- /dev/null +++ b/loadtests/JmeterTest/v17/PublishContent.jmx @@ -0,0 +1,1707 @@ + + + + + + + + false + false + + + + + + numberOfThread + 1 + = + + + duration + 120 + = + + + server + localhost + = + + + protocol + https + = + + + port + 44322 + = + + + backoffice_username + hnd@acceptance.test + = + + + backoffice_password + 0123456789 + = + + + + + + ${server} + ${port} + ${protocol} + + + + + + + + + true + false + + + + true + false + false + + + + false + + saveConfig + + + true + true + true + + true + true + true + false + false + true + false + false + false + false + true + false + false + true + true + 0 + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + ${numberOfThread} + ${numberOfThread} + ${duration} + true + true + continue + + -1 + false + + + + + true + false + + + + /umbraco + true + GET + true + false + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/server/status + true + GET + true + false + + + + + + + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/server/configuration + true + GET + true + false + + + + + + + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/manifest/manifest/public + true + GET + true + false + + + + + + + Generate PKCE then redirect to login + /umbraco/management/api/v1/security/back-office/authorize + true + GET + true + false + + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + client_id + umbraco-back-office + = + + + false + response_type + code + = + + + false + state + ${state} + = + + + false + scope + offline_access + = + + + false + prompt + consent + = + + + false + access_type + offline + = + + + false + code_challenge + ${code_challenge} + = + + + false + code_challenge_method + S256 + = + + + + + + + groovy + + + true + +import java.security.SecureRandom +import java.security.MessageDigest +import java.util.Base64 + +def random = new SecureRandom() +String chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" +StringBuilder sbState = new StringBuilder() +10.times { sbState.append(chars.charAt(random.nextInt(chars.length()))) } +vars.put("state", sbState.toString()) + +byte[] vBytes = new byte[96] +random.nextBytes(vBytes) +String codeVerifier = Base64.urlEncoder.withoutPadding().encodeToString(vBytes) +vars.put("code_verifier", codeVerifier) + +byte[] hashBytes = MessageDigest.getInstance("SHA-256").digest(codeVerifier.getBytes("ASCII")) +vars.put("code_challenge", Base64.urlEncoder.withoutPadding().encodeToString(hashBytes)) + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1/security/back-office/login + true + POST + true + true + + + + false + {"username":"${backoffice_username}","password":"${backoffice_password}"} + = + + + + + + + + + Content-Type + application/json + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + Sec-Fetch-Mode + cors + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/authorize + GET + true + false + + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + client_id + umbraco-back-office + = + + + false + response_type + code + = + + + false + state + ${state} + = + + + false + scope + offline_access + = + + + false + prompt + consent + = + + + false + access_type + offline + = + + + false + code_challenge + ${code_challenge} + = + + + false + code_challenge_method + S256 + = + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Referer + ${protocol}://${server}:${port}/umbraco/login + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + true + oauth_code + code=([^&\s\r\n]+) + $1$ + CODE_NOT_FOUND + 1 + false + + + + + /umbraco/management/api/v1/security/back-office/token + true + POST + true + false + + + + false + grant_type + authorization_code + = + + + false + client_id + umbraco-back-office + = + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + code + ${oauth_code} + = + + + false + code_verifier + ${code_verifier} + = + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + application/json, text/javascript, */*; q=0.01 + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + vars.put("access_token", json.access_token ?: "ACCESS_TOKEN_NOT_FOUND") + vars.put("refresh_token", json.refresh_token ?: "REFRESH_TOKEN_NOT_FOUND") +} catch(Exception e) { + vars.put("access_token", "ACCESS_TOKEN_NOT_FOUND") + vars.put("refresh_token", "REFRESH_TOKEN_NOT_FOUND") + log.error("token extract failed: " + e.getMessage()) +} + + + + + + /umbraco/management/api/v1/user/current/configuration + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/token + true + POST + true + false + + + + false + grant_type + refresh_token + = + + + false + client_id + umbraco-back-office + = + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + true + refresh_token + ${refresh_token} + = + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + application/json, text/javascript, */*; q=0.01 + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + if (json.access_token) vars.put("access_token", json.access_token) +} catch(Exception e) { + log.warn("refresh token update failed: " + e.getMessage()) +} + + + + + + /umbraco/management/api/v1/manifest/manifest/private + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/user/current + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/language + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1/recycle-bin/document/root + true + GET + true + false + + + + false + skip + 0 + = + true + + + false + take + 0 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/news-dashboard + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/tree/document/root + true + GET + true + false + + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size()-4)] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + vars.put("root_content_id", id) + // Set as target_document_id fallback – will be overridden if deeper levels exist + vars.put("target_document_id", id) + vars.put("published_root", published) + log.info("root_content_id=" + id + " published_root=" + published + " state=" + firstVariantState) + } else { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("target_document_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("published_root", "false") + log.error("No items in root tree response") + } +} catch(Exception e) { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("target_document_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("published_root", "false") + log.error("Extract root_content_id failed: " + e.getMessage()) +} + + + + + + + true + false + + + + ${__groovy(vars.get("root_content_id") != "ROOT_CONTENT_ID_NOT_FOUND" && vars.get("published_root") == "true")} + false + true + + + + /umbraco/management/api/v1/tree/document/children + true + GET + true + false + + + + false + parentId + ${root_content_id} + = + true + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size())] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + vars.put("node_id_l1", id) + // Update target_document_id – will be overridden if deeper levels exist + vars.put("target_document_id", id) + vars.put("published_l1", published) + log.info("node_id_l1=" + id + " published_l1=" + published + " state=" + firstVariantState) + } else { + vars.put("node_id_l1", "NODE_ID_L1_NOT_FOUND") + vars.put("published_l1", "false") + log.warn("L1: no children found under root_content_id=" + vars.get("root_content_id")) + } +} catch(Exception e) { + vars.put("node_id_l1", "NODE_ID_L1_NOT_FOUND") + vars.put("published_l1", "false") + log.error("Extract node_id_l1 failed: " + e.getMessage()) +} + + + + + + ${__groovy(vars.get("node_id_l1") != "NODE_ID_L1_NOT_FOUND" && vars.get("published_l1") == "true")} + false + true + + + + /umbraco/management/api/v1/tree/document/children + true + GET + true + false + + + + false + parentId + ${node_id_l1} + = + true + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size())] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + vars.put("node_id_l2", id) + // Update target_document_id – will be overridden if deeper levels exist + vars.put("target_document_id", id) + vars.put("published_l2", published) + log.info("node_id_l2=" + id + " published_l2=" + published + " state=" + firstVariantState) + } else { + vars.put("node_id_l2", "NODE_ID_L2_NOT_FOUND") + vars.put("published_l2", "false") + log.warn("L2: no children - target_document_id stays at L1=" + vars.get("target_document_id")) + } +} catch(Exception e) { + vars.put("node_id_l2", "NODE_ID_L2_NOT_FOUND") + vars.put("published_l2", "false") + log.error("Extract node_id_l2 failed: " + e.getMessage()) +} + + + + + + ${__groovy(vars.get("node_id_l2") != "NODE_ID_L2_NOT_FOUND" && vars.get("published_l2") == "true")} + false + true + + + + /umbraco/management/api/v1/tree/document/children + true + GET + true + false + + + + false + parentId + ${node_id_l2} + = + true + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size())] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + // Update target_document_id to deepest found level + vars.put("node_id_l3", id) + vars.put("target_document_id", id) + vars.put("published_l3", published) + log.info("target_document_id=" + id + " published_l3=" + published + " state=" + firstVariantState) + } else { + vars.put("node_id_l3", "NODE_ID_L3_NOT_FOUND") + vars.put("published_l3", "false") + log.warn("No items at level 4 – target_document_id stays at level 3") + } +} catch(Exception e) { + vars.put("node_id_l3", "NODE_ID_L3_NOT_FOUND") + vars.put("published_l3", "false") + log.error("Extract target_document_id failed: " + e.getMessage()) +} + + + + + + + + + /umbraco/management/api/v1/document/${target_document_id} + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content/workspace/document/edit/${target_document_id} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + vars.put("document_type_id", json?.documentType?.id?.toString() ?: "DOCTYPE_NOT_FOUND") + log.info("document_type_id=" + vars.get("document_type_id")) +} catch(Exception e) { + vars.put("document_type_id", "DOCTYPE_NOT_FOUND") + log.error("Extract document_type_id failed: " + e.getMessage()) +} + + + + + groovy + + + true + import groovy.json.JsonSlurper +import groovy.json.JsonOutput +import java.text.SimpleDateFormat +import java.util.Date + +def responseBody = prev.getResponseDataAsString() +try { + def doc = new JsonSlurper().parseText(responseBody) + def newName = 'LoadTest_' + new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()) + def cultures = [] + if (doc.variants) { + doc.variants.each { v -> + v.name = newName + if (v.culture) cultures << v.culture + } + } + def saveMap = doc.collectEntries { k, v -> [k, v] } + ["id", "documentType", "creator", "updater", "createDate", "updateDate", + "urls", "isDirty"].each { saveMap.remove(it) } + + vars.put("document_save_body", JsonOutput.toJson(saveMap)) + + def validateMap = new LinkedHashMap(saveMap) + validateMap.put("cultures", cultures ?: ["en-US"]) + vars.put("document_validate_body", JsonOutput.toJson(validateMap)) + + log.info("PUT bodies built. newName=" + newName + " | cultures=" + cultures) +} catch (Exception e) { + log.error("Build PUT body failed: " + e.getMessage()) + vars.put("document_save_body", '{"template":null,"values":[],"variants":[]}') + vars.put("document_validate_body", '{"template":null,"values":[],"variants":[],"cultures":["en-US"]}') +} + + + + + + /umbraco/management/api/v1/tree/document/ancestors + true + GET + true + false + + + + false + descendantId + ${target_document_id} + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document-type/${document_type_id} + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document/configuration + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1/document/${target_document_id} + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document/${target_document_id}/publish + true + PUT + true + true + + + + false + {"publishSchedules":[{"culture":"en-US"}]} + = + + + + + + + + + authorization + Bearer ${access_token} + + + content-type + application/json + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/tree/document/siblings + true + GET + true + false + + + + false + target + ${target_document_id} + = + true + + + false + before + 3 + = + true + + + false + after + 1 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1/security/back-office/revoke + true + POST + true + false + + + + true + token + ${access_token} + = + true + + + false + token_type_hint + access_token + = + true + + + false + client_id + umbraco-back-office + = + true + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/revoke + true + POST + true + false + + + + true + token + ${refresh_token} + = + true + + + false + token_type_hint + refresh_token + = + true + + + false + client_id + umbraco-back-office + = + true + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/signout + true + GET + true + false + + + + true + post_logout_redirect_uri + ${protocol}://${server}:${port}/umbraco/logout + = + true + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + + + diff --git a/loadtests/JmeterTest/v17/SaveAndPublishContent.jmx b/loadtests/JmeterTest/v17/SaveAndPublishContent.jmx new file mode 100644 index 0000000..51e4392 --- /dev/null +++ b/loadtests/JmeterTest/v17/SaveAndPublishContent.jmx @@ -0,0 +1,1896 @@ + + + + + + + + false + false + + + + + + numberOfThread + 1 + = + + + duration + 120 + = + + + server + localhost + = + + + protocol + https + = + + + port + 44322 + = + + + backoffice_username + hnd@acceptance.test + = + + + backoffice_password + 0123456789 + = + + + + + + ${server} + ${port} + ${protocol} + + + + + + + + + true + false + + + + true + false + false + + + + false + + saveConfig + + + true + true + true + + true + true + true + false + false + true + false + false + false + false + true + false + false + true + true + 0 + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + ${numberOfThread} + ${numberOfThread} + ${duration} + true + true + continue + + -1 + false + + + + + true + false + + + + /umbraco + true + GET + true + false + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/server/status + true + GET + true + false + + + + + + + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/server/configuration + true + GET + true + false + + + + + + + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/manifest/manifest/public + true + GET + true + false + + + + + + + Generate PKCE then redirect to login + /umbraco/management/api/v1/security/back-office/authorize + true + GET + true + false + + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + client_id + umbraco-back-office + = + + + false + response_type + code + = + + + false + state + ${state} + = + + + false + scope + offline_access + = + + + false + prompt + consent + = + + + false + access_type + offline + = + + + false + code_challenge + ${code_challenge} + = + + + false + code_challenge_method + S256 + = + + + + + + + groovy + + + true + +import java.security.SecureRandom +import java.security.MessageDigest +import java.util.Base64 + +def random = new SecureRandom() +String chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" +StringBuilder sbState = new StringBuilder() +10.times { sbState.append(chars.charAt(random.nextInt(chars.length()))) } +vars.put("state", sbState.toString()) + +byte[] vBytes = new byte[96] +random.nextBytes(vBytes) +String codeVerifier = Base64.urlEncoder.withoutPadding().encodeToString(vBytes) +vars.put("code_verifier", codeVerifier) + +byte[] hashBytes = MessageDigest.getInstance("SHA-256").digest(codeVerifier.getBytes("ASCII")) +vars.put("code_challenge", Base64.urlEncoder.withoutPadding().encodeToString(hashBytes)) + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1/security/back-office/login + true + POST + true + true + + + + false + {"username":"${backoffice_username}","password":"${backoffice_password}"} + = + + + + + + + + + Content-Type + application/json + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + Sec-Fetch-Mode + cors + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/authorize + GET + true + false + + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + client_id + umbraco-back-office + = + + + false + response_type + code + = + + + false + state + ${state} + = + + + false + scope + offline_access + = + + + false + prompt + consent + = + + + false + access_type + offline + = + + + false + code_challenge + ${code_challenge} + = + + + false + code_challenge_method + S256 + = + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Referer + ${protocol}://${server}:${port}/umbraco/login + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + true + oauth_code + code=([^&\s\r\n]+) + $1$ + CODE_NOT_FOUND + 1 + false + + + + + /umbraco/management/api/v1/security/back-office/token + true + POST + true + false + + + + false + grant_type + authorization_code + = + + + false + client_id + umbraco-back-office + = + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + code + ${oauth_code} + = + + + false + code_verifier + ${code_verifier} + = + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + application/json, text/javascript, */*; q=0.01 + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + vars.put("access_token", json.access_token ?: "ACCESS_TOKEN_NOT_FOUND") + vars.put("refresh_token", json.refresh_token ?: "REFRESH_TOKEN_NOT_FOUND") +} catch(Exception e) { + vars.put("access_token", "ACCESS_TOKEN_NOT_FOUND") + vars.put("refresh_token", "REFRESH_TOKEN_NOT_FOUND") + log.error("token extract failed: " + e.getMessage()) +} + + + + + + /umbraco/management/api/v1/user/current/configuration + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/token + true + POST + true + false + + + + false + grant_type + refresh_token + = + + + false + client_id + umbraco-back-office + = + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + true + refresh_token + ${refresh_token} + = + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + application/json, text/javascript, */*; q=0.01 + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + if (json.access_token) vars.put("access_token", json.access_token) +} catch(Exception e) { + log.warn("refresh token update failed: " + e.getMessage()) +} + + + + + + /umbraco/management/api/v1/manifest/manifest/private + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/user/current + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/language + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1/recycle-bin/document/root + true + GET + true + false + + + + false + skip + 0 + = + true + + + false + take + 0 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/news-dashboard + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/tree/document/root + true + GET + true + false + + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size()-4)] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + vars.put("root_content_id", id) + vars.put("target_document_id", id) + vars.put("published_root", published) + log.info("root_content_id=" + id + " published_root=" + published + " state=" + firstVariantState) + } else { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("target_document_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("published_root", "false") + log.error("No items in root tree response") + } +} catch(Exception e) { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("target_document_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("published_root", "false") + log.error("Extract root_content_id failed: " + e.getMessage()) +} + + + + + + + true + false + + + + ${__groovy(vars.get("root_content_id") != "ROOT_CONTENT_ID_NOT_FOUND" && vars.get("published_root") == "true")} + false + true + + + + /umbraco/management/api/v1/tree/document/children + true + GET + true + false + + + + false + parentId + ${root_content_id} + = + true + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size())] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + vars.put("node_id_l1", id) + vars.put("target_document_id", id) + vars.put("published_l1", published) + log.info("node_id_l1=" + id + " published_l1=" + published + " state=" + firstVariantState) + } else { + vars.put("node_id_l1", "NODE_ID_L1_NOT_FOUND") + vars.put("published_l1", "false") + log.warn("L1: no children found under root_content_id=" + vars.get("root_content_id")) + } +} catch(Exception e) { + vars.put("node_id_l1", "NODE_ID_L1_NOT_FOUND") + vars.put("published_l1", "false") + log.error("Extract node_id_l1 failed: " + e.getMessage()) +} + + + + + + ${__groovy(vars.get("node_id_l1") != "NODE_ID_L1_NOT_FOUND" && vars.get("published_l1") == "true")} + false + true + + + + /umbraco/management/api/v1/tree/document/children + true + GET + true + false + + + + false + parentId + ${node_id_l1} + = + true + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size())] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + vars.put("node_id_l2", id) + // Update target_document_id – will be overridden if deeper levels exist + vars.put("target_document_id", id) + vars.put("published_l2", published) + log.info("node_id_l2=" + id + " published_l2=" + published + " state=" + firstVariantState) + } else { + vars.put("node_id_l2", "NODE_ID_L2_NOT_FOUND") + vars.put("published_l2", "false") + log.warn("L2: no children - target_document_id stays at L1=" + vars.get("target_document_id")) + } +} catch(Exception e) { + vars.put("node_id_l2", "NODE_ID_L2_NOT_FOUND") + vars.put("published_l2", "false") + log.error("Extract node_id_l2 failed: " + e.getMessage()) +} + + + + + + ${__groovy(vars.get("node_id_l2") != "NODE_ID_L2_NOT_FOUND" && vars.get("published_l2") == "true")} + false + true + + + + /umbraco/management/api/v1/tree/document/children + true + GET + true + false + + + + false + parentId + ${node_id_l2} + = + true + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size())] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + // Update target_document_id to deepest found level + vars.put("node_id_l3", id) + vars.put("target_document_id", id) + vars.put("published_l3", published) + log.info("target_document_id=" + id + " published_l3=" + published + " state=" + firstVariantState) + } else { + vars.put("node_id_l3", "NODE_ID_L3_NOT_FOUND") + vars.put("published_l3", "false") + log.warn("No items at level 4 – target_document_id stays at level 3") + } +} catch(Exception e) { + vars.put("node_id_l3", "NODE_ID_L3_NOT_FOUND") + vars.put("published_l3", "false") + log.error("Extract target_document_id failed: " + e.getMessage()) +} + + + + + + + + + /umbraco/management/api/v1/document/${target_document_id} + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content/workspace/document/edit/${target_document_id} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + vars.put("document_type_id", json?.documentType?.id?.toString() ?: "DOCTYPE_NOT_FOUND") + log.info("document_type_id=" + vars.get("document_type_id")) +} catch(Exception e) { + vars.put("document_type_id", "DOCTYPE_NOT_FOUND") + log.error("Extract document_type_id failed: " + e.getMessage()) +} + + + + + groovy + + + true + import groovy.json.JsonSlurper +import groovy.json.JsonOutput +import java.text.SimpleDateFormat +import java.util.Date + +def responseBody = prev.getResponseDataAsString() +try { + def doc = new JsonSlurper().parseText(responseBody) + def timestamp = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()) + def newName = 'LoadTest_' + timestamp + def cultures = [] + def newNames = [] + if (doc.variants) { + doc.variants.each { v -> + v.name = newName + newNames << v.name + if (v.culture) cultures << v.culture + } + } + def saveMap = doc.collectEntries { k, v -> [k, v] } + ["id", "documentType", "creator", "updater", "createDate", "updateDate", + "urls", "isDirty"].each { saveMap.remove(it) } + + vars.put("document_save_body", JsonOutput.toJson(saveMap)) + + def validateMap = new LinkedHashMap(saveMap) + validateMap.put("cultures", cultures ?: ["en-US"]) + vars.put("document_validate_body", JsonOutput.toJson(validateMap)) + + log.info("PUT bodies built. newNames=" + newNames + " | cultures=" + cultures) +} catch (Exception e) { + log.error("Build PUT body failed: " + e.getMessage()) + vars.put("document_save_body", '{"template":null,"values":[],"variants":[]}') + vars.put("document_validate_body", '{"template":null,"values":[],"variants":[],"cultures":["en-US"]}') +} + + + + + /umbraco/management/api/v1/tree/document/ancestors + true + GET + true + false + + + + false + descendantId + ${target_document_id} + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document-type/${document_type_id} + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document/configuration + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1.1/document/${target_document_id}/validate + true + PUT + true + true + + + + false + ${document_validate_body} + = + + + + + + + + + authorization + Bearer ${access_token} + + + content-type + application/json + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document/${target_document_id} + true + PUT + true + true + + + + false + ${document_save_body} + = + + + + + + + + + authorization + Bearer ${access_token} + + + content-type + application/json + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document/${target_document_id} + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document/${target_document_id}/publish + true + PUT + true + true + + + + false + {"publishSchedules":[{"culture":"en-US"}]} + = + + + + + + + + + authorization + Bearer ${access_token} + + + content-type + application/json + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/tree/document/siblings + true + GET + true + false + + + + false + target + ${target_document_id} + = + + + false + before + 3 + = + + + false + after + 1 + = + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1/document/urls + true + GET + true + false + + + + false + id + ${target_document_id} + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + def first = (json instanceof List) ? (json ? json[0] : null) : json + def urlInfos = first?.urlInfos + if (urlInfos && urlInfos.size() > 0) { + def fullUrl = urlInfos[0].url?.toString() ?: "" + def path = fullUrl + def schemeIdx = fullUrl.indexOf("://") + if (schemeIdx >= 0) { + def afterScheme = fullUrl.substring(schemeIdx + 3) + def slashIdx = afterScheme.indexOf("/") + path = (slashIdx >= 0) ? afterScheme.substring(slashIdx) : "/" + } + if (!path.startsWith("/")) path = "/" + path + vars.put("content_url", path) + log.info("content_url=" + path + " (from " + fullUrl + ")") + } else { + vars.put("content_url", "CONTENT_URL_NOT_FOUND") + log.warn("No urlInfos found in document/urls response") + } +} catch(Exception e) { + vars.put("content_url", "CONTENT_URL_NOT_FOUND") + log.error("Extract content_url failed: " + e.getMessage()) +} + + + + + + ${content_url} + true + GET + true + false + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Accept-Language + en-US,en;q=0.9 + + + + + + + + true + false + + + + /umbraco/management/api/v1/security/back-office/revoke + true + POST + true + false + + + + true + token + ${access_token} + = + true + + + false + token_type_hint + access_token + = + true + + + false + client_id + umbraco-back-office + = + true + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/revoke + true + POST + true + false + + + + true + token + ${refresh_token} + = + true + + + false + token_type_hint + refresh_token + = + true + + + false + client_id + umbraco-back-office + = + true + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/signout + true + GET + true + false + + + + true + post_logout_redirect_uri + ${protocol}://${server}:${port}/umbraco/logout + = + true + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + + + diff --git a/loadtests/JmeterTest/v17/SaveContent.jmx b/loadtests/JmeterTest/v17/SaveContent.jmx new file mode 100644 index 0000000..81c3308 --- /dev/null +++ b/loadtests/JmeterTest/v17/SaveContent.jmx @@ -0,0 +1,1698 @@ + + + + + + + + false + false + + + + + + numberOfThread + 1 + = + + + duration + 120 + = + + + server + localhost + = + + + protocol + https + = + + + port + 44322 + = + + + backoffice_username + hnd@acceptance.test + = + + + backoffice_password + 0123456789 + = + + + + + + ${server} + ${port} + ${protocol} + + + + + + + + + true + false + + + + true + false + false + + + + false + + saveConfig + + + true + true + true + + true + true + true + false + false + true + false + false + false + false + true + false + false + true + true + 0 + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + ${numberOfThread} + ${numberOfThread} + ${duration} + true + true + continue + + -1 + false + + + + + true + false + + + + /umbraco + true + GET + true + false + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/server/status + true + GET + true + false + + + + + + + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/server/configuration + true + GET + true + false + + + + + + + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/manifest/manifest/public + true + GET + true + false + + + + + + + Generate PKCE then redirect to login + /umbraco/management/api/v1/security/back-office/authorize + true + GET + true + false + + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + client_id + umbraco-back-office + = + + + false + response_type + code + = + + + false + state + ${state} + = + + + false + scope + offline_access + = + + + false + prompt + consent + = + + + false + access_type + offline + = + + + false + code_challenge + ${code_challenge} + = + + + false + code_challenge_method + S256 + = + + + + + + + groovy + + + true + +import java.security.SecureRandom +import java.security.MessageDigest +import java.util.Base64 + +def random = new SecureRandom() +String chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" +StringBuilder sbState = new StringBuilder() +10.times { sbState.append(chars.charAt(random.nextInt(chars.length()))) } +vars.put("state", sbState.toString()) + +byte[] vBytes = new byte[96] +random.nextBytes(vBytes) +String codeVerifier = Base64.urlEncoder.withoutPadding().encodeToString(vBytes) +vars.put("code_verifier", codeVerifier) + +byte[] hashBytes = MessageDigest.getInstance("SHA-256").digest(codeVerifier.getBytes("ASCII")) +vars.put("code_challenge", Base64.urlEncoder.withoutPadding().encodeToString(hashBytes)) + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1/security/back-office/login + true + POST + true + true + + + + false + {"username":"${backoffice_username}","password":"${backoffice_password}"} + = + + + + + + + + + Content-Type + application/json + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + Sec-Fetch-Mode + cors + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/authorize + GET + true + false + + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + client_id + umbraco-back-office + = + + + false + response_type + code + = + + + false + state + ${state} + = + + + false + scope + offline_access + = + + + false + prompt + consent + = + + + false + access_type + offline + = + + + false + code_challenge + ${code_challenge} + = + + + false + code_challenge_method + S256 + = + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Referer + ${protocol}://${server}:${port}/umbraco/login + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + true + oauth_code + code=([^&\s\r\n]+) + $1$ + CODE_NOT_FOUND + 1 + false + + + + + /umbraco/management/api/v1/security/back-office/token + true + POST + true + false + + + + false + grant_type + authorization_code + = + + + false + client_id + umbraco-back-office + = + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + code + ${oauth_code} + = + + + false + code_verifier + ${code_verifier} + = + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + application/json, text/javascript, */*; q=0.01 + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + vars.put("access_token", json.access_token ?: "ACCESS_TOKEN_NOT_FOUND") + vars.put("refresh_token", json.refresh_token ?: "REFRESH_TOKEN_NOT_FOUND") +} catch(Exception e) { + vars.put("access_token", "ACCESS_TOKEN_NOT_FOUND") + vars.put("refresh_token", "REFRESH_TOKEN_NOT_FOUND") + log.error("token extract failed: " + e.getMessage()) +} + + + + + + /umbraco/management/api/v1/user/current/configuration + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/token + true + POST + true + false + + + + false + grant_type + refresh_token + = + + + false + client_id + umbraco-back-office + = + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + true + refresh_token + ${refresh_token} + = + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + application/json, text/javascript, */*; q=0.01 + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + if (json.access_token) vars.put("access_token", json.access_token) +} catch(Exception e) { + log.warn("refresh token update failed: " + e.getMessage()) +} + + + + + + /umbraco/management/api/v1/manifest/manifest/private + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/user/current + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/language + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1/recycle-bin/document/root + true + GET + true + false + + + + false + skip + 0 + = + true + + + false + take + 0 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/news-dashboard + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/tree/document/root + true + GET + true + false + + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size()-4)] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + vars.put("root_content_id", id) + // Set as target_document_id fallback – will be overridden if deeper levels exist + vars.put("target_document_id", id) + vars.put("published_root", published) + log.info("root_content_id=" + id + " published_root=" + published + " state=" + firstVariantState) + } else { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("target_document_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("published_root", "false") + log.error("No items in root tree response") + } +} catch(Exception e) { + vars.put("root_content_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("target_document_id", "ROOT_CONTENT_ID_NOT_FOUND") + vars.put("published_root", "false") + log.error("Extract root_content_id failed: " + e.getMessage()) +} + + + + + + + true + false + + + + ${__groovy(vars.get("root_content_id") != "ROOT_CONTENT_ID_NOT_FOUND" && vars.get("published_root") == "true")} + false + true + + + + /umbraco/management/api/v1/tree/document/children + true + GET + true + false + + + + false + parentId + ${root_content_id} + = + true + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size())] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + vars.put("node_id_l1", id) + // Update target_document_id – will be overridden if deeper levels exist + vars.put("target_document_id", id) + vars.put("published_l1", published) + log.info("node_id_l1=" + id + " published_l1=" + published + " state=" + firstVariantState) + } else { + vars.put("node_id_l1", "NODE_ID_L1_NOT_FOUND") + vars.put("published_l1", "false") + log.warn("L1: no children found under root_content_id=" + vars.get("root_content_id")) + } +} catch(Exception e) { + vars.put("node_id_l1", "NODE_ID_L1_NOT_FOUND") + vars.put("published_l1", "false") + log.error("Extract node_id_l1 failed: " + e.getMessage()) +} + + + + + + ${__groovy(vars.get("node_id_l1") != "NODE_ID_L1_NOT_FOUND" && vars.get("published_l1") == "true")} + false + true + + + + /umbraco/management/api/v1/tree/document/children + true + GET + true + false + + + + false + parentId + ${node_id_l1} + = + true + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size())] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + vars.put("node_id_l2", id) + // Update target_document_id – will be overridden if deeper levels exist + vars.put("target_document_id", id) + vars.put("published_l2", published) + log.info("node_id_l2=" + id + " published_l2=" + published + " state=" + firstVariantState) + } else { + vars.put("node_id_l2", "NODE_ID_L2_NOT_FOUND") + vars.put("published_l2", "false") + log.warn("L2: no children - target_document_id stays at L1=" + vars.get("target_document_id")) + } +} catch(Exception e) { + vars.put("node_id_l2", "NODE_ID_L2_NOT_FOUND") + vars.put("published_l2", "false") + log.error("Extract node_id_l2 failed: " + e.getMessage()) +} + + + + + + ${__groovy(vars.get("node_id_l2") != "NODE_ID_L2_NOT_FOUND" && vars.get("published_l2") == "true")} + false + true + + + + /umbraco/management/api/v1/tree/document/children + true + GET + true + false + + + + false + parentId + ${node_id_l2} + = + true + + + false + skip + 0 + = + true + + + false + take + 50 + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def items = new JsonSlurper().parseText(body).items + if (items && items.size() > 0) { + def item = items[new Random().nextInt(items.size())] + def id = item.id.toString() + def firstVariantState = item.variants ? item.variants[0]?.state?.toString() : null + def published = (firstVariantState == "Published") ? "true" : "false" + // Update target_document_id to deepest found level + vars.put("node_id_l3", id) + vars.put("target_document_id", id) + vars.put("published_l3", published) + log.info("target_document_id=" + id + " published_l3=" + published + " state=" + firstVariantState) + } else { + vars.put("node_id_l3", "NODE_ID_L3_NOT_FOUND") + vars.put("published_l3", "false") + log.warn("No items at level 4 – target_document_id stays at level 3") + } +} catch(Exception e) { + vars.put("node_id_l3", "NODE_ID_L3_NOT_FOUND") + vars.put("published_l3", "false") + log.error("Extract target_document_id failed: " + e.getMessage()) +} + + + + + + + + + /umbraco/management/api/v1/document/${target_document_id} + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco/section/content/workspace/document/edit/${target_document_id} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + vars.put("document_type_id", json?.documentType?.id?.toString() ?: "DOCTYPE_NOT_FOUND") + log.info("document_type_id=" + vars.get("document_type_id")) +} catch(Exception e) { + vars.put("document_type_id", "DOCTYPE_NOT_FOUND") + log.error("Extract document_type_id failed: " + e.getMessage()) +} + + + + + groovy + + + true + import groovy.json.JsonSlurper +import groovy.json.JsonOutput +import java.text.SimpleDateFormat +import java.util.Date + +def responseBody = prev.getResponseDataAsString() +try { + def doc = new JsonSlurper().parseText(responseBody) + def newName = 'LoadTest_' + new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()) + def cultures = [] + if (doc.variants) { + doc.variants.each { v -> + v.name = newName + if (v.culture) cultures << v.culture + } + } + def saveMap = doc.collectEntries { k, v -> [k, v] } + ["id", "documentType", "creator", "updater", "createDate", "updateDate", + "urls", "isDirty"].each { saveMap.remove(it) } + + vars.put("document_save_body", JsonOutput.toJson(saveMap)) + + def validateMap = new LinkedHashMap(saveMap) + validateMap.put("cultures", cultures ?: ["en-US"]) + vars.put("document_validate_body", JsonOutput.toJson(validateMap)) + + log.info("PUT bodies built. newName=" + newName + " | cultures=" + cultures) +} catch (Exception e) { + log.error("Build PUT body failed: " + e.getMessage()) + vars.put("document_save_body", '{"template":null,"values":[],"variants":[]}') + vars.put("document_validate_body", '{"template":null,"values":[],"variants":[],"cultures":["en-US"]}') +} + + + + + + /umbraco/management/api/v1/tree/document/ancestors + true + GET + true + false + + + + false + descendantId + ${target_document_id} + = + true + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document-type/${document_type_id} + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document/configuration + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1.1/document/${target_document_id}/validate + true + PUT + true + true + + + + false + ${document_validate_body} + = + + + + + + + + + authorization + Bearer ${access_token} + + + content-type + application/json + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document/${target_document_id} + true + PUT + true + true + + + + false + ${document_save_body} + = + + + + + + + + + authorization + Bearer ${access_token} + + + content-type + application/json + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/document/${target_document_id} + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1/security/back-office/revoke + true + POST + true + false + + + + true + token + ${access_token} + = + true + + + false + token_type_hint + access_token + = + true + + + false + client_id + umbraco-back-office + = + true + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/revoke + true + POST + true + false + + + + true + token + ${refresh_token} + = + true + + + false + token_type_hint + refresh_token + = + true + + + false + client_id + umbraco-back-office + = + true + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/signout + true + GET + true + false + + + + true + post_logout_redirect_uri + ${protocol}://${server}:${port}/umbraco/logout + = + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + + + diff --git a/loadtests/JmeterTest/v17/SaveDocumentType.jmx b/loadtests/JmeterTest/v17/SaveDocumentType.jmx new file mode 100644 index 0000000..8696c76 --- /dev/null +++ b/loadtests/JmeterTest/v17/SaveDocumentType.jmx @@ -0,0 +1,2297 @@ + + + + + + + + false + false + + + + + + numberOfThread + 1 + = + + + duration + 120 + = + + + server + localhost + = + + + protocol + https + = + + + port + 44322 + = + + + backoffice_username + hnd@acceptance.test + = + + + backoffice_password + 0123456789 + = + + + root_document_type + Variant Document Types + = + + + + + + ${server} + ${port} + ${protocol} + + + + + + + + + true + false + + + + true + false + false + + + + false + + saveConfig + + + true + true + true + + true + true + true + false + false + true + false + false + false + false + true + false + false + true + true + 0 + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + ${numberOfThread} + ${numberOfThread} + ${duration} + true + true + stopthread + + -1 + false + + + + + true + false + + + + /umbraco + true + GET + true + false + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/server/status + true + GET + true + false + + + + + + + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/server/configuration + true + GET + true + false + + + + + + + + + Accept + */* + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/manifest/manifest/public + true + GET + true + false + + + + + + + Generate PKCE then redirect to login + /umbraco/management/api/v1/security/back-office/authorize + true + GET + true + false + + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + client_id + umbraco-back-office + = + + + false + response_type + code + = + + + false + state + ${state} + = + + + false + scope + offline_access + = + + + false + prompt + consent + = + + + false + access_type + offline + = + + + false + code_challenge + ${code_challenge} + = + + + false + code_challenge_method + S256 + = + + + + + + + groovy + + + true + +import java.security.SecureRandom +import java.security.MessageDigest +import java.util.Base64 + +def random = new SecureRandom() +String chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" +StringBuilder sbState = new StringBuilder() +10.times { sbState.append(chars.charAt(random.nextInt(chars.length()))) } +vars.put("state", sbState.toString()) + +byte[] vBytes = new byte[96] +random.nextBytes(vBytes) +String codeVerifier = Base64.urlEncoder.withoutPadding().encodeToString(vBytes) +vars.put("code_verifier", codeVerifier) + +byte[] hashBytes = MessageDigest.getInstance("SHA-256").digest(codeVerifier.getBytes("ASCII")) +vars.put("code_challenge", Base64.urlEncoder.withoutPadding().encodeToString(hashBytes)) + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Referer + ${protocol}://${server}:${port}/umbraco + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + /umbraco/management/api/v1/security/back-office/login + true + POST + true + true + + + + false + {"username":"${backoffice_username}","password":"${backoffice_password}"} + = + + + + + + + + + Content-Type + application/json + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + Sec-Fetch-Mode + cors + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/authorize + GET + true + false + + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + client_id + umbraco-back-office + = + + + false + response_type + code + = + + + false + state + ${state} + = + + + false + scope + offline_access + = + + + false + prompt + consent + = + + + false + access_type + offline + = + + + false + code_challenge + ${code_challenge} + = + + + false + code_challenge_method + S256 + = + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Referer + ${protocol}://${server}:${port}/umbraco/login + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + true + oauth_code + code=([^&\s\r\n]+) + $1$ + CODE_NOT_FOUND + 1 + false + + + + + /umbraco/management/api/v1/security/back-office/token + true + POST + true + false + + + + false + grant_type + authorization_code + = + + + false + client_id + umbraco-back-office + = + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + false + code + ${oauth_code} + = + + + false + code_verifier + ${code_verifier} + = + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + application/json, text/javascript, */*; q=0.01 + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + vars.put("access_token", json.access_token ?: "ACCESS_TOKEN_NOT_FOUND") + vars.put("refresh_token", json.refresh_token ?: "REFRESH_TOKEN_NOT_FOUND") +} catch(Exception e) { + vars.put("access_token", "ACCESS_TOKEN_NOT_FOUND") + vars.put("refresh_token", "REFRESH_TOKEN_NOT_FOUND") + log.error("token extract failed: " + e.getMessage()) +} + + + + + + /umbraco/management/api/v1/user/current/configuration + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/token + true + POST + true + false + + + + false + grant_type + refresh_token + = + + + false + client_id + umbraco-back-office + = + + + true + redirect_uri + ${protocol}://${server}:${port}/umbraco/oauth_complete + = + + + true + refresh_token + ${refresh_token} + = + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + application/json, text/javascript, */*; q=0.01 + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + groovy + + + true + +import groovy.json.JsonSlurper +def body = prev.getResponseDataAsString() +try { + def json = new JsonSlurper().parseText(body) + if (json.access_token) vars.put("access_token", json.access_token) +} catch(Exception e) { + log.warn("refresh token update failed: " + e.getMessage()) +} + + + + + + /umbraco/management/api/v1/manifest/manifest/private + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/user/current + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/language + true + GET + true + false + + + + + + + + + authorization + Bearer ${access_token} + + + Accept + */* + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + true + false + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/document-type/root + true + GET + true + false + + + + false + skip + 0 + = + + + false + take + 0 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/member-type/root + true + GET + true + false + + + + false + skip + 0 + = + + + false + take + 0 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/media-type/root + true + GET + true + false + + + + false + skip + 0 + = + + + false + take + 0 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/document-blueprint/root + true + GET + true + false + + + + false + skip + 0 + = + + + false + take + 0 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/script/root + true + GET + true + false + + + + false + skip + 0 + = + + + false + take + 0 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/data-type/root + true + GET + true + false + + + + false + skip + 0 + = + + + false + take + 0 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/template/root + true + GET + true + false + + + + false + skip + 0 + = + + + false + take + 0 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/partial-view/root + true + GET + true + false + + + + false + skip + 0 + = + + + false + take + 0 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/stylesheet/root + true + GET + true + false + + + + false + skip + 0 + = + + + false + take + 0 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + + true + false + + + + ${server} + ${port} + ${protocol} + utf-8 + /umbraco/management/api/v1/tree/document-type/root + true + GET + true + false + + + + false + foldersOnly + false + = + + + false + skip + 0 + = + + + false + take + 50 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=0 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + variantDocTypeId + $..items[?(@.name=='${root_document_type}')].id + 1 + VARIANT_DOCTYPE_ID_NOT_FOUND + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/document-type/children + true + GET + true + false + + + + false + parentId + ${variantDocTypeId} + = + + + false + foldersOnly + false + = + + + false + skip + 0 + = + + + false + take + 50 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=0 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + randomDocTypeId + $..items[*].id + 0 + RANDOM_DOCTYPE_ID_NOT_FOUND + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/document-type/${randomDocTypeId} + true + GET + true + false + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings/workspace/document-type/edit/365bcb53-68b0-49ec-94d4-f41373a86eea + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + groovy + + + true + import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +def body = prev.getResponseDataAsString() +def json = new JsonSlurper().parseText(body) +// new name = 'LoadTest_' + timestamp +json.name = "LoadTest_" + System.currentTimeMillis() +vars.put("documentTypeJson", JsonOutput.toJson(json)) + + + + dataTypeId + $.properties[*].dataType.id + -1 + NOT_FOUND + + + + + dataTypeId + currentDataTypeId + true + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/data-type/${currentDataTypeId} + true + GET + true + false + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings/workspace/document-type/edit/${randomDocTypeId} + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + + ${server} + ${port} + ${protocol} + utf-8 + /umbraco/management/api/v1/tree/document-type/root + true + GET + true + false + + + + false + skip + 0 + = + + + false + take + 0 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings/workspace/document-type/edit/365bcb53-68b0-49ec-94d4-f41373a86eea + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/document-type/ancestors + true + GET + true + false + + + + false + descendantId + ${randomDocTypeId} + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings/workspace/document-type/edit/365bcb53-68b0-49ec-94d4-f41373a86eea + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + + true + false + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/document-type/${randomDocTypeId} + true + PUT + true + true + + + + false + ${documentTypeJson} + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings/workspace/document-type/edit/365bcb53-68b0-49ec-94d4-f41373a86eea/root + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Origin + ${protocol}://${server}:${port} + + + Priority + u=0 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + content-type + application/json + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + utf-8 + /umbraco/management/api/v1/document-type/${randomDocTypeId} + true + GET + true + false + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings/workspace/document-type/edit/365bcb53-68b0-49ec-94d4-f41373a86eea/root + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + ${server} + ${port} + ${protocol} + /umbraco/management/api/v1/tree/document-type/siblings + true + GET + true + false + + + + false + foldersOnly + false + = + + + false + target + ${randomDocTypeId} + = + + + false + before + 0 + = + + + false + after + 39 + = + + + + + + + + + Sec-Fetch-Mode + cors + + + Referer + ${protocol}://${server}:${port}/umbraco/section/settings/workspace/document-type/edit/365bcb53-68b0-49ec-94d4-f41373a86eea/root + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Priority + u=4 + + + Accept + */* + + + authorization + Bearer ${access_token} + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + empty + + + + + + + + true + false + + + + /umbraco/management/api/v1/security/back-office/revoke + true + POST + true + false + + + + true + token + ${access_token} + = + + + false + token_type_hint + access_token + = + + + false + client_id + umbraco-back-office + = + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/revoke + true + POST + true + false + + + + true + token + ${refresh_token} + = + + + false + token_type_hint + refresh_token + = + + + false + client_id + umbraco-back-office + = + + + + + + + + + Content-Type + application/x-www-form-urlencoded + + + Accept + */* + + + Origin + ${protocol}://${server}:${port} + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + /umbraco/management/api/v1/security/back-office/signout + true + GET + true + false + + + + true + post_logout_redirect_uri + ${protocol}://${server}:${port}/umbraco/logout + = + + + + + + + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + + + + + + + + diff --git a/loadtests/JmeterTest/v17/ViewHomePage.jmx b/loadtests/JmeterTest/v17/ViewHomePage.jmx new file mode 100644 index 0000000..abe99cc --- /dev/null +++ b/loadtests/JmeterTest/v17/ViewHomePage.jmx @@ -0,0 +1,513 @@ + + + + + + + + false + false + + + + + + numberOfThread + 1 + = + + + duration + 120 + = + + + server + localhost + = + + + protocol + https + = + + + port + 44322 + = + + + backoffice_username + hnd@acceptance.test + = + + + backoffice_password + 0123456789 + = + + + totalOfMember + 30 + = + + + member_password + Test1234! + = + + + + + + ${server} + ${port} + ${protocol} + + + + + + + + + true + false + + + + true + false + false + + + + false + + saveConfig + + + true + true + true + + true + true + true + false + false + true + false + false + false + false + true + false + false + true + true + 0 + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + ${numberOfThread} + ${numberOfThread} + ${duration} + true + true + continue + + -1 + false + + + + + groovy + + + true + int total = (vars.get("totalOfMember") ?: "1") as int +int idx = new Random().nextInt(total) + 1 +vars.put("member_username", "TestMember_" + idx) + + + + true + false + + + + ${server} + ${port} + ${protocol} + /testmember_member-login/ + true + GET + true + false + + + + + + + + + Sec-Fetch-Mode + navigate + + + Sec-Fetch-Site + none + + + Accept-Language + en-US,en;q=0.9 + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + true + false + + + + Detected the start of a redirect chain + ${server} + ${port} + ${protocol} + /umbraco/api/memberlogin/login + true + POST + true + false + + + + true + RedirectUrl + /testmember_member-area/ + = + true + + + true + LoginUrl + /testmember_member-login/ + = + true + + + false + Username + ${member_username} + = + true + + + true + Password + ${member_password} + = + true + + + + + + + + + Sec-Fetch-Mode + navigate + + + Referer + ${protocol}://${server}:${port}/testmember_member-login/ + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Origin + ${protocol}://${server}:${port} + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Content-Type + application/x-www-form-urlencoded + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + true + false + + + + ${server} + ${port} + ${protocol} + /testmember_member-area/ + true + GET + true + false + + + + + + + + + Sec-Fetch-Mode + navigate + + + Referer + ${protocol}://${server}:${port}/testmember_member-login/ + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + true + false + + + + Detected the start of a redirect chain + ${server} + ${port} + ${protocol} + /umbraco/api/memberlogin/logout + true + POST + true + false + + + + true + RedirectUrl + /testmember_member-login/ + = + true + + + + + + + + + Sec-Fetch-Mode + navigate + + + Referer + ${protocol}://${server}:${port}/testmember_member-area/ + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.9 + + + Origin + ${protocol}://${server}:${port} + + + Sec-Fetch-User + ?1 + + + Priority + u=0, i + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Content-Type + application/x-www-form-urlencoded + + + Accept-Encoding + gzip, deflate, br, zstd + + + User-Agent + Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0 + + + Sec-Fetch-Dest + document + + + + + + + + + + diff --git a/loadtests/_helpers.py b/loadtests/_helpers.py new file mode 100644 index 0000000..dcb9132 --- /dev/null +++ b/loadtests/_helpers.py @@ -0,0 +1,187 @@ +""" +Shared building blocks for scenario locustfiles. + +Each scenario ships its own locustfile at loadtests/scenarios//locustfile.py +and imports from here. Azure Load Testing flattens testPlan + configurationFiles +into one engine working dir, so `from _helpers import ...` resolves both on ALT +(flat layout) and locally (nested layout - locustfiles add the parent dir to +sys.path before importing; see the snippet at the top of each scenario file). + +Contents: + - INVENTORY_PATH / DELIVERY_API_LIST_PATH: endpoint constants. + - register_inventory_probe(): wires a test_start listener that fetches the + seeder inventory and buckets URLs onto environment.urls. + - register_delivery_api_probe(): wires a test_start listener that probes the + Delivery API and seeds environment.delivery_ids when reachable. + - pick_url(user, bucket, name): pick a random URL from a probed bucket. + Raises on empty bucket so a misconfigured run (seeder didn't seed, probe + failed) surfaces as a visible 100%-error task rather than silently + distorting the workload with fallback traffic. + +Scenarios declare their own @task methods explicitly so the workload that +runs for a given scenario is fully visible in that scenario's locustfile. +""" + +import logging +import os +import random +import uuid +import requests +from locust import events + +logger = logging.getLogger(__name__) + +INVENTORY_PATH = "/umbraco/api/seederstatus/inventory" +DELIVERY_API_LIST_PATH = "/umbraco/delivery/api/v2/content" + +# Deterministic PRNG seed. Fixed default across runs so URL picks (and any other +# random.* call) follow the same sequence, dropping cell-to-cell variance that's +# purely "which URLs got hit" rather than infra. Override per-run via the +# LOCUST_RANDOM_SEED env var when you specifically want randomised content +# selection (e.g. validating that the harness ISN'T sensitive to URL choice). +# Note: each engine/worker process re-seeds at module import, so multi-engine +# runs get the same per-worker sequence; full request-by-request reproducibility +# isn't promised, but aggregate URL distribution per run is now stable. +random.seed(int(os.environ.get("LOCUST_RANDOM_SEED", "42"))) + + +def register_inventory_probe(): + """Fetch the seeded URL inventory at test start and bucket by content type.""" + @events.test_start.add_listener + def _on_start(environment, **_): + environment.urls = {"section": [], "category": [], "page": [], "detail": [], "media": []} + if not environment.host: + logger.warning("No host configured; skipping inventory probe") + return + + try: + response = requests.get(environment.host.rstrip("/") + INVENTORY_PATH, timeout=15) + response.raise_for_status() + data = response.json() + except (requests.RequestException, ValueError) as ex: + # Buckets stay empty; pick_url then raises per-call so the run shows up + # as 100%-error on the affected tasks rather than silently homepage-only. + logger.error(f"Inventory unreachable ({ex}); workload tasks will fail") + return + + if not isinstance(data, dict): + logger.error(f"Inventory returned non-object JSON ({type(data).__name__}); workload tasks will fail") + return + + # Defensive filtering - a single malformed sample would otherwise crash + # the listener and skip workload setup for every VU in this engine. + samples = [s for s in data.get("sampleContentUrls", []) if isinstance(s, dict) and "url" in s] + environment.urls = { + "section": [u for u in data.get("rootSectionUrls", []) if isinstance(u, str)], + "category": [s["url"] for s in samples if s.get("docType") == "Category"], + "page": [s["url"] for s in samples if s.get("docType") == "Page"], + "detail": [s["url"] for s in samples if s.get("docType") == "Detail"], + "media": [u for u in data.get("sampleMediaUrls", []) if isinstance(u, str)], + } + counts = ", ".join(f"{k}={len(v)}" for k, v in environment.urls.items()) + logger.info(f"Inventory loaded: {counts}") + + +def register_delivery_api_probe(): + """Probe the Delivery API at test start. On 200 seed environment.delivery_ids. + + Pages through the full set (in 100-item batches up to a safety cap) so + delivery_item picks from every seeded item, not a 50-item slice that would + sit hot in cache and make the test measure cache lookup rather than query + work. The safety cap is in place so a future seeder bug yielding millions + of items doesn't stall test_start. + """ + PAGE_SIZE = 100 + MAX_ITEMS = 5000 # safety cap; current Massive preset is ~10k docs + + @events.test_start.add_listener + def _on_start(environment, **_): + environment.delivery_ids = [] + if not environment.host: + return + + base = environment.host.rstrip("/") + DELIVERY_API_LIST_PATH + ids = [] + skip = 0 + while len(ids) < MAX_ITEMS: + try: + response = requests.get(f"{base}?skip={skip}&take={PAGE_SIZE}", timeout=10) + except requests.RequestException as ex: + logger.warning(f"Delivery API probe failed at skip={skip} ({ex})") + break + + if response.status_code != 200: + if skip == 0: + logger.warning(f"Delivery API probe returned {response.status_code} (expected 200 in this scenario)") + break + + try: + body = response.json() + except ValueError: + logger.warning(f"Delivery API probe returned 200 at skip={skip} but body was not JSON") + break + + if not isinstance(body, dict): + break + + page = [i["id"] for i in body.get("items", []) if isinstance(i, dict) and "id" in i] + if not page: + break # ran past the end + ids.extend(page) + if len(page) < PAGE_SIZE: + break # last page + skip += PAGE_SIZE + + environment.delivery_ids = ids + logger.info(f"Delivery API enabled: {len(ids)} items inventoried") + + +def pick_url(user, bucket: str, name: str) -> None: + """Pick a random URL from a probed bucket. Raises on empty bucket.""" + urls = user.environment.urls.get(bucket) + if not urls: + raise RuntimeError(f"Bucket '{bucket}' is empty - inventory probe failed or seeder didn't seed it") + user.client.get(random.choice(urls), name=name) + + +def post_contact_form(user, name: str = "ContactFormSubmit") -> None: + """Submit a contact form with a randomised payload, asserting on the body. + + Per-call uuid so DB unique-constraints (if added) and SQL Server's plan/page + cache can't short-circuit the write path — an identical payload every call + masks real insert pressure and undercounts the tier-differentiating SQL load. + + catch_response so a 200 OK carrying validation errors (e.g. {"success": + false, "errors": [...]}) isn't silently counted as a successful write. The + endpoint normally returns either a JSON success payload or a 4xx — anything + else (including 2xx with an error body) is treated as a failure. + """ + token = uuid.uuid4().hex + payload = { + "name": f"LoadTest VU {token[:8]}", + "email": f"loadtest+{token}@example.com", + "subject": f"Locust submission {token[:8]}", + "message": "Auto-generated submission from the Umbraco load-test locustfile.", + } + with user.client.post( + "/umbraco/api/contactform/submit", + json=payload, + name=name, + catch_response=True, + ) as response: + if response.status_code >= 400: + response.failure(f"HTTP {response.status_code}") + return + try: + body = response.json() + except ValueError: + if response.text: + response.failure("non-JSON body") + return + if isinstance(body, dict): + if body.get("success") is False: + response.failure(f"success=false: {body.get('errors') or body}") + return + errors = body.get("errors") or body.get("Errors") + if errors: + response.failure(f"errors in body: {errors}") diff --git a/loadtests/scenarios/Default/AdditionalSetup/appsettings.json b/loadtests/scenarios/Default/AdditionalSetup/appsettings.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/loadtests/scenarios/Default/AdditionalSetup/appsettings.json @@ -0,0 +1 @@ +{} diff --git a/loadtests/scenarios/Default/locustfile.py b/loadtests/scenarios/Default/locustfile.py new file mode 100644 index 0000000..afd7c95 --- /dev/null +++ b/loadtests/scenarios/Default/locustfile.py @@ -0,0 +1,65 @@ +""" +Default scenario: traditional Umbraco customer profile. + +Browses rendered pages (homepage, sections, categories, pages, details, media) +and submits the occasional contact form. Models a customer who uses Umbraco's +standard content delivery without the Delivery API. + +Weights skew toward Detail (deepest read path, most SQL pressure); homepage +is output-cached and kept low so it doesn't dominate metrics. Total weight +113; write share 8/113 ≈ 7%. +""" + +# Locally: locustfile lives in scenarios// but imports _helpers.py from +# loadtests/. Insert the parent's parent on sys.path so the import resolves. +# On Azure Load Testing this is a no-op since ALT flattens all files into one +# engine working dir (helpers land next to the locustfile). +import sys +import pathlib +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2])) + +from locust import FastHttpUser, between, task + +from _helpers import pick_url, post_contact_form, register_inventory_probe + +register_inventory_probe() + + +class FrontEndUser(FastHttpUser): + # ~0.5 req/s per VU. 100 VUs ≈ 500-1500 real visitors in load-equivalent + # (humans wait 5-30 s between clicks). Fine for relative comparison; + # do not read VU counts as concurrent humans. + wait_time = between(1, 3) + + # @task(N) is a relative pick weight - Locust picks per VU by weighted + # random, so e.g. detail (35) fires 35/113 ≈ 31% of the time. Only the + # ratios matter; multiplying every weight by 10 changes nothing. + + @task(5) + def homepage(self): + self.client.get("/", name="Homepage") + + @task(10) + def section(self): + pick_url(self, "section", "Section") + + @task(20) + def category(self): + pick_url(self, "category", "Category") + + @task(30) + def page(self): + pick_url(self, "page", "Page") + + @task(35) + def detail(self): + pick_url(self, "detail", "Detail") + + @task(5) + def media(self): + pick_url(self, "media", "Media") + + # Write path - each call creates an Umbraco content node → ~10-15 SQL inserts. + @task(8) + def submit_contact_form(self): + post_contact_form(self) diff --git a/loadtests/scenarios/Default/scenario.yaml b/loadtests/scenarios/Default/scenario.yaml new file mode 100644 index 0000000..5889f90 --- /dev/null +++ b/loadtests/scenarios/Default/scenario.yaml @@ -0,0 +1 @@ +description: "Default Umbraco configuration; load profile from pipeline parameters." diff --git a/loadtests/scenarios/DeliveryApi/AdditionalSetup/Program.cs b/loadtests/scenarios/DeliveryApi/AdditionalSetup/Program.cs new file mode 100644 index 0000000..3068a4d --- /dev/null +++ b/loadtests/scenarios/DeliveryApi/AdditionalSetup/Program.cs @@ -0,0 +1,39 @@ +// Program.cs overlay for the DeliveryApi scenario. +// +// Modern Umbraco's Delivery API needs BOTH an appsettings flag AND a code-side +// builder registration. The appsettings overlay (AdditionalSetup/appsettings.json) +// flips the feature on; this Program.cs adds .AddDeliveryApi() so the DI services +// behind the API endpoints (IRequestSegmentService and friends) are registered. +// Without this, hitting /umbraco/delivery/api/v2/content returns 500 with +// "Unable to resolve service for type 'IRequestSegmentService'". +// +// The rest of this file mirrors the default `dotnet new umbraco` Program.cs — +// the install script overwrites the template-generated Program.cs with this one, +// so anything not here is dropped. Keep in sync with v17's template shape. + +WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + +builder.CreateUmbracoBuilder() + .AddBackOffice() + .AddWebsite() + .AddDeliveryApi() + .AddComposers() + .Build(); + +WebApplication app = builder.Build(); + +await app.BootUmbracoAsync(); + +app.UseUmbraco() + .WithMiddleware(u => + { + u.UseBackOffice(); + u.UseWebsite(); + }) + .WithEndpoints(u => + { + u.UseBackOfficeEndpoints(); + u.UseWebsiteEndpoints(); + }); + +await app.RunAsync(); diff --git a/loadtests/scenarios/DeliveryApi/AdditionalSetup/appsettings.json b/loadtests/scenarios/DeliveryApi/AdditionalSetup/appsettings.json new file mode 100644 index 0000000..da6d6be --- /dev/null +++ b/loadtests/scenarios/DeliveryApi/AdditionalSetup/appsettings.json @@ -0,0 +1,11 @@ +{ + "Umbraco": { + "CMS": { + "DeliveryApi": { + "Enabled": true, + "PublicAccess": true, + "RichTextOutputAsJson": false + } + } + } +} diff --git a/loadtests/scenarios/DeliveryApi/locustfile.py b/loadtests/scenarios/DeliveryApi/locustfile.py new file mode 100644 index 0000000..3be579c --- /dev/null +++ b/loadtests/scenarios/DeliveryApi/locustfile.py @@ -0,0 +1,76 @@ +""" +DeliveryApi scenario: headless Umbraco customer profile. + +Hits the Content Delivery API (list + item-by-id), plus media (which a +headless frontend still loads from Umbraco for assets), plus the occasional +write back to Umbraco. No rendered-page traffic - real headless customers +run a separate frontend (Next.js / Astro / etc.) that consumes the API; the +rendered Umbraco URLs either don't exist or aren't user-facing in production. + +Weights: delivery_item dominates because each frontend page render typically +fetches one item by id. delivery_list models occasional pagination. Media is +supporting traffic (one or two images per rendered page). Writes are rare +but exercised so SQL-write contention is on the radar. +Total weight 48; write share 8/48 ≈ 17% (high - bring down once more write +tasks land beyond just contact form). +""" + +# Locally: locustfile lives in scenarios// but imports _helpers.py from +# loadtests/. Insert the parent's parent on sys.path so the import resolves. +# On Azure Load Testing this is a no-op since ALT flattens all files into one +# engine working dir (helpers land next to the locustfile). +import sys +import pathlib +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2])) + +import random + +from locust import FastHttpUser, between, task + +from _helpers import ( + DELIVERY_API_LIST_PATH, + pick_url, + post_contact_form, + register_delivery_api_probe, + register_inventory_probe, +) + +# Inventory probe is needed for the media bucket; delivery probe seeds the +# ids list for delivery_item. Both must succeed or the relevant tasks fail. +register_inventory_probe() +register_delivery_api_probe() + + +class FrontEndUser(FastHttpUser): + wait_time = between(1, 3) + + # @task(N) is a relative pick weight - Locust picks per VU by weighted + # random, so e.g. delivery_item (25) fires 25/48 ≈ 52% of the time. Only + # the ratios matter; multiplying every weight by 10 changes nothing. + + @task(10) + def delivery_list(self): + skip = random.randint(0, 5) * 20 + self.client.get( + f"{DELIVERY_API_LIST_PATH}?skip={skip}&take=20", + name="DeliveryApiList", + ) + + @task(25) + def delivery_item(self): + ids = self.environment.delivery_ids + if not ids: + raise RuntimeError("Delivery API probe returned zero items - scenario misconfigured") + self.client.get( + f"{DELIVERY_API_LIST_PATH}/item/{random.choice(ids)}", + name="DeliveryApiItem", + ) + + @task(5) + def media(self): + pick_url(self, "media", "Media") + + # Write path - each call creates an Umbraco content node → ~10-15 SQL inserts. + @task(8) + def submit_contact_form(self): + post_contact_form(self) diff --git a/loadtests/scenarios/DeliveryApi/scenario.yaml b/loadtests/scenarios/DeliveryApi/scenario.yaml new file mode 100644 index 0000000..20940e0 --- /dev/null +++ b/loadtests/scenarios/DeliveryApi/scenario.yaml @@ -0,0 +1 @@ +description: "Headless mode: Umbraco Content Delivery API enabled with public access. Dedicated locustfile hits delivery list + item endpoints." diff --git a/loadtests/tiers.json b/loadtests/tiers.json new file mode 100644 index 0000000..ae1f230 --- /dev/null +++ b/loadtests/tiers.json @@ -0,0 +1,20 @@ +{ + "tiers": { + "Starter": { + "app_sku": "P0v3", + "dtu_max": 20 + }, + "Standard": { + "app_sku": "P1v3", + "dtu_max": 50 + }, + "Pro": { + "app_sku": "P2v3", + "dtu_max": 100 + }, + "Enterprise": { + "app_sku": "P3v3", + "dtu_max": 200 + } + } +} diff --git a/scripts/_helpers.ps1 b/scripts/_helpers.ps1 new file mode 100644 index 0000000..4aadc95 --- /dev/null +++ b/scripts/_helpers.ps1 @@ -0,0 +1,93 @@ +# Shared helpers; dot-source via `. "$PSScriptRoot/_helpers.ps1"`. + +# Nearest-rank percentile (ceil-based, so p99 of N=100 is the 99th value, not max). +function Get-Pct ($Sorted, [double]$Pct) { + if ($Sorted.Count -eq 0) { return 0 } + $i = [int][math]::Ceiling($Sorted.Count * $Pct / 100.0) - 1 + if ($i -lt 0) { $i = 0 } + if ($i -gt $Sorted.Count - 1) { $i = $Sorted.Count - 1 } + return $Sorted[$i] +} + +function Get-StorageAccountKey { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string]$StorageAccountName, + [Parameter(Mandatory)] [string]$ResourceGroupName + ) + $key = az storage account keys list ` + -n $StorageAccountName -g $ResourceGroupName ` + --query "[0].value" -o tsv + if (-not $key) { + Write-PipelineError "Could not read storage key for '$StorageAccountName' in '$ResourceGroupName' (requires Microsoft.Storage/storageAccounts/listKeys/action)." + } + return $key +} + +# ##vso[task.logissue type=error] surfaces in the AzDO run summary; plain Write-Error doesn't. +function Write-PipelineError([string] $Message) { + Write-Host "##vso[task.logissue type=error]$Message" + exit 1 +} + +function Get-UmbracoMajor([string] $Version) { + $raw = ($Version -split '\.')[0] + $major = 0 + if (-not [int]::TryParse($raw, [ref]$major)) { + Write-PipelineError "Cannot parse Umbraco major from '$Version' (expected X.Y.Z[-suffix])." + } + return $major +} + +# Stream-parse one engine_results.csv (JMeter format). Memory-bounded: keeps +# per-task sample lists, not the whole CSV. Returns: +# @{ ByLabel = @{ '