Widthdom · Widthdom · Jul 25, 2026 · Jul 25, 2026
diff --git a/.github/workflows/benchmark-regression.yml b/.github/workflows/benchmark-regression.yml
@@ -20,6 +20,12 @@ on:
       - 'CLAUDE.md'
       - 'LICENSE'
   workflow_dispatch:
+    inputs:
+      publish_baseline:
+        description: 'Publish this main-branch run as an intentional trusted baseline'
+        required: false
+        default: false
+        type: boolean
 
 permissions:
   contents: read
@@ -84,21 +90,44 @@ jobs:
             echo "Baseline branch gh-benchmarks does not exist yet. Will be created on first push to main."
           fi
 
-      - name: Detect performance regression
+      - name: Resolve intended benchmark base
+        id: benchmark-base
+        shell: bash
+        run: |
+          base_sha="${{ github.event.pull_request.base.sha || github.event.before || github.sha }}"
+          if [[ ! "$base_sha" =~ ^[0-9a-f]{40}$ ]] || ! git cat-file -e "$base_sha^{commit}"; then
+            base_sha="$GITHUB_SHA"
+          fi
+          echo "sha=$base_sha" >> "$GITHUB_OUTPUT"
+
+      - name: Fetch trusted benchmark history
         if: steps.check-baseline.outputs.exists == 'true'
-        uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1
-        with:
-          name: 'FolderDiffIL4DotNet Performance'
-          tool: 'benchmarkdotnet'
-          output-file-path: BenchmarkDotNet.Artifacts/results/combined-report.json
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          auto-push: false
-          alert-threshold: '200%'
-          comment-on-alert: false
-          fail-on-alert: true
-          summary-always: true
-          gh-pages-branch: gh-benchmarks
-          benchmark-data-dir-path: dev/bench
+        run: |
+          git fetch --no-tags origin gh-benchmarks:refs/remotes/origin/gh-benchmarks
+          git show refs/remotes/origin/gh-benchmarks:dev/bench/data.js \
+            > BenchmarkDotNet.Artifacts/results/baseline-data.js
+
+      - name: Check evidence-based performance thresholds
+        if: steps.check-baseline.outputs.exists == 'true'
+        run: |
+          python3 scripts/check_benchmark_regressions.py \
+            --current-report BenchmarkDotNet.Artifacts/results/combined-report.json \
+            --history-data BenchmarkDotNet.Artifacts/results/baseline-data.js \
+            --policy benchmark-regression-policy.json \
+            --baseline-ancestor "${{ steps.benchmark-base.outputs.sha }}" \
+            --repository-root . \
+            --summary "$GITHUB_STEP_SUMMARY" \
+            --allow-failure "${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.publish_baseline }}"
+
+      - name: Report missing benchmark baseline
+        if: steps.check-baseline.outputs.exists != 'true'
+        run: |
+          {
+            echo "## Performance regression gate"
+            echo
+            echo "⚠️ **WARMUP** — No trusted gh-benchmarks baseline exists yet."
+            echo "This run remains visible and will seed the baseline only from a trusted main-branch publication."
+          } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Upload benchmark results
         if: always()
@@ -117,7 +146,11 @@ jobs:
 
   publish-benchmark-baseline:
     needs: benchmark-regression
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    if: >-
+      (github.event_name == 'push' && github.ref == 'refs/heads/main') ||
+      (github.event_name == 'workflow_dispatch' &&
+       github.ref == 'refs/heads/main' &&
+       inputs.publish_baseline)
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -161,10 +194,6 @@ jobs:
           output-file-path: BenchmarkDotNet.Artifacts/results/combined-report.json
           github-token: ${{ secrets.GITHUB_TOKEN }}
           auto-push: true
-          alert-threshold: '200%'
-          comment-on-alert: false
-          fail-on-alert: true
-          summary-always: true
           gh-pages-branch: gh-benchmarks
           benchmark-data-dir-path: dev/bench
           skip-fetch-gh-pages: ${{ steps.check-baseline.outputs.exists != 'true' }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,6 +21,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 #### Changed
 
+- **Evidence-based benchmark regression limits** — Replaced the single 200% emergency ceiling with a compatible-history median gate and per-stability-group warning/failure limits from 10/20% through 50/100%. The policy records seven same-definition `ubuntu-latest` samples, observed variance, threshold rationale, definition fingerprint inputs, sample minimum, and baseline revision. Actions summaries now show every pass, warning, failure, warmup, and excluded incompatible history entry; controlled fixtures protect calculation and exact boundaries. Trusted `main` publication remains automatic, with an explicit main-only manual mode for reviewed baseline resets. Affected: `.github/workflows/benchmark-regression.yml`, `benchmark-regression-policy.json`, `scripts/check_benchmark_regressions.py`, `scripts/tests/test_check_benchmark_regressions.py`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, `doc/PERFORMANCE_GUIDE.md`, `doc/DEVELOPER_GUIDE.md`, `doc/TESTING_GUIDE.md`. Issue: #225.
+
 - **Private vulnerability reporting and a streamlined security policy** — The repository now accepts confidential reports through GitHub private vulnerability reporting. `SECURITY.md` has one complete English section and one equivalent Japanese section covering supported versions, the direct private-reporting path, report contents, response targets, coordinated disclosure, and the preserved threat model. Regression coverage protects the reporting link, bilingual policy structure, and Japanese documentation anchors. Affected: `SECURITY.md`, `README.md`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, and the GitHub repository security setting. Issue: #223.
 
 - **Release notes now use explicit dynamic comparison links** — The release workflow resolves the latest non-draft, non-prerelease stable GitHub Release by publication time, combines its tag with the current release tag to build the `Full Changelog` comparison URL, and appends the NuGet install/update commands in a consistent release-note template. Regression coverage keeps the release filtering, publication-time selection, dynamic URL, section order, and `nildiff` commands intact. Affected: `.github/workflows/release.yml`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`.
@@ -1706,6 +1708,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 #### 変更
 
+- **実測に基づくベンチマーク回帰上限** — 単一の緊急上限 200% を、互換履歴の中央値ゲートと、安定性グループ別の warning/failure 上限（10/20%～50/100%）へ置き換えました。ポリシーには、同一定義の `ubuntu-latest` 7 サンプル、観測分散、閾値根拠、定義フィンガープリント入力、最小サンプル数、baseline revision を記録します。Actions summary はすべての pass、warning、failure、warmup、除外した非互換履歴を表示し、制御 fixture で計算と正確な境界を保護します。信頼済み `main` の公開は引き続き自動で、レビュー済みベースラインリセット用に `main` 限定の明示的な手動モードも追加しました。対象: `.github/workflows/benchmark-regression.yml`, `benchmark-regression-policy.json`, `scripts/check_benchmark_regressions.py`, `scripts/tests/test_check_benchmark_regressions.py`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, `doc/PERFORMANCE_GUIDE.md`, `doc/DEVELOPER_GUIDE.md`, `doc/TESTING_GUIDE.md`。Issue: #225。
+
 - **Private vulnerability reporting とセキュリティポリシーの整理** — リポジトリは GitHub private vulnerability reporting を通じて機密報告を受け付けるようになりました。`SECURITY.md` を、サポート対象バージョン、private report への直接リンク、報告内容、対応目標、協調開示、維持した脅威モデルを一読で確認できる、完全な英語節と同等の日本語節へ整理しました。報告リンク、英日ポリシー構造、日本語文書のアンカーを回帰テストで保護します。対象: `SECURITY.md`, `README.md`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, GitHub リポジトリのセキュリティ設定。Issue: #223.
 
 - **リリースノートの比較リンクを明示的かつ動的に生成** — リリース workflow は公開日時を基準に、非 Draft・非 Prerelease の最新安定 GitHub Release を解決し、そのタグを今回のリリースタグと組み合わせて `Full Changelog` の比較 URL を構築し、NuGet の install/update コマンドを統一されたリリースノートテンプレートへ追加するようになりました。Release の絞り込み、公開日時による選択、動的 URL、セクション順序、`nildiff` コマンドを回帰テストで保護します。対象: `.github/workflows/release.yml`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`。

diff --git a/FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs b/FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs
@@ -627,26 +627,41 @@ public void DotNetWorkflow_BuildsAndSmokeTestsOnMacOs()
         }
 
         /// <summary>
-        /// Verifies that the benchmark regression workflow detects performance degradation on PRs.
-        /// ベンチマークリグレッションワークフローが PR でパフォーマンス劣化を検知することを検証します。
+        /// Verifies that benchmark regression CI uses evidence-based compatible-history thresholds.
+        /// ベンチマーク回帰 CI が実測に基づく互換履歴の閾値を使用することを検証します。
         /// </summary>
         [Fact]
         [Trait("Category", "Unit")]
         public void BenchmarkRegressionWorkflow_DetectsPerformanceDegradation()
         {
             var workflow = File.ReadAllText(GetRepositoryFilePath(".github", "workflows", "benchmark-regression.yml"));
+            var policy = File.ReadAllText(GetRepositoryFilePath("benchmark-regression-policy.json"));
 
             Assert.Contains("name: Performance Regression Test", workflow, StringComparison.Ordinal);
             Assert.Contains("pull_request:", workflow, StringComparison.Ordinal);
             Assert.Contains(
                 "benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1",
                 workflow,
                 StringComparison.Ordinal);
-            Assert.Contains("alert-threshold: '200%'", workflow, StringComparison.Ordinal);
-            Assert.Contains("fail-on-alert:", workflow, StringComparison.Ordinal);
             Assert.Contains("FolderDiffIL4DotNet.Benchmarks", workflow, StringComparison.Ordinal);
             Assert.Contains("--exporters json", workflow, StringComparison.Ordinal);
             Assert.Contains("combined-report.json", workflow, StringComparison.Ordinal);
+            Assert.Contains("scripts/check_benchmark_regressions.py", workflow, StringComparison.Ordinal);
+            Assert.Contains("--policy benchmark-regression-policy.json", workflow, StringComparison.Ordinal);
+            Assert.Contains("--baseline-ancestor \"${{ steps.benchmark-base.outputs.sha }}\"", workflow, StringComparison.Ordinal);
+            Assert.Contains("--summary \"$GITHUB_STEP_SUMMARY\"", workflow, StringComparison.Ordinal);
+            Assert.Contains("git show refs/remotes/origin/gh-benchmarks:dev/bench/data.js", workflow, StringComparison.Ordinal);
+            Assert.Contains("publish_baseline:", workflow, StringComparison.Ordinal);
+            Assert.Contains("github.ref == 'refs/heads/main'", workflow, StringComparison.Ordinal);
+            Assert.Contains("github.event.pull_request.base.sha || github.event.before || github.sha", workflow, StringComparison.Ordinal);
+            Assert.DoesNotContain("alert-threshold: '200%'", workflow, StringComparison.Ordinal);
+
+            Assert.Contains("\"sample_count\": 7", policy, StringComparison.Ordinal);
+            Assert.Contains("\"statistic\": \"median of compatible runner-level means\"", policy, StringComparison.Ordinal);
+            Assert.Contains("\"warning_percent\": 10", policy, StringComparison.Ordinal);
+            Assert.Contains("\"failure_percent\": 20", policy, StringComparison.Ordinal);
+            Assert.Contains("\"warning_percent\": 50", policy, StringComparison.Ordinal);
+            Assert.Contains("\"failure_percent\": 100", policy, StringComparison.Ordinal);
         }
 
         /// <summary>
@@ -782,13 +797,16 @@ public void Workflows_RestrictWritePermissionsToTrustedEvents()
                 StringComparison.Ordinal);
             Assert.True(benchmarkPublishStart >= 0);
             var benchmarkReadOnlySection = benchmarkWorkflow[..benchmarkPublishStart];
+            var benchmarkPublishSection = benchmarkWorkflow[benchmarkPublishStart..];
             Assert.Contains(
-                "if: github.event_name == 'push' && github.ref == 'refs/heads/main'",
-                benchmarkWorkflow,
+                "(github.event_name == 'push' && github.ref == 'refs/heads/main')",
+                benchmarkPublishSection,
                 StringComparison.Ordinal);
-            Assert.Contains("auto-push: false", benchmarkReadOnlySection, StringComparison.Ordinal);
-            Assert.Contains("comment-on-alert: false", benchmarkReadOnlySection, StringComparison.Ordinal);
-            Assert.Contains("fail-on-alert: true", benchmarkReadOnlySection, StringComparison.Ordinal);
+            Assert.Contains("github.event_name == 'workflow_dispatch'", benchmarkPublishSection, StringComparison.Ordinal);
+            Assert.Contains("inputs.publish_baseline", benchmarkPublishSection, StringComparison.Ordinal);
+            Assert.DoesNotContain("contents: write", benchmarkReadOnlySection, StringComparison.Ordinal);
+            Assert.DoesNotContain("auto-push: true", benchmarkReadOnlySection, StringComparison.Ordinal);
+            Assert.Contains("scripts/check_benchmark_regressions.py", benchmarkReadOnlySection, StringComparison.Ordinal);
 
             var codeqlAnalyzeStart = codeqlWorkflow.IndexOf("  analyze:", StringComparison.Ordinal);
             var codeqlUploadStart = codeqlWorkflow.IndexOf("  upload-results:", StringComparison.Ordinal);

diff --git a/benchmark-regression-policy.json b/benchmark-regression-policy.json
@@ -0,0 +1,84 @@
+{
+  "schema_version": 1,
+  "benchmark_suite": "FolderDiffIL4DotNet Performance",
+  "baseline_revision": 0,
+  "legacy_baseline_revision": 0,
+  "minimum_compatible_samples": 5,
+  "history_window": 10,
+  "benchmark_definition": {
+    "roots": [
+      "FolderDiffIL4DotNet.Benchmarks"
+    ],
+    "files": [
+      "global.json"
+    ]
+  },
+  "evidence": {
+    "source": "gh-benchmarks/dev/bench/data.js",
+    "runner": "ubuntu-latest",
+    "statistic": "median of compatible runner-level means",
+    "sample_count": 7,
+    "first_sample_utc": "2026-07-25T08:40:12.287Z",
+    "last_sample_utc": "2026-07-25T16:48:33.440Z",
+    "compatible_commits": [
+      "c2796a65b9ac24db38b9036020774c4c764b3f0f",
+      "da2bd1f81ee6027112e1cff6989b905e5279480b",
+      "22fec07ddb9dbf5aa7d92a665cb18726b61c5e0c",
+      "aa24e789cafdeacdd9a5bd195a50107cfda55db3",
+      "8163b0b91ff3f62f44a5a0fc892080ef13f5a001",
+      "1f4987e10e43436683664f7fcccf69ab9c26e9ec",
+      "4b32a77390d82468d23363ec8c4121596d64094a"
+    ],
+    "threshold_rule": "Warning is the observed maximum slowdown rounded up to 5 percentage points plus a 5-point guard band; failure is twice the warning threshold."
+  },
+  "groups": [
+    {
+      "name": "folder-io",
+      "patterns": [
+        "FolderDiffIL4DotNet.Benchmarks.FolderDiffBenchmarks.*"
+      ],
+      "observed_max_slowdown_percent": 4.7,
+      "warning_percent": 10,
+      "failure_percent": 20
+    },
+    {
+      "name": "short-path-sanitizers",
+      "patterns": [
+        "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.Sanitize_ShortPath",
+        "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.Sanitize_UnicodePath"
+      ],
+      "observed_max_slowdown_percent": 13.4,
+      "warning_percent": 20,
+      "failure_percent": 40
+    },
+    {
+      "name": "long-path-sanitizer",
+      "patterns": [
+        "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.Sanitize_LongPath"
+      ],
+      "observed_max_slowdown_percent": 30.3,
+      "warning_percent": 40,
+      "failure_percent": 80
+    },
+    {
+      "name": "standard-text-diff",
+      "patterns": [
+        "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.TextDiffer_*",
+        "FolderDiffIL4DotNet.Benchmarks.TextDifferBenchmarks.SmallFile_5Changes",
+        "FolderDiffIL4DotNet.Benchmarks.TextDifferBenchmarks.MediumFile_20Changes"
+      ],
+      "observed_max_slowdown_percent": 14.0,
+      "warning_percent": 20,
+      "failure_percent": 40
+    },
+    {
+      "name": "million-line-text-diff",
+      "patterns": [
+        "FolderDiffIL4DotNet.Benchmarks.TextDifferBenchmarks.LargeFile_10Changes"
+      ],
+      "observed_max_slowdown_percent": 42.7,
+      "warning_percent": 50,
+      "failure_percent": 100
+    }
+  ]
+}