Add a pure RegressionReport service

docs/results.md

@@ -4,6 +4,20 @@ date = 2024-12-10T07:07:07+01:00
 description = 'Fetch results from the KernelCI ecosystem.'
 +++
 
+## Regression comparison and CI gates
+
+```shell
+kci-dev results compare --giturl URL --branch BRANCH --format json BASE HEAD
+kci-dev results gate --giturl URL --branch BRANCH --base BASE --head HEAD \
+  --fail-on regression --format json
+```
+
+Reports classify executions as `regression`, `fixed`, `unstable`,
+`persistent_fail`, `new`, or `missing`, while preserving duplicates. Identity
+includes origin, platform, architecture, compiler, configuration, and path.
+Exit status is 0 without a policy violation, 1 for a policy violation, and 2
+when API or infrastructure failures make the result incomplete.
+
 `kci-dev` pulls from our Dashboard API. As of now, it is an EXPERIMENTAL tooling under development with close collaboration from Linux kernel maintainers.
 
 > KNOWN ISSUE: The Dashboard endpoint we are using returns a file of a few megabytes in size, so download may take

kcidev/api.py

@@ -55,6 +55,7 @@
     send_jobretry,
     send_patchset,
 )
+from kcidev.libs.regression import RegressionReport
 from kcidev.main import get_cli
 
 
@@ -500,6 +501,56 @@ def get_tree_report(
             min_age_in_hours,
         )
 
+    def compare_results(
+        self, base, head, giturl, branch, origin="maestro", include_issues=True
+    ):
+        """Compare two dashboard checkouts and return a CI-grade report dict."""
+
+        def checkout(commit):
+            return {
+                "builds": self.get_builds(origin, giturl, branch, commit).get(
+                    "builds", []
+                ),
+                "boots": self.get_boots(origin, giturl, branch, commit).get(
+                    "boots", []
+                ),
+                "tests": self.get_tests(origin, giturl, branch, commit).get(
+                    "tests", []
+                ),
+            }
+
+        base_results, head_results = checkout(base), checkout(head)
+        # tree-report supplies history-aware unstable/regression decisions.  If
+        # HEAD is not the newest checkout the raw transition remains useful.
+        history = self.get_tree_report(origin, branch, giturl)
+        report = RegressionReport.compare(
+            base, head, base_results, head_results, history
+        )
+        if include_issues:
+            for item in report.items:
+                if item["category"] not in ("regression", "persistent_fail"):
+                    continue
+                result_id = item["head_id"]
+                if not result_id:
+                    continue
+                try:
+                    issues = (
+                        self.get_build_issues(result_id)
+                        if item["identity"]["kind"] == "build"
+                        else self.get_boot_issues(result_id)
+                    )
+                except KciDevError as exc:
+                    if "no issues" in str(exc).lower():
+                        issues = []
+                    else:
+                        report.incomplete = True
+                        continue
+                item["known_issues"] = [
+                    issue.get("id", issue) if isinstance(issue, dict) else issue
+                    for issue in issues
+                ]
+        return report.to_dict()
+
     def _instance_setting(self, key, *, human_readable_key=None):
         value = ((self.cfg or {}).get(self.instance) or {}).get(key)
         if not value:

kcidev/libs/regression.py

@@ -0,0 +1,152 @@
+"""Pure regression classification and reporting.
+
+This module deliberately has no Click or HTTP dependencies.  It can therefore
+be used by the command line, the public Python client, and the MCP server.
+"""
+
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+
+CATEGORIES = ("regression", "fixed", "unstable", "persistent_fail", "new", "missing")
+FAIL_STATUSES = {"FAIL", "ERROR"}
+
+
+def _value(item, *names, default="unknown"):
+    for name in names:
+        value = item.get(name)
+        if value not in (None, ""):
+            return value
+    return default
+
+
+def result_identity(item, kind="test"):
+    """Return the complete, stable identity of a result (never its result id)."""
+    environment = item.get("environment_misc") or {}
+    return {
+        "kind": kind,
+        "origin": _value(item, "origin"),
+        "platform": _value(
+            item, "platform", "hardware", default=environment.get("platform", "unknown")
+        ),
+        "architecture": _value(item, "architecture", "arch"),
+        "compiler": _value(item, "compiler"),
+        "config": _value(item, "config", "config_name"),
+        "path": _value(
+            item, "path", "test_path", default="build" if kind == "build" else "unknown"
+        ),
+    }
+
+
+def _key(item, kind):
+    return tuple(result_identity(item, kind).values())
+
+
+def _tree_report_keys(tree_report, category):
+    keys = set()
+    for platform, configs in (tree_report or {}).get(category, {}).items():
+        for config, arch_compilers in configs.items():
+            for arch_compiler, paths in arch_compilers.items():
+                architecture, _, compiler = arch_compiler.partition("/")
+                for path, tests in paths.items():
+                    for test in tests or [{}]:
+                        identity = result_identity(
+                            {
+                                **test,
+                                "platform": platform,
+                                "config": config,
+                                "architecture": architecture,
+                                "compiler": compiler,
+                                "path": path,
+                            },
+                            "test",
+                        )
+                        keys.add(tuple(identity.values()))
+    return keys
+
+
+@dataclass
+class RegressionReport:
+    """A deterministic comparison report which preserves duplicate results."""
+
+    base: str
+    head: str
+    items: list = field(default_factory=list)
+    incomplete: bool = False
+
+    @classmethod
+    def compare(cls, base, head, base_results, head_results, tree_report=None):
+        report = cls(base=base, head=head)
+        history = {
+            "regression": _tree_report_keys(tree_report, "possible_regressions"),
+            "fixed": _tree_report_keys(tree_report, "fixed_regressions"),
+            "unstable": _tree_report_keys(tree_report, "unstable_tests"),
+        }
+        for kind in ("build", "boot", "test"):
+            old = defaultdict(list)
+            new = defaultdict(list)
+            for item in base_results.get(kind + "s", []):
+                old[_key(item, kind)].append(item)
+            for item in head_results.get(kind + "s", []):
+                new[_key(item, kind)].append(item)
+            for key in sorted(set(old) | set(new)):
+                count = max(len(old[key]), len(new[key]))
+                for occurrence in range(count):
+                    before = (
+                        old[key][occurrence] if occurrence < len(old[key]) else None
+                    )
+                    after = new[key][occurrence] if occurrence < len(new[key]) else None
+                    category = cls._classify(before, after, key, history)
+                    if category is None:
+                        continue
+                    chosen = after or before
+                    report.items.append(
+                        {
+                            "category": category,
+                            "identity": result_identity(chosen, kind),
+                            "occurrence": occurrence,
+                            "base_status": before.get("status") if before else None,
+                            "head_status": after.get("status") if after else None,
+                            "base_id": before.get("id") if before else None,
+                            "head_id": after.get("id") if after else None,
+                            "known_issues": [],
+                        }
+                    )
+        return report
+
+    @staticmethod
+    def _classify(before, after, key, history):
+        if before is None:
+            return "new"
+        if after is None:
+            return "missing"
+        old = str(before.get("status") or "UNKNOWN").upper()
+        new = str(after.get("status") or "UNKNOWN").upper()
+        if key in history["unstable"]:
+            return "unstable"
+        if key in history["regression"] or (old == "PASS" and new in FAIL_STATUSES):
+            return "regression"
+        if key in history["fixed"] or (old in FAIL_STATUSES and new == "PASS"):
+            return "fixed"
+        if old in FAIL_STATUSES and new in FAIL_STATUSES:
+            return "persistent_fail"
+        if old != new:
+            return "unstable"
+        return None
+
+    @property
+    def counts(self):
+        counts = Counter(item["category"] for item in self.items)
+        return {category: counts[category] for category in CATEGORIES}
+
+    def has_violation(self, fail_on="regression"):
+        policies = {part.strip() for part in fail_on.split(",") if part.strip()}
+        return any(self.counts.get(policy, 0) for policy in policies)
+
+    def to_dict(self):
+        return {
+            "base": self.base,
+            "head": self.head,
+            "counts": self.counts,
+            "incomplete": self.incomplete,
+            "items": self.items,
+        }

kcidev/mcp/tools_dashboard.py

@@ -78,6 +78,23 @@ def get_summary(
     }
 
 
+@tool_errors
+def compare_checkouts(
+    giturl: str,
+    branch: str,
+    base: str,
+    head: str,
+    origin: str = "maestro",
+):
+    """Compare two checkouts, classifying regressions, fixes and unstable tests.
+
+    The returned report preserves duplicate executions and includes known issue
+    ids for failing/regressing results.  ``incomplete`` means it must not be
+    treated as a successful CI gate.
+    """
+    return _current_client().compare_results(base, head, giturl, branch, origin)
+
+
 @tool_errors
 def list_commits(giturl: str, branch: str, commit: str, origin: str = "maestro"):
     """List recent checkouts of a tree with per-commit result counts.
@@ -294,6 +311,7 @@ def get_issue_tests(
 READ_ONLY_TOOLS = (
     list_trees,
     get_summary,
+    compare_checkouts,
     list_commits,
     list_builds,
     list_boots,

kcidev/subcommands/results/__init__.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+import json
 import sys
 from functools import wraps
 
@@ -384,8 +385,11 @@ def boot(op_id, download_logs, use_json):
     default=True,
 )
 @click.argument("commits", nargs=-1, required=False)
-@results_display_options
-def compare(origin, giturl, branch, latest, commits, use_json):
+@click.option(
+    "--format", "output_format", type=click.Choice(["human", "json"]), default="human"
+)
+@click.option("--json", "use_json", is_flag=True, hidden=True)
+def compare(origin, giturl, branch, latest, commits, output_format, use_json):
     """Compare test results between commits with summary and regressions.
 
     Compares test results between commits showing summary statistics
@@ -404,15 +408,84 @@ def compare(origin, giturl, branch, latest, commits, use_json):
       # Compare specific commits
       kci-dev results compare --giturl https://git.kernel.org/... abc123 def456
     """
-    if latest and not commits:
-        # Use latest commits from history
-        cmd_compare(origin, giturl, branch, None, use_json)
-    elif len(commits) == 2:
-        # Use specific commits provided
-        cmd_compare(origin, giturl, branch, list(commits), use_json)
+    from kcidev.api import KciDevError, KernelCIClient
+
+    json_output = use_json or output_format == "json"
+    if len(commits) != 2:
+        raise click.UsageError("exactly BASE and HEAD commits are required")
+    try:
+        report = KernelCIClient().compare_results(
+            commits[0], commits[1], giturl, branch, origin
+        )
+    except KciDevError as exc:
+        if json_output:
+            click.echo(json.dumps({"error": str(exc), "incomplete": True}))
+        raise click.exceptions.Exit(2) from exc
+    if json_output:
+        # This is deliberately the only stdout write in JSON mode.
+        click.echo(json.dumps(report, sort_keys=True))
+
     else:
-        click.echo("Error: Provide either --latest flag or exactly 2 commit hashes")
-        raise click.Abort()
+        click.echo(f"Compared {commits[0]} -> {commits[1]}")
+        for category, count in report["counts"].items():
+            click.echo(f"  {category}: {count}")
+    if report["incomplete"]:
+        raise click.exceptions.Exit(2)
+    if report["counts"]["regression"]:
+        raise click.exceptions.Exit(1)
+
+
+@results.command()
+@click.option("--origin", default="maestro", help="Select KCIDB origin")
+@click.option("--giturl", required=True, help="Git repository URL")
+@click.option("--branch", required=True, help="Git branch name")
+@click.option("--base", help="Baseline checkout commit (defaults to previous)")
+@click.option("--head", help="Candidate checkout commit (defaults to latest)")
+@click.option(
+    "--fail-on",
+    default="regression",
+    show_default=True,
+    help="Comma-separated report categories which fail the gate",
+)
+@click.option(
+    "--format", "output_format", type=click.Choice(["human", "json"]), default="human"
+)
+def gate(origin, giturl, branch, base, head, fail_on, output_format):
+    """Gate a checkout using the regression comparison policy."""
+    from kcidev.api import KciDevError, KernelCIClient
+
+    try:
+        client = KernelCIClient()
+        if bool(base) != bool(head):
+            raise click.UsageError("provide both --base and --head, or neither")
+        if not base:
+            giturl, branch, latest = set_giturl_branch_commit(
+                origin, giturl, branch, None, True, None
+            )
+            history = client.get_commits_history(origin, giturl, branch, latest)
+            commits = (
+                history if isinstance(history, list) else history.get("commits", [])
+            )
+            if len(commits) < 2:
+                raise KciDevError("fewer than two checkouts are available")
+            head, base = commits[0]["git_commit_hash"], commits[1]["git_commit_hash"]
+        report = client.compare_results(base, head, giturl, branch, origin)
+    except KciDevError as exc:
+        if output_format == "json":
+            click.echo(json.dumps({"error": str(exc), "incomplete": True}))
+        else:
+            click.echo(f"Incomplete comparison: {exc}", err=True)
+        raise click.exceptions.Exit(2) from exc
+    click.echo(
+        json.dumps(report, sort_keys=True)
+        if output_format == "json"
+        else "\n".join(f"{key}: {value}" for key, value in report["counts"].items())
+    )
+    if report["incomplete"]:
+        raise click.exceptions.Exit(2)
+    policies = {value.strip() for value in fail_on.split(",")}
+    if any(report["counts"].get(value, 0) for value in policies):
+        raise click.exceptions.Exit(1)
 
 
 def get_issues(ctx, origin, item_type, giturl, branch, commit, tree_name, arch):