diff --git a/docs/results.md b/docs/results.md index 77a4886..aabe312 100644 --- a/docs/results.md +++ b/docs/results.md @@ -4,6 +4,20 @@ date = 2024-12-10T07:07:07+01:00 description = 'Fetch results from the KernelCI ecosystem.' +++ +## Regression comparison and CI gates + +```shell +kci-dev results compare --giturl URL --branch BRANCH --format json BASE HEAD +kci-dev results gate --giturl URL --branch BRANCH --base BASE --head HEAD \ + --fail-on regression --format json +``` + +Reports classify executions as `regression`, `fixed`, `unstable`, +`persistent_fail`, `new`, or `missing`, while preserving duplicates. Identity +includes origin, platform, architecture, compiler, configuration, and path. +Exit status is 0 without a policy violation, 1 for a policy violation, and 2 +when API or infrastructure failures make the result incomplete. + `kci-dev` pulls from our Dashboard API. As of now, it is an EXPERIMENTAL tooling under development with close collaboration from Linux kernel maintainers. > KNOWN ISSUE: The Dashboard endpoint we are using returns a file of a few megabytes in size, so download may take diff --git a/kcidev/api.py b/kcidev/api.py index 0067b23..226f279 100644 --- a/kcidev/api.py +++ b/kcidev/api.py @@ -55,6 +55,7 @@ send_jobretry, send_patchset, ) +from kcidev.libs.regression import RegressionReport from kcidev.main import get_cli @@ -500,6 +501,56 @@ def get_tree_report( min_age_in_hours, ) + def compare_results( + self, base, head, giturl, branch, origin="maestro", include_issues=True + ): + """Compare two dashboard checkouts and return a CI-grade report dict.""" + + def checkout(commit): + return { + "builds": self.get_builds(origin, giturl, branch, commit).get( + "builds", [] + ), + "boots": self.get_boots(origin, giturl, branch, commit).get( + "boots", [] + ), + "tests": self.get_tests(origin, giturl, branch, commit).get( + "tests", [] + ), + } + + base_results, head_results = checkout(base), checkout(head) + # tree-report supplies history-aware unstable/regression decisions. If + # HEAD is not the newest checkout the raw transition remains useful. + history = self.get_tree_report(origin, branch, giturl) + report = RegressionReport.compare( + base, head, base_results, head_results, history + ) + if include_issues: + for item in report.items: + if item["category"] not in ("regression", "persistent_fail"): + continue + result_id = item["head_id"] + if not result_id: + continue + try: + issues = ( + self.get_build_issues(result_id) + if item["identity"]["kind"] == "build" + else self.get_boot_issues(result_id) + ) + except KciDevError as exc: + if "no issues" in str(exc).lower(): + issues = [] + else: + report.incomplete = True + continue + item["known_issues"] = [ + issue.get("id", issue) if isinstance(issue, dict) else issue + for issue in issues + ] + return report.to_dict() + def _instance_setting(self, key, *, human_readable_key=None): value = ((self.cfg or {}).get(self.instance) or {}).get(key) if not value: diff --git a/kcidev/libs/regression.py b/kcidev/libs/regression.py new file mode 100644 index 0000000..3a903f3 --- /dev/null +++ b/kcidev/libs/regression.py @@ -0,0 +1,152 @@ +"""Pure regression classification and reporting. + +This module deliberately has no Click or HTTP dependencies. It can therefore +be used by the command line, the public Python client, and the MCP server. +""" + +from collections import Counter, defaultdict +from dataclasses import dataclass, field + +CATEGORIES = ("regression", "fixed", "unstable", "persistent_fail", "new", "missing") +FAIL_STATUSES = {"FAIL", "ERROR"} + + +def _value(item, *names, default="unknown"): + for name in names: + value = item.get(name) + if value not in (None, ""): + return value + return default + + +def result_identity(item, kind="test"): + """Return the complete, stable identity of a result (never its result id).""" + environment = item.get("environment_misc") or {} + return { + "kind": kind, + "origin": _value(item, "origin"), + "platform": _value( + item, "platform", "hardware", default=environment.get("platform", "unknown") + ), + "architecture": _value(item, "architecture", "arch"), + "compiler": _value(item, "compiler"), + "config": _value(item, "config", "config_name"), + "path": _value( + item, "path", "test_path", default="build" if kind == "build" else "unknown" + ), + } + + +def _key(item, kind): + return tuple(result_identity(item, kind).values()) + + +def _tree_report_keys(tree_report, category): + keys = set() + for platform, configs in (tree_report or {}).get(category, {}).items(): + for config, arch_compilers in configs.items(): + for arch_compiler, paths in arch_compilers.items(): + architecture, _, compiler = arch_compiler.partition("/") + for path, tests in paths.items(): + for test in tests or [{}]: + identity = result_identity( + { + **test, + "platform": platform, + "config": config, + "architecture": architecture, + "compiler": compiler, + "path": path, + }, + "test", + ) + keys.add(tuple(identity.values())) + return keys + + +@dataclass +class RegressionReport: + """A deterministic comparison report which preserves duplicate results.""" + + base: str + head: str + items: list = field(default_factory=list) + incomplete: bool = False + + @classmethod + def compare(cls, base, head, base_results, head_results, tree_report=None): + report = cls(base=base, head=head) + history = { + "regression": _tree_report_keys(tree_report, "possible_regressions"), + "fixed": _tree_report_keys(tree_report, "fixed_regressions"), + "unstable": _tree_report_keys(tree_report, "unstable_tests"), + } + for kind in ("build", "boot", "test"): + old = defaultdict(list) + new = defaultdict(list) + for item in base_results.get(kind + "s", []): + old[_key(item, kind)].append(item) + for item in head_results.get(kind + "s", []): + new[_key(item, kind)].append(item) + for key in sorted(set(old) | set(new)): + count = max(len(old[key]), len(new[key])) + for occurrence in range(count): + before = ( + old[key][occurrence] if occurrence < len(old[key]) else None + ) + after = new[key][occurrence] if occurrence < len(new[key]) else None + category = cls._classify(before, after, key, history) + if category is None: + continue + chosen = after or before + report.items.append( + { + "category": category, + "identity": result_identity(chosen, kind), + "occurrence": occurrence, + "base_status": before.get("status") if before else None, + "head_status": after.get("status") if after else None, + "base_id": before.get("id") if before else None, + "head_id": after.get("id") if after else None, + "known_issues": [], + } + ) + return report + + @staticmethod + def _classify(before, after, key, history): + if before is None: + return "new" + if after is None: + return "missing" + old = str(before.get("status") or "UNKNOWN").upper() + new = str(after.get("status") or "UNKNOWN").upper() + if key in history["unstable"]: + return "unstable" + if key in history["regression"] or (old == "PASS" and new in FAIL_STATUSES): + return "regression" + if key in history["fixed"] or (old in FAIL_STATUSES and new == "PASS"): + return "fixed" + if old in FAIL_STATUSES and new in FAIL_STATUSES: + return "persistent_fail" + if old != new: + return "unstable" + return None + + @property + def counts(self): + counts = Counter(item["category"] for item in self.items) + return {category: counts[category] for category in CATEGORIES} + + def has_violation(self, fail_on="regression"): + policies = {part.strip() for part in fail_on.split(",") if part.strip()} + return any(self.counts.get(policy, 0) for policy in policies) + + def to_dict(self): + return { + "base": self.base, + "head": self.head, + "counts": self.counts, + "incomplete": self.incomplete, + "items": self.items, + } diff --git a/kcidev/mcp/tools_dashboard.py b/kcidev/mcp/tools_dashboard.py index 6f15c03..c965b06 100644 --- a/kcidev/mcp/tools_dashboard.py +++ b/kcidev/mcp/tools_dashboard.py @@ -78,6 +78,23 @@ def get_summary( } +@tool_errors +def compare_checkouts( + giturl: str, + branch: str, + base: str, + head: str, + origin: str = "maestro", +): + """Compare two checkouts, classifying regressions, fixes and unstable tests. + + The returned report preserves duplicate executions and includes known issue + ids for failing/regressing results. ``incomplete`` means it must not be + treated as a successful CI gate. + """ + return _current_client().compare_results(base, head, giturl, branch, origin) + + @tool_errors def list_commits(giturl: str, branch: str, commit: str, origin: str = "maestro"): """List recent checkouts of a tree with per-commit result counts. @@ -294,6 +311,7 @@ def get_issue_tests( READ_ONLY_TOOLS = ( list_trees, get_summary, + compare_checkouts, list_commits, list_builds, list_boots, diff --git a/kcidev/subcommands/results/__init__.py b/kcidev/subcommands/results/__init__.py index 20965a2..06e0567 100644 --- a/kcidev/subcommands/results/__init__.py +++ b/kcidev/subcommands/results/__init__.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import json import sys from functools import wraps @@ -384,8 +385,11 @@ def boot(op_id, download_logs, use_json): default=True, ) @click.argument("commits", nargs=-1, required=False) -@results_display_options -def compare(origin, giturl, branch, latest, commits, use_json): +@click.option( + "--format", "output_format", type=click.Choice(["human", "json"]), default="human" +) +@click.option("--json", "use_json", is_flag=True, hidden=True) +def compare(origin, giturl, branch, latest, commits, output_format, use_json): """Compare test results between commits with summary and regressions. Compares test results between commits showing summary statistics @@ -404,15 +408,84 @@ def compare(origin, giturl, branch, latest, commits, use_json): # Compare specific commits kci-dev results compare --giturl https://git.kernel.org/... abc123 def456 """ - if latest and not commits: - # Use latest commits from history - cmd_compare(origin, giturl, branch, None, use_json) - elif len(commits) == 2: - # Use specific commits provided - cmd_compare(origin, giturl, branch, list(commits), use_json) + from kcidev.api import KciDevError, KernelCIClient + + json_output = use_json or output_format == "json" + if len(commits) != 2: + raise click.UsageError("exactly BASE and HEAD commits are required") + try: + report = KernelCIClient().compare_results( + commits[0], commits[1], giturl, branch, origin + ) + except KciDevError as exc: + if json_output: + click.echo(json.dumps({"error": str(exc), "incomplete": True})) + raise click.exceptions.Exit(2) from exc + if json_output: + # This is deliberately the only stdout write in JSON mode. + click.echo(json.dumps(report, sort_keys=True)) + else: - click.echo("Error: Provide either --latest flag or exactly 2 commit hashes") - raise click.Abort() + click.echo(f"Compared {commits[0]} -> {commits[1]}") + for category, count in report["counts"].items(): + click.echo(f" {category}: {count}") + if report["incomplete"]: + raise click.exceptions.Exit(2) + if report["counts"]["regression"]: + raise click.exceptions.Exit(1) + + +@results.command() +@click.option("--origin", default="maestro", help="Select KCIDB origin") +@click.option("--giturl", required=True, help="Git repository URL") +@click.option("--branch", required=True, help="Git branch name") +@click.option("--base", help="Baseline checkout commit (defaults to previous)") +@click.option("--head", help="Candidate checkout commit (defaults to latest)") +@click.option( + "--fail-on", + default="regression", + show_default=True, + help="Comma-separated report categories which fail the gate", +) +@click.option( + "--format", "output_format", type=click.Choice(["human", "json"]), default="human" +) +def gate(origin, giturl, branch, base, head, fail_on, output_format): + """Gate a checkout using the regression comparison policy.""" + from kcidev.api import KciDevError, KernelCIClient + + try: + client = KernelCIClient() + if bool(base) != bool(head): + raise click.UsageError("provide both --base and --head, or neither") + if not base: + giturl, branch, latest = set_giturl_branch_commit( + origin, giturl, branch, None, True, None + ) + history = client.get_commits_history(origin, giturl, branch, latest) + commits = ( + history if isinstance(history, list) else history.get("commits", []) + ) + if len(commits) < 2: + raise KciDevError("fewer than two checkouts are available") + head, base = commits[0]["git_commit_hash"], commits[1]["git_commit_hash"] + report = client.compare_results(base, head, giturl, branch, origin) + except KciDevError as exc: + if output_format == "json": + click.echo(json.dumps({"error": str(exc), "incomplete": True})) + else: + click.echo(f"Incomplete comparison: {exc}", err=True) + raise click.exceptions.Exit(2) from exc + click.echo( + json.dumps(report, sort_keys=True) + if output_format == "json" + else "\n".join(f"{key}: {value}" for key, value in report["counts"].items()) + ) + if report["incomplete"]: + raise click.exceptions.Exit(2) + policies = {value.strip() for value in fail_on.split(",")} + if any(report["counts"].get(value, 0) for value in policies): + raise click.exceptions.Exit(1) def get_issues(ctx, origin, item_type, giturl, branch, commit, tree_name, arch): diff --git a/tests/test_regression.py b/tests/test_regression.py new file mode 100644 index 0000000..df0da3b --- /dev/null +++ b/tests/test_regression.py @@ -0,0 +1,88 @@ +import json + +from click.testing import CliRunner + +from kcidev.libs.regression import RegressionReport +from kcidev.main import get_cli + + +def result(result_id, status, path="suite.case", platform="qemu"): + return { + "id": result_id, + "origin": "maestro", + "status": status, + "path": path, + "environment_misc": {"platform": platform}, + "architecture": "arm64", + "compiler": "gcc-14", + "config": "defconfig", + } + + +def test_report_classifies_transitions_and_preserves_duplicates(): + base = { + "tests": [ + result("r1", "PASS"), + result("r2", "PASS"), + result("f", "FAIL", "fixed"), + result("p", "FAIL", "persistent"), + result("m", "PASS", "missing"), + result("u", "SKIP", "unstable"), + ] + } + head = { + "tests": [ + result("r3", "FAIL"), + result("r4", "FAIL"), + result("f2", "PASS", "fixed"), + result("p2", "FAIL", "persistent"), + result("n", "PASS", "new"), + result("u2", "PASS", "unstable"), + ] + } + report = RegressionReport.compare("base", "head", base, head) + assert report.counts == { + "regression": 2, + "fixed": 1, + "unstable": 1, + "persistent_fail": 1, + "new": 1, + "missing": 1, + } + regressions = [i for i in report.items if i["category"] == "regression"] + assert [i["occurrence"] for i in regressions] == [0, 1] + assert regressions[0]["identity"] == { + "kind": "test", + "origin": "maestro", + "platform": "qemu", + "architecture": "arm64", + "compiler": "gcc-14", + "config": "defconfig", + "path": "suite.case", + } + + +def test_compare_json_is_one_document_and_regressions_exit_one(monkeypatch): + report = RegressionReport("base", "head") + report.items.append({"category": "regression"}) + monkeypatch.setattr( + "kcidev.api.KernelCIClient.compare_results", + lambda *args, **kwargs: report.to_dict(), + ) + result = CliRunner().invoke( + get_cli(), + [ + "results", + "compare", + "--giturl", + "url", + "--branch", + "main", + "--format", + "json", + "base", + "head", + ], + ) + assert result.exit_code == 1 + assert json.loads(result.stdout)["counts"]["regression"] == 1