Add BC PR-Review engine arm with BCQuality ref override

.github/workflows/copilot-evaluation.yml

@@ -69,6 +69,12 @@ concurrency:
 
 env:
   EVALUATION_RESULTS_DIR: evaluation_results
+  # Code-review only: set on a branch to run the review through the BC PR-Review engine
+  # arm against this BCQuality ref (branch/tag/SHA). Empty (default) keeps the current
+  # Copilot code-review. The engine reads BCQUALITY_REF directly.
+  BCQUALITY_REF: ""
+  # Engine (microsoft/BC-ALAgents) ref to check out when the engine arm runs.
+  ENGINE_REF: v1.0.5
 
 jobs:
   get-entries:
@@ -129,11 +135,21 @@ jobs:
       - name: Install evaluation CLIs
         uses: ./.github/actions/install-eval-clis
 
+      - name: Checkout PR-Review engine
+        if: ${{ inputs.category == 'code-review' && env.BCQUALITY_REF != '' }}
+        uses: actions/checkout@v5
+        with:
+          repository: microsoft/BC-ALAgents
+          ref: ${{ env.ENGINE_REF }}
+          path: engine
+          token: ${{ secrets.ENGINE_REPO_TOKEN || github.token }}
+
       - name: Run GitHub Copilot CLI for entry ${{ matrix.entry }}
         timeout-minutes: 120
         shell: pwsh
         env:
           COPILOT_GITHUB_TOKEN: ${{ github.token }}
+          GH_TOKEN: ${{ github.token }}
         run: |
           Write-Output "::add-mask::$env:COPILOT_GITHUB_TOKEN"
 
@@ -143,7 +159,8 @@ jobs:
             --repo-path "${{ steps.setup-env.outputs.repo_path }}" `
             --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" `
             ${{ inputs.al-mcp && '--al-mcp' || '' }} `
-            ${{ inputs.al-lsp && '--al-lsp' || '' }}
+            ${{ inputs.al-lsp && '--al-lsp' || '' }} `
+            ${{ (inputs.category == 'code-review' && env.BCQUALITY_REF != '') && '--engine-scripts-dir engine/agent/scripts' || '' }}
 
       - name: Upload evaluation results
         uses: actions/upload-artifact@v6

src/bcbench/commands/evaluate.py

@@ -22,6 +22,7 @@
 from bcbench.config import get_config
 from bcbench.dataset import BaseDatasetEntry, NL2ALEntry
 from bcbench.evaluate import EvaluationPipeline
+from bcbench.evaluate.codereview_engine import run_engine_review
 from bcbench.evaluate.codereview_judge_calibration import run_calibration
 from bcbench.logger import get_logger
 from bcbench.results import BaseEvaluationResult, CodeReviewResult, ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult
@@ -54,6 +55,13 @@ def evaluate_copilot(
     run_id: RunId = "copilot_test_run",
     al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False,
     al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False,
+    engine_scripts_dir: Annotated[
+        Path | None,
+        typer.Option(
+            envvar="ENGINE_SCRIPTS_DIR",
+            help="Code-review only: route through the BC PR-Review engine at this agent/scripts dir instead of Copilot. Set BCQUALITY_REF in env to pin a BCQuality ref.",
+        ),
+    ] = None,
 ) -> None:
     """
     Evaluate GitHub Copilot CLI on single dataset entry.
@@ -63,34 +71,46 @@ def evaluate_copilot(
     entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0]
     run_dir = _prepare_run_dir(output_dir, run_id)
 
-    logger.info(f"Running evaluation on entry {entry_id} with GitHub Copilot CLI")
-
     container = ContainerConfig(name=container_name, username=username, password=password) if container_name else None
+    agent_name = "BC PR-Review Engine" if engine_scripts_dir else "GitHub Copilot"
+
+    logger.info(f"Running evaluation on entry {entry_id} with {agent_name}")
 
     context = EvaluationContext(
         entry=entry,
         repo_path=repo_path,
         result_dir=run_dir,
         container=container,
         model=model,
-        agent_name="GitHub Copilot",
+        agent_name=agent_name,
         category=category,
     )
 
-    pipeline = category.pipeline
-    pipeline.execute(
-        context,
-        lambda ctx: run_copilot_agent(
-            entry=ctx.entry,
-            repo_path=ctx.repo_path,
-            category=category,
-            model=ctx.model,
-            output_dir=ctx.result_dir,
-            al_mcp=al_mcp if ctx.container else False,
-            al_lsp=al_lsp,
-            container_name=ctx.get_container().name if ctx.container else "",
-        ),
-    )
+    if engine_scripts_dir:
+        context.category.pipeline.execute(
+            context,
+            lambda ctx: run_engine_review(
+                entry=ctx.entry,
+                model=ctx.model,
+                repo_path=ctx.repo_path,
+                output_dir=ctx.result_dir,
+                engine_scripts_dir=engine_scripts_dir,
+            ),
+        )
+    else:
+        category.pipeline.execute(
+            context,
+            lambda ctx: run_copilot_agent(
+                entry=ctx.entry,
+                repo_path=ctx.repo_path,
+                category=category,
+                model=ctx.model,
+                output_dir=ctx.result_dir,
+                al_mcp=al_mcp if ctx.container else False,
+                al_lsp=al_lsp,
+                container_name=ctx.get_container().name if ctx.container else "",
+            ),
+        )
 
     logger.info("Evaluation complete!")
     logger.info(f"Results saved to: {run_dir}")

src/bcbench/evaluate/codereview_engine.py

@@ -0,0 +1,152 @@
+"""BC PR-Review engine runner for the code-review category (faithful convergence arm).
+
+Instead of BC-Bench's own review pipeline, this runner invokes the engine's
+self-contained local-review entry point (``Invoke-LocalReview.ps1`` from
+``microsoft/BC-ALAgents``), which fetches+filters BCQuality and runs the
+exact production reviewer against the patched worktree. BC-Bench then normalizes
+the engine's ``al-code-review-findings.json`` into the ``review.json`` shape its
+scorer already understands.
+"""
+
+import json
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+from bcbench.dataset import BaseDatasetEntry
+from bcbench.exceptions import AgentError, AgentTimeoutError
+from bcbench.logger import get_logger
+from bcbench.types import AgentMetrics, ExperimentConfiguration
+
+logger = get_logger(__name__)
+
+LOCAL_REVIEW_SCRIPT_NAME = "Invoke-LocalReview.ps1"
+FINDINGS_FILE_NAME = "al-code-review-findings.json"
+REVIEW_OUTPUT_FILE = "review.json"
+_ENGINE_TIMEOUT_SECONDS = 1800
+
+
+def _pwsh() -> str:
+    pwsh = shutil.which("pwsh")
+    if not pwsh:
+        raise AgentError("pwsh (PowerShell 7+) is required to run the PR-review engine but was not found on PATH")
+    return pwsh
+
+
+def _git(repo_path: Path, *args: str) -> str:
+    result = subprocess.run(["git", *args], cwd=repo_path, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        raise AgentError(f"git {' '.join(args)} failed: {result.stderr.strip()}")
+    return result.stdout.strip()
+
+
+def _commit_patched_worktree(repo_path: Path) -> str:
+    base_ref = _git(repo_path, "rev-parse", "HEAD")
+    _git(repo_path, "add", "-A")
+    _git(
+        repo_path,
+        "-c",
+        "user.name=bcbench",
+        "-c",
+        "user.email=bcbench@local",
+        "-c",
+        "commit.gpgsign=false",
+        "commit",
+        "--allow-empty",
+        "-m",
+        "bcbench: apply dataset patch for local review",
+    )
+    return base_ref
+
+
+def _run_local_review(
+    local_review_script: Path,
+    repo_path: Path,
+    engine_output_dir: Path,
+    base_ref: str,
+    model: str,
+) -> None:
+    # Invoke-LocalReview.ps1 self-contains fetch+filter+run and reads GH_TOKEN and the optional
+    # BCQUALITY_REF override from the inherited env (via the engine's Get-BCQualityConfig).
+    args = [
+        _pwsh(),
+        "-NoProfile",
+        "-File",
+        str(local_review_script),
+        "-Workspace",
+        str(repo_path),
+        "-BaseRef",
+        base_ref,
+        "-OutputDir",
+        str(engine_output_dir),
+        "-Model",
+        model,
+    ]
+    logger.info(f"Invoking PR-review engine: {local_review_script}")
+    try:
+        result = subprocess.run(args, capture_output=True, text=True, timeout=_ENGINE_TIMEOUT_SECONDS, check=False)
+    except subprocess.TimeoutExpired as exc:
+        raise AgentTimeoutError(f"PR-review engine timed out after {_ENGINE_TIMEOUT_SECONDS}s") from exc
+    if result.stdout:
+        logger.info(result.stdout)
+    if result.returncode != 0:
+        raise AgentError(f"PR-review engine failed (exit {result.returncode}): {result.stderr.strip() or result.stdout.strip()}")
+
+
+def _findings_to_review_comments(payload: dict) -> list[dict]:
+    comments: list[dict] = []
+    for finding in payload.get("findings") or []:
+        file_path = finding.get("filePath")
+        line = finding.get("lineNumber")
+        if not file_path or not line:
+            continue
+        issue = (finding.get("issue") or "").strip()
+        recommendation = (finding.get("recommendation") or "").strip()
+        body = (f"{issue}\n\nRecommendation: {recommendation}" if issue else recommendation) if recommendation else issue
+        comment: dict = {"file": file_path, "line_start": line, "body": body}
+        severity = finding.get("severity")
+        if severity:
+            comment["severity"] = str(severity).lower()
+        comments.append(comment)
+    return comments
+
+
+def _write_review_json(review_output_dir: Path, repo_path: Path) -> int:
+    findings_file = review_output_dir / FINDINGS_FILE_NAME
+    if not findings_file.exists():
+        raise AgentError(f"Engine did not produce {FINDINGS_FILE_NAME} at {review_output_dir}")
+    payload = json.loads(findings_file.read_text(encoding="utf-8"))
+    comments = _findings_to_review_comments(payload)
+    (repo_path / REVIEW_OUTPUT_FILE).write_text(json.dumps(comments, indent=2), encoding="utf-8")
+    logger.info(f"Wrote {len(comments)} review comment(s) to {repo_path / REVIEW_OUTPUT_FILE}")
+    return len(comments)
+
+
+def run_engine_review(
+    entry: BaseDatasetEntry,
+    model: str,
+    repo_path: Path,
+    output_dir: Path,
+    engine_scripts_dir: Path,
+) -> tuple[AgentMetrics | None, ExperimentConfiguration]:
+    local_review_script = engine_scripts_dir / LOCAL_REVIEW_SCRIPT_NAME
+    if not local_review_script.exists():
+        raise AgentError(f"Engine script not found: {local_review_script}")
+
+    logger.info(f"Running PR-review engine on: {entry.instance_id}")
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    engine_output_dir = output_dir / "engine-output"
+
+    base_ref = _commit_patched_worktree(repo_path)
+
+    started_at = time.monotonic()
+    _run_local_review(local_review_script, repo_path, engine_output_dir, base_ref, model)
+    execution_time = time.monotonic() - started_at
+
+    _write_review_json(engine_output_dir / "review-output", repo_path)
+
+    metrics = AgentMetrics(execution_time=execution_time)
+    experiment = ExperimentConfiguration(custom_agent="bc-pr-review-engine")
+    return metrics, experiment