diff --git a/.github/workflows/copilot-evaluation.yml b/.github/workflows/copilot-evaluation.yml index cc65cece0..3784d0d3a 100644 --- a/.github/workflows/copilot-evaluation.yml +++ b/.github/workflows/copilot-evaluation.yml @@ -69,6 +69,12 @@ concurrency: env: EVALUATION_RESULTS_DIR: evaluation_results + # Code-review only: set on a branch to run the review through the BC PR-Review engine + # arm against this BCQuality ref (branch/tag/SHA). Empty (default) keeps the current + # Copilot code-review. The engine reads BCQUALITY_REF directly. + BCQUALITY_REF: "" + # Engine (microsoft/BC-ALAgents) ref to check out when the engine arm runs. + ENGINE_REF: v1.0.5 jobs: get-entries: @@ -129,11 +135,21 @@ jobs: - name: Install evaluation CLIs uses: ./.github/actions/install-eval-clis + - name: Checkout PR-Review engine + if: ${{ inputs.category == 'code-review' && env.BCQUALITY_REF != '' }} + uses: actions/checkout@v5 + with: + repository: microsoft/BC-ALAgents + ref: ${{ env.ENGINE_REF }} + path: engine + token: ${{ secrets.ENGINE_REPO_TOKEN || github.token }} + - name: Run GitHub Copilot CLI for entry ${{ matrix.entry }} timeout-minutes: 120 shell: pwsh env: COPILOT_GITHUB_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ github.token }} run: | Write-Output "::add-mask::$env:COPILOT_GITHUB_TOKEN" @@ -143,7 +159,8 @@ jobs: --repo-path "${{ steps.setup-env.outputs.repo_path }}" ` --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" ` ${{ inputs.al-mcp && '--al-mcp' || '' }} ` - ${{ inputs.al-lsp && '--al-lsp' || '' }} + ${{ inputs.al-lsp && '--al-lsp' || '' }} ` + ${{ (inputs.category == 'code-review' && env.BCQUALITY_REF != '') && '--engine-scripts-dir engine/agent/scripts' || '' }} - name: Upload evaluation results uses: actions/upload-artifact@v6 diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 608d4005b..aca75006f 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -22,6 +22,7 @@ from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry, NL2ALEntry from bcbench.evaluate import EvaluationPipeline +from bcbench.evaluate.codereview_engine import run_engine_review from bcbench.evaluate.codereview_judge_calibration import run_calibration from bcbench.logger import get_logger from bcbench.results import BaseEvaluationResult, CodeReviewResult, ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult @@ -54,6 +55,13 @@ def evaluate_copilot( run_id: RunId = "copilot_test_run", al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + engine_scripts_dir: Annotated[ + Path | None, + typer.Option( + envvar="ENGINE_SCRIPTS_DIR", + help="Code-review only: route through the BC PR-Review engine at this agent/scripts dir instead of Copilot. Set BCQUALITY_REF in env to pin a BCQuality ref.", + ), + ] = None, ) -> None: """ Evaluate GitHub Copilot CLI on single dataset entry. @@ -63,9 +71,10 @@ def evaluate_copilot( entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] run_dir = _prepare_run_dir(output_dir, run_id) - logger.info(f"Running evaluation on entry {entry_id} with GitHub Copilot CLI") - container = ContainerConfig(name=container_name, username=username, password=password) if container_name else None + agent_name = "BC PR-Review Engine" if engine_scripts_dir else "GitHub Copilot" + + logger.info(f"Running evaluation on entry {entry_id} with {agent_name}") context = EvaluationContext( entry=entry, @@ -73,24 +82,35 @@ def evaluate_copilot( result_dir=run_dir, container=container, model=model, - agent_name="GitHub Copilot", + agent_name=agent_name, category=category, ) - pipeline = category.pipeline - pipeline.execute( - context, - lambda ctx: run_copilot_agent( - entry=ctx.entry, - repo_path=ctx.repo_path, - category=category, - model=ctx.model, - output_dir=ctx.result_dir, - al_mcp=al_mcp if ctx.container else False, - al_lsp=al_lsp, - container_name=ctx.get_container().name if ctx.container else "", - ), - ) + if engine_scripts_dir: + context.category.pipeline.execute( + context, + lambda ctx: run_engine_review( + entry=ctx.entry, + model=ctx.model, + repo_path=ctx.repo_path, + output_dir=ctx.result_dir, + engine_scripts_dir=engine_scripts_dir, + ), + ) + else: + category.pipeline.execute( + context, + lambda ctx: run_copilot_agent( + entry=ctx.entry, + repo_path=ctx.repo_path, + category=category, + model=ctx.model, + output_dir=ctx.result_dir, + al_mcp=al_mcp if ctx.container else False, + al_lsp=al_lsp, + container_name=ctx.get_container().name if ctx.container else "", + ), + ) logger.info("Evaluation complete!") logger.info(f"Results saved to: {run_dir}") diff --git a/src/bcbench/evaluate/codereview_engine.py b/src/bcbench/evaluate/codereview_engine.py new file mode 100644 index 000000000..547c0f47b --- /dev/null +++ b/src/bcbench/evaluate/codereview_engine.py @@ -0,0 +1,152 @@ +"""BC PR-Review engine runner for the code-review category (faithful convergence arm). + +Instead of BC-Bench's own review pipeline, this runner invokes the engine's +self-contained local-review entry point (``Invoke-LocalReview.ps1`` from +``microsoft/BC-ALAgents``), which fetches+filters BCQuality and runs the +exact production reviewer against the patched worktree. BC-Bench then normalizes +the engine's ``al-code-review-findings.json`` into the ``review.json`` shape its +scorer already understands. +""" + +import json +import shutil +import subprocess +import time +from pathlib import Path + +from bcbench.dataset import BaseDatasetEntry +from bcbench.exceptions import AgentError, AgentTimeoutError +from bcbench.logger import get_logger +from bcbench.types import AgentMetrics, ExperimentConfiguration + +logger = get_logger(__name__) + +LOCAL_REVIEW_SCRIPT_NAME = "Invoke-LocalReview.ps1" +FINDINGS_FILE_NAME = "al-code-review-findings.json" +REVIEW_OUTPUT_FILE = "review.json" +_ENGINE_TIMEOUT_SECONDS = 1800 + + +def _pwsh() -> str: + pwsh = shutil.which("pwsh") + if not pwsh: + raise AgentError("pwsh (PowerShell 7+) is required to run the PR-review engine but was not found on PATH") + return pwsh + + +def _git(repo_path: Path, *args: str) -> str: + result = subprocess.run(["git", *args], cwd=repo_path, capture_output=True, text=True, check=False) + if result.returncode != 0: + raise AgentError(f"git {' '.join(args)} failed: {result.stderr.strip()}") + return result.stdout.strip() + + +def _commit_patched_worktree(repo_path: Path) -> str: + base_ref = _git(repo_path, "rev-parse", "HEAD") + _git(repo_path, "add", "-A") + _git( + repo_path, + "-c", + "user.name=bcbench", + "-c", + "user.email=bcbench@local", + "-c", + "commit.gpgsign=false", + "commit", + "--allow-empty", + "-m", + "bcbench: apply dataset patch for local review", + ) + return base_ref + + +def _run_local_review( + local_review_script: Path, + repo_path: Path, + engine_output_dir: Path, + base_ref: str, + model: str, +) -> None: + # Invoke-LocalReview.ps1 self-contains fetch+filter+run and reads GH_TOKEN and the optional + # BCQUALITY_REF override from the inherited env (via the engine's Get-BCQualityConfig). + args = [ + _pwsh(), + "-NoProfile", + "-File", + str(local_review_script), + "-Workspace", + str(repo_path), + "-BaseRef", + base_ref, + "-OutputDir", + str(engine_output_dir), + "-Model", + model, + ] + logger.info(f"Invoking PR-review engine: {local_review_script}") + try: + result = subprocess.run(args, capture_output=True, text=True, timeout=_ENGINE_TIMEOUT_SECONDS, check=False) + except subprocess.TimeoutExpired as exc: + raise AgentTimeoutError(f"PR-review engine timed out after {_ENGINE_TIMEOUT_SECONDS}s") from exc + if result.stdout: + logger.info(result.stdout) + if result.returncode != 0: + raise AgentError(f"PR-review engine failed (exit {result.returncode}): {result.stderr.strip() or result.stdout.strip()}") + + +def _findings_to_review_comments(payload: dict) -> list[dict]: + comments: list[dict] = [] + for finding in payload.get("findings") or []: + file_path = finding.get("filePath") + line = finding.get("lineNumber") + if not file_path or not line: + continue + issue = (finding.get("issue") or "").strip() + recommendation = (finding.get("recommendation") or "").strip() + body = (f"{issue}\n\nRecommendation: {recommendation}" if issue else recommendation) if recommendation else issue + comment: dict = {"file": file_path, "line_start": line, "body": body} + severity = finding.get("severity") + if severity: + comment["severity"] = str(severity).lower() + comments.append(comment) + return comments + + +def _write_review_json(review_output_dir: Path, repo_path: Path) -> int: + findings_file = review_output_dir / FINDINGS_FILE_NAME + if not findings_file.exists(): + raise AgentError(f"Engine did not produce {FINDINGS_FILE_NAME} at {review_output_dir}") + payload = json.loads(findings_file.read_text(encoding="utf-8")) + comments = _findings_to_review_comments(payload) + (repo_path / REVIEW_OUTPUT_FILE).write_text(json.dumps(comments, indent=2), encoding="utf-8") + logger.info(f"Wrote {len(comments)} review comment(s) to {repo_path / REVIEW_OUTPUT_FILE}") + return len(comments) + + +def run_engine_review( + entry: BaseDatasetEntry, + model: str, + repo_path: Path, + output_dir: Path, + engine_scripts_dir: Path, +) -> tuple[AgentMetrics | None, ExperimentConfiguration]: + local_review_script = engine_scripts_dir / LOCAL_REVIEW_SCRIPT_NAME + if not local_review_script.exists(): + raise AgentError(f"Engine script not found: {local_review_script}") + + logger.info(f"Running PR-review engine on: {entry.instance_id}") + + output_dir.mkdir(parents=True, exist_ok=True) + engine_output_dir = output_dir / "engine-output" + + base_ref = _commit_patched_worktree(repo_path) + + started_at = time.monotonic() + _run_local_review(local_review_script, repo_path, engine_output_dir, base_ref, model) + execution_time = time.monotonic() - started_at + + _write_review_json(engine_output_dir / "review-output", repo_path) + + metrics = AgentMetrics(execution_time=execution_time) + experiment = ExperimentConfiguration(custom_agent="bc-pr-review-engine") + return metrics, experiment