From b95acc373328ffdeaa0ee0c3712cf30791d681bf Mon Sep 17 00:00:00 2001
From: gsoosk <30385917+gsoosk@users.noreply.github.com>
Date: Mon, 10 Aug 2026 20:14:31 +0100
Subject: [PATCH] feat: add GitHub Copilot CLI as LLM backend
Adds a `copilot_cli` provider that drives the GitHub Copilot CLI
(`copilot -p`) as a non-interactive LLM backend. Authentication comes
from the CLI's own login session (or a `COPILOT_GITHUB_TOKEN`), so no API
key is required, and any model exposed by a Copilot subscription can be
used.
The CLI has no `--system-prompt` flag, so the system message is prepended
to the prompt. `reasoning_effort` maps to `--effort` and is validated
against the values the CLI accepts; `max_ai_credits` maps to
`--max-ai-credits`. Generation runs with `--silent --no-color
--no-ask-user --no-custom-instructions --disable-builtin-mcps` to keep
stdout clean and prompts reproducible. Tools are not pre-approved by
default since evolution only needs text generation; `allow_all_tools`
opts in.
A non-zero exit is raised rather than warned about, because the CLI
reports failures (such as an unavailable model) on stderr with an empty
stdout, and a missing binary fails immediately instead of consuming the
retry budget.
`temperature`, `top_p` and `max_tokens` have no CLI equivalent and are
ignored by this backend.
Adds a quickstart example that reuses the function_minimization program
and evaluator.
---
README.md | 27 +
examples/copilot_cli_quickstart/README.md | 82 +++
examples/copilot_cli_quickstart/config.yaml | 57 ++
examples/copilot_cli_quickstart/evaluator.py | 493 ++++++++++++++++++
.../copilot_cli_quickstart/initial_program.py | 51 ++
openevolve/config.py | 7 +
openevolve/llm/__init__.py | 3 +
openevolve/llm/copilot_cli.py | 177 +++++++
openevolve/llm/ensemble.py | 7 +
tests/test_copilot_cli_llm.py | 256 +++++++++
10 files changed, 1160 insertions(+)
create mode 100644 examples/copilot_cli_quickstart/README.md
create mode 100644 examples/copilot_cli_quickstart/config.yaml
create mode 100644 examples/copilot_cli_quickstart/evaluator.py
create mode 100644 examples/copilot_cli_quickstart/initial_program.py
create mode 100644 openevolve/llm/copilot_cli.py
create mode 100644 tests/test_copilot_cli_llm.py
diff --git a/README.md b/README.md
index 785a1d5804..ac7338ca00 100644
--- a/README.md
+++ b/README.md
@@ -384,6 +384,33 @@ See the [Claude Code quickstart example](examples/claude_code_quickstart/) for a
+
+🤖 GitHub Copilot CLI (No API Key)
+
+Use the [GitHub Copilot CLI](https://docs.github.com/copilot/how-tos/copilot-cli) as the LLM backend — no API keys needed, authentication uses your GitHub Copilot subscription.
+
+```bash
+# Install and authenticate
+npm install -g @github/copilot
+copilot login
+```
+
+```yaml
+# config.yaml
+llm:
+ provider: "copilot_cli"
+ models:
+ - name: "claude-sonnet-4.6"
+ weight: 0.8
+ reasoning_effort: "medium"
+ - name: "gpt-5.4"
+ weight: 0.2
+```
+
+See the [Copilot CLI quickstart example](examples/copilot_cli_quickstart/) for a complete walkthrough.
+
+
+
## Examples Gallery
diff --git a/examples/copilot_cli_quickstart/README.md b/examples/copilot_cli_quickstart/README.md
new file mode 100644
index 0000000000..e1d9700d30
--- /dev/null
+++ b/examples/copilot_cli_quickstart/README.md
@@ -0,0 +1,82 @@
+# GitHub Copilot CLI Quickstart
+
+This example shows how to use the [GitHub Copilot CLI](https://docs.github.com/copilot/how-tos/copilot-cli) as the LLM backend for OpenEvolve. No API keys are needed — authentication uses your GitHub Copilot subscription.
+
+## Prerequisites
+
+1. **Install GitHub Copilot CLI:**
+ ```bash
+ npm install -g @github/copilot
+ ```
+
+2. **Authenticate:**
+ ```bash
+ copilot login
+ ```
+
+ For headless runs you can instead export a fine-grained personal access token with the "Copilot Requests" permission as `COPILOT_GITHUB_TOKEN`.
+
+3. **Install OpenEvolve:**
+ ```bash
+ pip install openevolve
+ ```
+
+## Run
+
+```bash
+python openevolve-run.py \
+ examples/copilot_cli_quickstart/initial_program.py \
+ examples/copilot_cli_quickstart/evaluator.py \
+ --config examples/copilot_cli_quickstart/config.yaml \
+ --iterations 50
+```
+
+## How It Works
+
+The `config.yaml` sets `provider: "copilot_cli"` which routes all LLM calls through the `copilot -p` subprocess instead of the OpenAI-compatible API. The CLI handles authentication, model selection, and billing.
+
+Because the CLI has no `--system-prompt` flag, the system message is prepended to the prompt. Each call runs with `--silent --no-color --no-ask-user --no-custom-instructions --disable-builtin-mcps` so that only the model response reaches stdout and prompts stay reproducible across machines.
+
+### Key Config Options
+
+| Field | Description | Default |
+|-------|-------------|---------|
+| `provider` | Set to `"copilot_cli"` to use the CLI backend | `"openai"` |
+| `name` | Model passed to `--model`; use `"auto"` to let Copilot choose | `"auto"` |
+| `reasoning_effort` | One of `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max` | unset |
+| `max_ai_credits` | Per-call AI credit budget | unset |
+| `allow_all_tools` | Pre-approve the agent's tools | `false` |
+| `timeout` | CLI timeout in seconds | `300` |
+| `retries` | Number of retry attempts on failure | `3` |
+| `retry_delay` | Seconds between retries | `5` |
+
+`temperature`, `top_p` and `max_tokens` have no equivalent CLI flag and are ignored by this backend.
+
+Evolution only needs text generation, so tools are left unapproved by default. Set `allow_all_tools: true` only if you want the agent to read and write files while generating.
+
+### Ensemble Example
+
+A single provider covers every model your subscription exposes, so an ensemble can mix vendors:
+
+```yaml
+llm:
+ provider: "copilot_cli"
+ models:
+ - name: "claude-sonnet-4.6"
+ weight: 0.5
+ - name: "gpt-5.4"
+ weight: 0.3
+ - name: "gemini-3.1-pro-preview"
+ weight: 0.2
+```
+
+### Programmatic Usage
+
+You can also inject the Copilot CLI backend at runtime without modifying config files:
+
+```python
+from openevolve.llm.copilot_cli import init_copilot_cli_client
+
+for model_cfg in config.llm.models:
+ model_cfg.init_client = init_copilot_cli_client
+```
diff --git a/examples/copilot_cli_quickstart/config.yaml b/examples/copilot_cli_quickstart/config.yaml
new file mode 100644
index 0000000000..046d426988
--- /dev/null
+++ b/examples/copilot_cli_quickstart/config.yaml
@@ -0,0 +1,57 @@
+# Configuration for function minimization using GitHub Copilot CLI as the LLM backend.
+# No API keys needed - authentication uses `copilot login`.
+#
+# Prerequisites:
+# 1. Install GitHub Copilot CLI: npm install -g @github/copilot
+# 2. Authenticate: copilot login
+#
+# Run:
+# python openevolve-run.py \
+# examples/copilot_cli_quickstart/initial_program.py \
+# examples/copilot_cli_quickstart/evaluator.py \
+# --config examples/copilot_cli_quickstart/config.yaml \
+# --iterations 50
+
+max_iterations: 50
+checkpoint_interval: 10
+
+llm:
+ provider: "copilot_cli"
+ models:
+ - name: "claude-sonnet-4.6"
+ weight: 0.8
+ timeout: 300
+ reasoning_effort: "medium"
+ max_ai_credits: 5.0
+ - name: "gpt-5.4"
+ weight: 0.2
+ timeout: 300
+ reasoning_effort: "low"
+ max_ai_credits: 2.0
+ retries: 3
+ retry_delay: 5
+
+prompt:
+ system_message: >
+ You are an expert programmer specializing in optimization algorithms.
+ Your task is to improve a function minimization algorithm to find the
+ global minimum of a complex function with many local minima.
+ The function is f(x, y) = sin(x) * cos(y) + sin(x*y) + (x^2 + y^2)/20.
+ Focus on improving the search_algorithm function to reliably find the
+ global minimum, escaping local minima that might trap simple algorithms.
+
+database:
+ population_size: 50
+ archive_size: 20
+ num_islands: 3
+ elite_selection_ratio: 0.2
+ exploitation_ratio: 0.7
+ similarity_threshold: 0.99
+
+evaluator:
+ timeout: 60
+ cascade_thresholds: [1.3]
+ parallel_evaluations: 3
+
+diff_based_evolution: true
+max_code_length: 20000
diff --git a/examples/copilot_cli_quickstart/evaluator.py b/examples/copilot_cli_quickstart/evaluator.py
new file mode 100644
index 0000000000..f16318125b
--- /dev/null
+++ b/examples/copilot_cli_quickstart/evaluator.py
@@ -0,0 +1,493 @@
+"""
+Evaluator for the function minimization example
+"""
+
+import importlib.util
+import numpy as np
+import time
+import concurrent.futures
+import traceback
+import signal
+from openevolve.evaluation_result import EvaluationResult
+
+
+def run_with_timeout(func, args=(), kwargs={}, timeout_seconds=5):
+ """
+ Run a function with a timeout using concurrent.futures
+
+ Args:
+ func: Function to run
+ args: Arguments to pass to the function
+ kwargs: Keyword arguments to pass to the function
+ timeout_seconds: Timeout in seconds
+
+ Returns:
+ Result of the function or raises TimeoutError
+ """
+ with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+ future = executor.submit(func, *args, **kwargs)
+ try:
+ result = future.result(timeout=timeout_seconds)
+ return result
+ except concurrent.futures.TimeoutError:
+ raise TimeoutError(f"Function timed out after {timeout_seconds} seconds")
+
+
+def safe_float(value):
+ """Convert a value to float safely"""
+ try:
+ return float(value)
+ except (TypeError, ValueError):
+ print(f"Warning: Could not convert {value} of type {type(value)} to float")
+ return 0.0
+
+
+def evaluate(program_path):
+ """
+ Evaluate the program by running it multiple times and checking how close
+ it gets to the known global minimum.
+
+ Args:
+ program_path: Path to the program file
+
+ Returns:
+ Dictionary of metrics
+ """
+ # Known global minimum (approximate)
+ GLOBAL_MIN_X = -1.704
+ GLOBAL_MIN_Y = 0.678
+ GLOBAL_MIN_VALUE = -1.519
+
+ try:
+ # Load the program
+ spec = importlib.util.spec_from_file_location("program", program_path)
+ program = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(program)
+
+ # Check if the required function exists
+ if not hasattr(program, "run_search"):
+ print(f"Error: program does not have 'run_search' function")
+
+ error_artifacts = {
+ "error_type": "MissingFunction",
+ "error_message": "Program is missing required 'run_search' function",
+ "suggestion": "Make sure your program includes a function named 'run_search' that returns (x, y, value) or (x, y)"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "value_score": 0.0,
+ "distance_score": 0.0,
+ "reliability_score": 0.0,
+ "combined_score": 0.0,
+ "error": "Missing run_search function",
+ },
+ artifacts=error_artifacts
+ )
+
+ # Run multiple trials
+ num_trials = 10
+ x_values = []
+ y_values = []
+ values = []
+ distances = []
+ times = []
+ success_count = 0
+
+ for trial in range(num_trials):
+ try:
+ start_time = time.time()
+
+ # Run with timeout
+ result = run_with_timeout(program.run_search, timeout_seconds=5)
+
+ # Handle different result formats
+ if isinstance(result, tuple):
+ if len(result) == 3:
+ x, y, value = result
+ elif len(result) == 2:
+ # Assume it's (x, y) and calculate value
+ x, y = result
+ # Calculate the function value since it wasn't returned
+ value = np.sin(x) * np.cos(y) + np.sin(x * y) + (x**2 + y**2) / 20
+ print(f"Trial {trial}: Got 2 values, calculated function value: {value}")
+ else:
+ print(
+ f"Trial {trial}: Invalid result format, expected tuple of 2 or 3 values but got {len(result)}"
+ )
+ continue
+ else:
+ print(
+ f"Trial {trial}: Invalid result format, expected tuple but got {type(result)}"
+ )
+ continue
+
+ end_time = time.time()
+
+ # Ensure all values are float
+ x = safe_float(x)
+ y = safe_float(y)
+ value = safe_float(value)
+
+ # Check if the result is valid (not NaN or infinite)
+ if (
+ np.isnan(x)
+ or np.isnan(y)
+ or np.isnan(value)
+ or np.isinf(x)
+ or np.isinf(y)
+ or np.isinf(value)
+ ):
+ print(f"Trial {trial}: Invalid result, got x={x}, y={y}, value={value}")
+ continue
+
+ # Calculate metrics
+ x_diff = x - GLOBAL_MIN_X
+ y_diff = y - GLOBAL_MIN_Y
+ distance_to_global = np.sqrt(x_diff**2 + y_diff**2)
+
+ x_values.append(x)
+ y_values.append(y)
+ values.append(value)
+ distances.append(distance_to_global)
+ times.append(end_time - start_time)
+ success_count += 1
+
+ except TimeoutError as e:
+ print(f"Trial {trial}: {str(e)}")
+ continue
+ except IndexError as e:
+ # Specifically handle IndexError which often happens with early termination checks
+ print(f"Trial {trial}: IndexError - {str(e)}")
+ print(
+ "This is likely due to a list index check before the list is fully populated."
+ )
+ continue
+ except Exception as e:
+ print(f"Trial {trial}: Error - {str(e)}")
+ print(traceback.format_exc())
+ continue
+
+ # If all trials failed, return zero scores
+ if success_count == 0:
+ error_artifacts = {
+ "error_type": "AllTrialsFailed",
+ "error_message": f"All {num_trials} trials failed - common issues: timeouts, crashes, or invalid return values",
+ "suggestion": "Check for infinite loops, ensure function returns (x, y) or (x, y, value), and verify algorithm terminates within time limit"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "value_score": 0.0,
+ "distance_score": 0.0,
+ "reliability_score": 0.0,
+ "combined_score": 0.0,
+ "error": "All trials failed",
+ },
+ artifacts=error_artifacts
+ )
+
+ # Calculate metrics
+ avg_value = float(np.mean(values))
+ avg_distance = float(np.mean(distances))
+ avg_time = float(np.mean(times)) if times else 1.0
+
+ # Convert to scores (higher is better)
+ value_score = float(1.0 / (1.0 + abs(avg_value - GLOBAL_MIN_VALUE)))
+ distance_score = float(1.0 / (1.0 + avg_distance))
+
+ # Add reliability score based on success rate
+ reliability_score = float(success_count / num_trials)
+
+ # Calculate solution quality based on distance to global minimum
+ if avg_distance < 0.5: # Very close to the correct solution
+ solution_quality_multiplier = 1.5 # 50% bonus
+ elif avg_distance < 1.5: # In the right region
+ solution_quality_multiplier = 1.2 # 20% bonus
+ elif avg_distance < 3.0: # Getting closer
+ solution_quality_multiplier = 1.0 # No adjustment
+ else: # Not finding the right region
+ solution_quality_multiplier = 0.7 # 30% penalty
+
+ # Calculate combined score that prioritizes finding the global minimum
+ # Base score from value and distance, then apply solution quality multiplier
+ base_score = 0.5 * value_score + 0.3 * distance_score + 0.2 * reliability_score
+ combined_score = float(base_score * solution_quality_multiplier)
+
+ # Add artifacts for successful runs
+ artifacts = {
+ "convergence_info": f"Converged in {num_trials} trials with {success_count} successes",
+ "best_position": f"Final position: x={x_values[-1]:.4f}, y={y_values[-1]:.4f}" if x_values else "No successful trials",
+ "average_distance_to_global": f"{avg_distance:.4f}",
+ "search_efficiency": f"Success rate: {reliability_score:.2%}"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "value_score": value_score,
+ "distance_score": distance_score,
+ "reliability_score": reliability_score,
+ "combined_score": combined_score,
+ },
+ artifacts=artifacts
+ )
+ except Exception as e:
+ print(f"Evaluation failed completely: {str(e)}")
+ print(traceback.format_exc())
+
+ # Create error artifacts
+ error_artifacts = {
+ "error_type": type(e).__name__,
+ "error_message": str(e),
+ "full_traceback": traceback.format_exc(),
+ "suggestion": "Check for syntax errors or missing imports in the generated code"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "value_score": 0.0,
+ "distance_score": 0.0,
+ "reliability_score": 0.0,
+ "combined_score": 0.0,
+ "error": str(e),
+ },
+ artifacts=error_artifacts
+ )
+
+
+# Stage-based evaluation for cascade evaluation
+def evaluate_stage1(program_path):
+ """First stage evaluation with fewer trials"""
+ # Known global minimum (approximate)
+ GLOBAL_MIN_X = float(-1.704)
+ GLOBAL_MIN_Y = float(0.678)
+ GLOBAL_MIN_VALUE = float(-1.519)
+
+ # Quick check to see if the program runs without errors
+ try:
+ # Load the program
+ spec = importlib.util.spec_from_file_location("program", program_path)
+ program = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(program)
+
+ # Check if the required function exists
+ if not hasattr(program, "run_search"):
+ print(f"Stage 1 validation: Program does not have 'run_search' function")
+
+ error_artifacts = {
+ "error_type": "MissingFunction",
+ "error_message": "Stage 1: Program is missing required 'run_search' function",
+ "suggestion": "Make sure your program includes a function named 'run_search' that returns (x, y, value) or (x, y)"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "runs_successfully": 0.0,
+ "combined_score": 0.0,
+ "error": "Missing run_search function"
+ },
+ artifacts=error_artifacts
+ )
+
+ try:
+ # Run a single trial with timeout
+ result = run_with_timeout(program.run_search, timeout_seconds=5)
+
+ # Handle different result formats
+ if isinstance(result, tuple):
+ if len(result) == 3:
+ x, y, value = result
+ elif len(result) == 2:
+ # Assume it's (x, y) and calculate value
+ x, y = result
+ # Calculate the function value since it wasn't returned
+ value = np.sin(x) * np.cos(y) + np.sin(x * y) + (x**2 + y**2) / 20
+ print(f"Stage 1: Got 2 values, calculated function value: {value}")
+ else:
+ print(
+ f"Stage 1: Invalid result format, expected tuple of 2 or 3 values but got {len(result)}"
+ )
+
+ error_artifacts = {
+ "error_type": "InvalidReturnFormat",
+ "error_message": f"Stage 1: Function returned tuple with {len(result)} values, expected 2 or 3",
+ "suggestion": "run_search() must return (x, y) or (x, y, value) - check your return statement"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "runs_successfully": 0.0,
+ "combined_score": 0.0,
+ "error": "Invalid result format"
+ },
+ artifacts=error_artifacts
+ )
+ else:
+ print(f"Stage 1: Invalid result format, expected tuple but got {type(result)}")
+
+ error_artifacts = {
+ "error_type": "InvalidReturnType",
+ "error_message": f"Stage 1: Function returned {type(result)}, expected tuple",
+ "suggestion": "run_search() must return a tuple like (x, y) or (x, y, value), not a single value or other type"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "runs_successfully": 0.0,
+ "combined_score": 0.0,
+ "error": "Invalid result format"
+ },
+ artifacts=error_artifacts
+ )
+
+ # Ensure all values are float
+ x = safe_float(x)
+ y = safe_float(y)
+ value = safe_float(value)
+
+ # Check if the result is valid
+ if (
+ np.isnan(x)
+ or np.isnan(y)
+ or np.isnan(value)
+ or np.isinf(x)
+ or np.isinf(y)
+ or np.isinf(value)
+ ):
+ print(f"Stage 1 validation: Invalid result, got x={x}, y={y}, value={value}")
+
+ error_artifacts = {
+ "error_type": "InvalidResultValues",
+ "error_message": f"Stage 1: Got invalid values - x={x}, y={y}, value={value}",
+ "suggestion": "Function returned NaN or infinite values. Check for division by zero, invalid math operations, or uninitialized variables"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "runs_successfully": 0.5,
+ "combined_score": 0.0,
+ "error": "Invalid result values"
+ },
+ artifacts=error_artifacts
+ )
+
+ # Calculate distance safely
+ x_diff = float(x) - GLOBAL_MIN_X
+ y_diff = float(y) - GLOBAL_MIN_Y
+ distance = float(np.sqrt(x_diff**2 + y_diff**2))
+
+ # Calculate value-based score
+ value_score = float(1.0 / (1.0 + abs(value - GLOBAL_MIN_VALUE)))
+ distance_score = float(1.0 / (1.0 + distance))
+
+ # Calculate solution quality based on distance to global minimum
+ if distance < 0.5: # Very close to the correct solution
+ solution_quality_multiplier = 1.4 # 40% bonus
+ elif distance < 1.5: # In the right region
+ solution_quality_multiplier = 1.15 # 15% bonus
+ elif distance < 3.0: # Getting closer
+ solution_quality_multiplier = 1.0 # No adjustment
+ else: # Not finding the right region
+ solution_quality_multiplier = 0.8 # 20% penalty
+
+ # Calculate combined score for stage 1
+ base_score = 0.6 * value_score + 0.4 * distance_score
+ combined_score = float(base_score * solution_quality_multiplier)
+
+ # Add artifacts for successful stage 1
+ stage1_artifacts = {
+ "stage1_result": f"Found solution at x={x:.4f}, y={y:.4f} with value={value:.4f}",
+ "distance_to_global": f"{distance:.4f}",
+ "solution_quality": f"Distance < 0.5: Very close" if distance < 0.5 else f"Distance < 1.5: Good region" if distance < 1.5 else "Could be improved"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "runs_successfully": 1.0,
+ "value_score": value_score,
+ "distance_score": distance_score,
+ "combined_score": combined_score,
+ },
+ artifacts=stage1_artifacts
+ )
+ except TimeoutError as e:
+ print(f"Stage 1 evaluation timed out: {e}")
+
+ error_artifacts = {
+ "error_type": "TimeoutError",
+ "error_message": "Stage 1: Function execution exceeded 5 second timeout",
+ "suggestion": "Function is likely stuck in infinite loop or doing too much computation. Try reducing iterations or adding early termination conditions"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "runs_successfully": 0.0,
+ "combined_score": 0.0,
+ "error": "Timeout"
+ },
+ artifacts=error_artifacts
+ )
+ except IndexError as e:
+ # Specifically handle IndexError which often happens with early termination checks
+ print(f"Stage 1 evaluation failed with IndexError: {e}")
+ print("This is likely due to a list index check before the list is fully populated.")
+
+ error_artifacts = {
+ "error_type": "IndexError",
+ "error_message": f"Stage 1: {str(e)}",
+ "suggestion": "List index out of range - likely accessing empty list or wrong index. Check list initialization and bounds"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "runs_successfully": 0.0,
+ "combined_score": 0.0,
+ "error": f"IndexError: {str(e)}"
+ },
+ artifacts=error_artifacts
+ )
+ except Exception as e:
+ print(f"Stage 1 evaluation failed: {e}")
+ print(traceback.format_exc())
+
+ error_artifacts = {
+ "error_type": type(e).__name__,
+ "error_message": f"Stage 1: {str(e)}",
+ "full_traceback": traceback.format_exc(),
+ "suggestion": "Unexpected error occurred. Check the traceback for specific issue"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "runs_successfully": 0.0,
+ "combined_score": 0.0,
+ "error": str(e)
+ },
+ artifacts=error_artifacts
+ )
+
+ except Exception as e:
+ print(f"Stage 1 evaluation failed: {e}")
+ print(traceback.format_exc())
+
+ error_artifacts = {
+ "error_type": type(e).__name__,
+ "error_message": f"Stage 1 outer exception: {str(e)}",
+ "full_traceback": traceback.format_exc(),
+ "suggestion": "Critical error during stage 1 evaluation. Check program syntax and imports"
+ }
+
+ return EvaluationResult(
+ metrics={
+ "runs_successfully": 0.0,
+ "combined_score": 0.0,
+ "error": str(e)
+ },
+ artifacts=error_artifacts
+ )
+
+
+def evaluate_stage2(program_path):
+ """Second stage evaluation with more thorough testing"""
+ # Full evaluation as in the main evaluate function
+ return evaluate(program_path)
diff --git a/examples/copilot_cli_quickstart/initial_program.py b/examples/copilot_cli_quickstart/initial_program.py
new file mode 100644
index 0000000000..670c02cc45
--- /dev/null
+++ b/examples/copilot_cli_quickstart/initial_program.py
@@ -0,0 +1,51 @@
+# EVOLVE-BLOCK-START
+"""Function minimization example for OpenEvolve"""
+import numpy as np
+
+
+def search_algorithm(iterations=1000, bounds=(-5, 5)):
+ """
+ A simple random search algorithm that often gets stuck in local minima.
+
+ Args:
+ iterations: Number of iterations to run
+ bounds: Bounds for the search space (min, max)
+
+ Returns:
+ Tuple of (best_x, best_y, best_value)
+ """
+ # Initialize with a random point
+ best_x = np.random.uniform(bounds[0], bounds[1])
+ best_y = np.random.uniform(bounds[0], bounds[1])
+ best_value = evaluate_function(best_x, best_y)
+
+ for _ in range(iterations):
+ # Simple random search
+ x = np.random.uniform(bounds[0], bounds[1])
+ y = np.random.uniform(bounds[0], bounds[1])
+ value = evaluate_function(x, y)
+
+ if value < best_value:
+ best_value = value
+ best_x, best_y = x, y
+
+ return best_x, best_y, best_value
+
+
+# EVOLVE-BLOCK-END
+
+
+# This part remains fixed (not evolved)
+def evaluate_function(x, y):
+ """The complex function we're trying to minimize"""
+ return np.sin(x) * np.cos(y) + np.sin(x * y) + (x**2 + y**2) / 20
+
+
+def run_search():
+ x, y, value = search_algorithm()
+ return x, y, value
+
+
+if __name__ == "__main__":
+ x, y, value = run_search()
+ print(f"Found minimum at ({x}, {y}) with value {value}")
diff --git a/openevolve/config.py b/openevolve/config.py
index c19ab4ca1d..4dbd720a65 100644
--- a/openevolve/config.py
+++ b/openevolve/config.py
@@ -57,6 +57,7 @@ class LLMModelConfig:
name: str = None
# LLM provider: "openai" (default), "claude_code" (Claude Code CLI)
+ # Also supports "copilot_cli" (GitHub Copilot CLI)
provider: Optional[str] = None
# Custom LLM client
@@ -85,6 +86,12 @@ class LLMModelConfig:
# Claude Code CLI budget per call (USD)
max_budget_usd: Optional[float] = None
+ # GitHub Copilot CLI budget per call (AI credits)
+ max_ai_credits: Optional[float] = None
+
+ # GitHub Copilot CLI: run the agent with all tools pre-approved
+ allow_all_tools: Optional[bool] = None
+
# Manual mode (human-in-the-loop)
manual_mode: Optional[bool] = None
_manual_queue_dir: Optional[str] = None
diff --git a/openevolve/llm/__init__.py b/openevolve/llm/__init__.py
index 856843d4b0..6c5b938fd5 100644
--- a/openevolve/llm/__init__.py
+++ b/openevolve/llm/__init__.py
@@ -6,11 +6,14 @@
from openevolve.llm.ensemble import LLMEnsemble
from openevolve.llm.openai import OpenAILLM
from openevolve.llm.claude_code import ClaudeCodeLLM, init_claude_code_client
+from openevolve.llm.copilot_cli import CopilotCLILLM, init_copilot_cli_client
__all__ = [
"LLMInterface",
"OpenAILLM",
"ClaudeCodeLLM",
"init_claude_code_client",
+ "CopilotCLILLM",
+ "init_copilot_cli_client",
"LLMEnsemble",
]
diff --git a/openevolve/llm/copilot_cli.py b/openevolve/llm/copilot_cli.py
new file mode 100644
index 0000000000..1b70ecc7f0
--- /dev/null
+++ b/openevolve/llm/copilot_cli.py
@@ -0,0 +1,177 @@
+"""
+GitHub Copilot CLI interface for LLMs.
+
+Uses the GitHub Copilot CLI (`copilot -p`) as a non-interactive LLM backend,
+enabling OpenEvolve to run with any model offered by a GitHub Copilot
+subscription without requiring direct API keys — authentication is handled by
+the CLI's own login session.
+
+Usage in config.yaml:
+ llm:
+ provider: "copilot_cli"
+ models:
+ - name: "claude-sonnet-4.6"
+ weight: 1.0
+ timeout: 300
+ reasoning_effort: "medium"
+ max_ai_credits: 5.0
+
+Or inject programmatically:
+ from openevolve.llm.copilot_cli import init_copilot_cli_client
+ for model_cfg in config.llm.models:
+ model_cfg.init_client = init_copilot_cli_client
+
+Note that `temperature`, `top_p` and `max_tokens` have no equivalent CLI flag
+and are therefore ignored by this backend.
+"""
+
+import asyncio
+import logging
+import subprocess
+from typing import Any, Dict, List
+
+from openevolve.llm.base import LLMInterface
+
+logger = logging.getLogger(__name__)
+
+VALID_REASONING_EFFORTS = ("none", "minimal", "low", "medium", "high", "xhigh", "max")
+
+
+def _cfg_value(model_cfg: Any, attr: str, default: Any) -> Any:
+ value = getattr(model_cfg, attr, None)
+ return default if value is None else value
+
+
+class CopilotCLILLM(LLMInterface):
+ """LLM interface that uses the GitHub Copilot CLI for generation.
+
+ Requires the `copilot` CLI to be installed and authenticated
+ (run `copilot login` first).
+ """
+
+ def __init__(self, model_cfg=None):
+ self.model = _cfg_value(model_cfg, "name", "auto")
+ self.system_message = getattr(model_cfg, "system_message", None)
+ self.timeout = _cfg_value(model_cfg, "timeout", 300)
+ self.weight = _cfg_value(model_cfg, "weight", 1.0)
+ self.retries = _cfg_value(model_cfg, "retries", 3)
+ self.retry_delay = _cfg_value(model_cfg, "retry_delay", 5)
+ self.reasoning_effort = getattr(model_cfg, "reasoning_effort", None)
+ self.max_ai_credits = getattr(model_cfg, "max_ai_credits", None)
+ self.allow_all_tools = _cfg_value(model_cfg, "allow_all_tools", False)
+ self.cwd = getattr(model_cfg, "cwd", None)
+ logger.info(f"Initialized CopilotCLILLM with model: {self.model}")
+
+ async def generate(self, prompt: str, **kwargs) -> str:
+ sys_msg = kwargs.pop("system_message", self.system_message) or ""
+ return await self.generate_with_context(
+ system_message=sys_msg,
+ messages=[{"role": "user", "content": prompt}],
+ **kwargs,
+ )
+
+ async def generate_with_context(
+ self, system_message: str, messages: List[Dict[str, str]], **kwargs
+ ) -> str:
+ user_content = "\n\n".join(
+ m.get("content", "") for m in messages if m.get("role") == "user"
+ )
+
+ cmd = self._build_command(system_message, user_content, **kwargs)
+
+ timeout = kwargs.get("timeout", self.timeout)
+ retries = kwargs.get("retries", self.retries)
+ retry_delay = kwargs.get("retry_delay", self.retry_delay)
+
+ loop = asyncio.get_event_loop()
+ for attempt in range(retries + 1):
+ try:
+ return await asyncio.wait_for(
+ loop.run_in_executor(None, lambda: self._run_cli(cmd, timeout)),
+ timeout=timeout + 30,
+ )
+ except FileNotFoundError:
+ logger.error("The `copilot` CLI was not found. Install it and run `copilot login`.")
+ raise
+ except asyncio.TimeoutError:
+ if attempt < retries:
+ logger.warning(
+ f"Copilot CLI timeout on attempt {attempt + 1}/{retries + 1}. Retrying..."
+ )
+ await asyncio.sleep(retry_delay)
+ else:
+ logger.error(f"All {retries + 1} attempts failed with timeout")
+ raise
+ except Exception as e:
+ if attempt < retries:
+ logger.warning(
+ f"Copilot CLI error on attempt {attempt + 1}/{retries + 1}: {e}. Retrying..."
+ )
+ await asyncio.sleep(retry_delay)
+ else:
+ logger.error(f"All {retries + 1} attempts failed with error: {e}")
+ raise
+
+ def _build_command(self, system_message: str, user_content: str, **kwargs) -> List[str]:
+ prompt = f"{system_message}\n\n{user_content}" if system_message else user_content
+
+ cmd = [
+ "copilot",
+ "-p",
+ prompt,
+ "--model",
+ self.model,
+ "--silent",
+ "--no-color",
+ "--no-ask-user",
+ "--no-custom-instructions",
+ "--disable-builtin-mcps",
+ ]
+
+ effort = kwargs.get("reasoning_effort", self.reasoning_effort)
+ if effort is not None:
+ if effort not in VALID_REASONING_EFFORTS:
+ raise ValueError(
+ f"Invalid reasoning_effort: {effort}. "
+ f"Expected one of {', '.join(VALID_REASONING_EFFORTS)}"
+ )
+ cmd.extend(["--effort", effort])
+
+ credits = kwargs.get("max_ai_credits", self.max_ai_credits)
+ if credits is not None:
+ cmd.extend(["--max-ai-credits", str(credits)])
+
+ if kwargs.get("allow_all_tools", self.allow_all_tools):
+ cmd.append("--allow-all-tools")
+
+ return cmd
+
+ def _run_cli(self, cmd: List[str], timeout: int) -> str:
+ try:
+ result = subprocess.run(
+ cmd,
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ cwd=self.cwd,
+ )
+ except subprocess.TimeoutExpired:
+ raise asyncio.TimeoutError("Copilot CLI subprocess timed out")
+
+ stderr = (result.stderr or "").strip()
+ if result.returncode != 0:
+ raise RuntimeError(
+ f"Copilot CLI exited with code {result.returncode}. stderr: {stderr[:500]}"
+ )
+ if stderr:
+ logger.warning(f"Copilot CLI stderr: {stderr[:500]}")
+
+ output = (result.stdout or "").strip()
+ if not output:
+ raise RuntimeError(f"Empty response from Copilot CLI. stderr: {stderr[:500]}")
+ return output
+
+
+def init_copilot_cli_client(model_cfg):
+ """Factory function compatible with OpenEvolve's init_client config hook."""
+ return CopilotCLILLM(model_cfg)
diff --git a/openevolve/llm/ensemble.py b/openevolve/llm/ensemble.py
index b9161382a1..c05b428291 100644
--- a/openevolve/llm/ensemble.py
+++ b/openevolve/llm/ensemble.py
@@ -23,6 +23,13 @@
except ImportError:
pass
+try:
+ from openevolve.llm.copilot_cli import CopilotCLILLM
+
+ _PROVIDER_REGISTRY["copilot_cli"] = lambda cfg: CopilotCLILLM(cfg)
+except ImportError:
+ pass
+
def _create_model(model_cfg: LLMModelConfig) -> LLMInterface:
if model_cfg.init_client:
diff --git a/tests/test_copilot_cli_llm.py b/tests/test_copilot_cli_llm.py
new file mode 100644
index 0000000000..8074fb4a8e
--- /dev/null
+++ b/tests/test_copilot_cli_llm.py
@@ -0,0 +1,256 @@
+"""Tests for the GitHub Copilot CLI LLM backend."""
+
+import asyncio
+import unittest
+from unittest.mock import MagicMock, patch
+
+from openevolve.llm.copilot_cli import CopilotCLILLM, init_copilot_cli_client
+
+
+def _make_cfg(**overrides):
+ defaults = {
+ "name": "claude-sonnet-4.6",
+ "system_message": None,
+ "timeout": 10,
+ "weight": 1.0,
+ "retries": 3,
+ "retry_delay": 5,
+ "reasoning_effort": None,
+ "max_ai_credits": None,
+ "allow_all_tools": None,
+ "cwd": None,
+ }
+ defaults.update(overrides)
+ cfg = MagicMock()
+ for key, value in defaults.items():
+ setattr(cfg, key, value)
+ return cfg
+
+
+def _ok(stdout="Generated response text"):
+ return MagicMock(returncode=0, stdout=stdout, stderr="")
+
+
+class TestCopilotCLILLM(unittest.TestCase):
+ def test_init_defaults(self):
+ llm = CopilotCLILLM(_make_cfg())
+ self.assertEqual(llm.model, "claude-sonnet-4.6")
+ self.assertEqual(llm.timeout, 10)
+ self.assertEqual(llm.weight, 1.0)
+ self.assertEqual(llm.retries, 3)
+ self.assertFalse(llm.allow_all_tools)
+
+ def test_init_falls_back_when_config_values_are_none(self):
+ llm = CopilotCLILLM(_make_cfg(name=None, timeout=None, retries=None, retry_delay=None))
+ self.assertEqual(llm.model, "auto")
+ self.assertEqual(llm.timeout, 300)
+ self.assertEqual(llm.retries, 3)
+ self.assertEqual(llm.retry_delay, 5)
+
+ def test_init_keeps_zero_retry_delay(self):
+ llm = CopilotCLILLM(_make_cfg(retry_delay=0))
+ self.assertEqual(llm.retry_delay, 0)
+
+ def test_factory_function(self):
+ llm = init_copilot_cli_client(_make_cfg())
+ self.assertIsInstance(llm, CopilotCLILLM)
+ self.assertEqual(llm.model, "claude-sonnet-4.6")
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_generate_calls_cli(self, mock_run):
+ mock_run.return_value = _ok()
+ llm = CopilotCLILLM(_make_cfg())
+ result = asyncio.run(llm.generate("test prompt"))
+ self.assertEqual(result, "Generated response text")
+ mock_run.assert_called_once()
+ cmd = mock_run.call_args[0][0]
+ self.assertEqual(cmd[0], "copilot")
+ self.assertEqual(cmd[cmd.index("-p") + 1], "test prompt")
+ self.assertEqual(cmd[cmd.index("--model") + 1], "claude-sonnet-4.6")
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_non_interactive_flags_are_set(self, mock_run):
+ mock_run.return_value = _ok()
+ llm = CopilotCLILLM(_make_cfg())
+ asyncio.run(llm.generate("test prompt"))
+ cmd = mock_run.call_args[0][0]
+ for flag in (
+ "--silent",
+ "--no-color",
+ "--no-ask-user",
+ "--no-custom-instructions",
+ "--disable-builtin-mcps",
+ ):
+ self.assertIn(flag, cmd)
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_system_message_is_prepended_to_prompt(self, mock_run):
+ mock_run.return_value = _ok()
+ llm = CopilotCLILLM(_make_cfg())
+ asyncio.run(llm.generate("prompt", system_message="You are an expert."))
+ cmd = mock_run.call_args[0][0]
+ self.assertEqual(cmd[cmd.index("-p") + 1], "You are an expert.\n\nprompt")
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_reasoning_effort_forwarded(self, mock_run):
+ mock_run.return_value = _ok()
+ llm = CopilotCLILLM(_make_cfg(reasoning_effort="high"))
+ asyncio.run(llm.generate("prompt"))
+ cmd = mock_run.call_args[0][0]
+ self.assertEqual(cmd[cmd.index("--effort") + 1], "high")
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_invalid_reasoning_effort_raises(self, mock_run):
+ llm = CopilotCLILLM(_make_cfg(reasoning_effort="turbo", retries=0))
+ with self.assertRaisesRegex(ValueError, "Invalid reasoning_effort: turbo"):
+ asyncio.run(llm.generate("prompt"))
+ mock_run.assert_not_called()
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_max_ai_credits_forwarded(self, mock_run):
+ mock_run.return_value = _ok()
+ llm = CopilotCLILLM(_make_cfg(max_ai_credits=2.5))
+ asyncio.run(llm.generate("prompt"))
+ cmd = mock_run.call_args[0][0]
+ self.assertEqual(cmd[cmd.index("--max-ai-credits") + 1], "2.5")
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_tools_are_not_approved_by_default(self, mock_run):
+ mock_run.return_value = _ok()
+ llm = CopilotCLILLM(_make_cfg())
+ asyncio.run(llm.generate("prompt"))
+ self.assertNotIn("--allow-all-tools", mock_run.call_args[0][0])
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_allow_all_tools_opt_in(self, mock_run):
+ mock_run.return_value = _ok()
+ llm = CopilotCLILLM(_make_cfg(allow_all_tools=True))
+ asyncio.run(llm.generate("prompt"))
+ self.assertIn("--allow-all-tools", mock_run.call_args[0][0])
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_non_zero_exit_raises(self, mock_run):
+ mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="model not available")
+ llm = CopilotCLILLM(_make_cfg(retries=0))
+ with self.assertRaisesRegex(RuntimeError, "model not available"):
+ asyncio.run(llm.generate("test prompt"))
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_empty_response_raises(self, mock_run):
+ mock_run.return_value = MagicMock(returncode=0, stdout=" ", stderr="")
+ llm = CopilotCLILLM(_make_cfg(retries=0))
+ with self.assertRaisesRegex(RuntimeError, "Empty response"):
+ asyncio.run(llm.generate("test prompt"))
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_missing_cli_raises_without_retrying(self, mock_run):
+ mock_run.side_effect = FileNotFoundError("copilot")
+ llm = CopilotCLILLM(_make_cfg(retries=3, retry_delay=0))
+ with self.assertRaises(FileNotFoundError):
+ asyncio.run(llm.generate("test prompt"))
+ self.assertEqual(mock_run.call_count, 1)
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_retry_on_failure(self, mock_run):
+ mock_run.side_effect = [
+ MagicMock(returncode=1, stdout="", stderr="transient error"),
+ _ok("success after retry"),
+ ]
+ llm = CopilotCLILLM(_make_cfg(retries=1, retry_delay=0))
+ result = asyncio.run(llm.generate("test prompt", retry_delay=0))
+ self.assertEqual(result, "success after retry")
+ self.assertEqual(mock_run.call_count, 2)
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_retries_exhausted_raises(self, mock_run):
+ mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="persistent error")
+ llm = CopilotCLILLM(_make_cfg(retries=2, retry_delay=0))
+ with self.assertRaises(RuntimeError):
+ asyncio.run(llm.generate("test prompt", retry_delay=0))
+ self.assertEqual(mock_run.call_count, 3)
+
+ @patch("openevolve.llm.copilot_cli.subprocess.run")
+ def test_generate_with_context(self, mock_run):
+ mock_run.return_value = _ok("ctx response")
+ llm = CopilotCLILLM(_make_cfg())
+ result = asyncio.run(
+ llm.generate_with_context(
+ system_message="sys",
+ messages=[
+ {"role": "user", "content": "first"},
+ {"role": "assistant", "content": "ignored"},
+ {"role": "user", "content": "second"},
+ ],
+ )
+ )
+ self.assertEqual(result, "ctx response")
+ prompt = mock_run.call_args[0][0][2]
+ self.assertEqual(prompt, "sys\n\nfirst\n\nsecond")
+ self.assertNotIn("ignored", prompt)
+
+
+class TestCopilotCLIConfig(unittest.TestCase):
+ def test_new_fields_default_to_none(self):
+ from openevolve.config import LLMModelConfig
+
+ cfg = LLMModelConfig()
+ self.assertIsNone(cfg.max_ai_credits)
+ self.assertIsNone(cfg.allow_all_tools)
+
+ def test_fields_from_dict(self):
+ from openevolve.config import Config
+
+ config = Config.from_dict(
+ {
+ "llm": {
+ "provider": "copilot_cli",
+ "models": [
+ {
+ "name": "claude-sonnet-4.6",
+ "weight": 1.0,
+ "max_ai_credits": 5.0,
+ "allow_all_tools": True,
+ "reasoning_effort": "medium",
+ }
+ ],
+ }
+ }
+ )
+ model = config.llm.models[0]
+ self.assertEqual(model.provider, "copilot_cli")
+ self.assertEqual(model.max_ai_credits, 5.0)
+ self.assertTrue(model.allow_all_tools)
+ self.assertEqual(model.reasoning_effort, "medium")
+
+
+class TestProviderRegistry(unittest.TestCase):
+ def test_copilot_cli_in_registry(self):
+ from openevolve.llm.ensemble import _PROVIDER_REGISTRY
+
+ self.assertIn("copilot_cli", _PROVIDER_REGISTRY)
+
+ def test_ensemble_creates_copilot_cli(self):
+ from openevolve.llm.ensemble import _create_model
+
+ cfg = _make_cfg()
+ cfg.init_client = None
+ cfg.provider = "copilot_cli"
+ self.assertIsInstance(_create_model(cfg), CopilotCLILLM)
+
+ def test_provider_propagates_from_llm_config(self):
+ from openevolve.config import Config
+
+ config = Config.from_dict(
+ {
+ "llm": {
+ "provider": "copilot_cli",
+ "models": [{"name": "gpt-5.4", "weight": 1.0}],
+ }
+ }
+ )
+ self.assertEqual(config.llm.models[0].provider, "copilot_cli")
+ self.assertEqual(config.llm.evaluator_models[0].provider, "copilot_cli")
+
+
+if __name__ == "__main__":
+ unittest.main()