diff --git a/.github/workflows/mcp_conformance.yml b/.github/workflows/mcp_conformance.yml index 3ca1dc0..712bcca 100644 --- a/.github/workflows/mcp_conformance.yml +++ b/.github/workflows/mcp_conformance.yml @@ -28,6 +28,12 @@ jobs: - name: Test conformance reporter run: tests/conformance/report-baseline-diff-test.sh + - name: Test client conformance adapter + run: | + python3 -m py_compile tests/conformance/write_client_config.py + tests/conformance/client-under-test-test.sh + tests/conformance/bless-client-baseline-test.sh + - name: Download the conformance binary uses: actions/download-artifact@v8.0.1 with: @@ -96,9 +102,12 @@ jobs: id: runner run: tests/conformance/run-conformance.sh + - name: Run scoped MCP 2026-07-28 client conformance + run: tests/conformance/run-client-conformance.sh + - name: Report conformance baseline diff if: always() - run: tests/conformance/report-baseline-diff.sh + run: tests/conformance/report-baseline-diff.sh conformance-results/server - name: Print live stack logs if: always() diff --git a/Makefile b/Makefile index 5f3ea13..78ea164 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ conformance: ## Build the data plane and run official MCP 2026-07-28 conformance docker build -t "$(CF_DATAPLANE_IMAGE)" -f docker/conformance.Dockerfile . CF_DATAPLANE_IMAGE="$(CF_DATAPLANE_IMAGE)" tests/conformance/run-local.sh -conformance-bless: ## Run conformance and update the expected-failure baseline +conformance-bless: ## Run conformance and update the server and client expected-failure baselines MCP_CONFORMANCE_BLESS=true $(MAKE) conformance docs-serve: ## Serve the wiki book locally at http://127.0.0.1:3000 diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index 65d7644..270668d 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -37,25 +37,39 @@ These run in `cargo nextest run` with no Docker dependencies. ## MCP Conformance CI `.github/workflows/mcp_conformance.yml` runs the pinned official conformance -suite `0.2.0-alpha.11` with `--requirements 2026-07-28`. Its small live path is -official runner → nginx → checked-out dataplane → fixture proxy → official -fixture, with the published `latest` control plane registering and publishing +suite `0.2.0-alpha.11` for MCP `2026-07-28` in both directions. The server leg +is official client → nginx → checked-out dataplane → fixture proxy → official +server, with the published `latest` control plane registering and publishing the fixture through Redis. The backend-only proxy rewrites `Host` to `localhost:3000`, which the official fixture's DNS-rebinding protection requires, while leaving dataplane header protections unchanged. The control plane uses ephemeral SQLite, so PostgreSQL is unnecessary. The harness lives in `tests/conformance/`. +The scoped client leg then treats the dataplane as an MCP client: the official +runner starts a scenario backend, the adapter publishes an isolated route to +Redis, and a downstream `tools/call` makes the dataplane connect to that +backend. It covers tool calls, per-request client metadata and protocol-version +retry, standard MCP headers, and custom parameter headers. The control plane is +stopped first so its periodic publisher cannot replace the scenario route or +probe the observation backend. OAuth client scenarios remain a control-plane +responsibility. Server and client results are written below `server/` and +`client/`, with separate `expected-failures.yml` and +`client-expected-failures.yml` baselines. + +`make conformance` runs both legs locally, while `make conformance-bless` runs +both and refreshes both expected-failure baselines from that run. + Because this conformance CLI cannot set a bearer header, nginx adds an ephemeral control-plane token when one is absent; there is no auth proxy or repository-owned JavaScript. A route probe prevents control-plane fallback. -Counts and the official fixture log appear directly in the Actions log, and -`expected-failures.yml` guards the current baseline. The job does not retain a -separate conformance artifact. `upstream-fixture-failures.yml` records the -pinned fixture's seven scored failures and one warning; its other 47 failures -are extension or pending scenarios and are already unscored. CI prints the -exact actual-versus-baseline diff, adds annotations for unexpected and stale -entries, and writes the same comparison to the job summary. +Counts and the official fixture log appear directly in the Actions log. The +job does not retain a separate conformance artifact. +`upstream-fixture-failures.yml` records the pinned fixture's seven scored +failures and one warning; its other 47 failures are extension or pending +scenarios and are already unscored. CI prints the exact server +actual-versus-baseline diff, adds annotations for unexpected and stale entries, +and writes the same comparison to the job summary. ## Full-Stack Integration Harness diff --git a/tests/conformance/bless-client-baseline-test.sh b/tests/conformance/bless-client-baseline-test.sh new file mode 100755 index 0000000..1cd3a19 --- /dev/null +++ b/tests/conformance/bless-client-baseline-test.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-client-baseline-test.XXXXXX")" +results_dir="${state_dir}/results" +baseline_file="${state_dir}/client-expected-failures.yml" + +cleanup() { + rm -rf -- "${state_dir}" +} +trap cleanup EXIT INT TERM + +mkdir -p \ + "${results_dir}/tools_call-2026-08-20T00-00-00-000Z" \ + "${results_dir}/request-metadata-2026-08-20T00-00-00-000Z" \ + "${results_dir}/http-standard-headers-2026-08-20T00-00-00-000Z" \ + "${results_dir}/http-custom-headers-2026-08-20T00-00-00-000Z" + +cat > "${baseline_file}" <<'EOF' +client: [] +EOF +for scenario in tools_call request-metadata http-standard-headers; do + cat > "${results_dir}/${scenario}-2026-08-20T00-00-00-000Z/checks.json" <<'EOF' +[{"id":"passes","status":"SUCCESS"}] +EOF +done +cat > "${results_dir}/http-custom-headers-2026-08-20T00-00-00-000Z/checks.json" <<'EOF' +[ + {"id":"mirrors","status":"SUCCESS"}, + {"id":"mirrors","status":"FAILURE"}, + {"id":"encodes","status":"WARNING"}, + {"id":"omits-null","status":"SUCCESS"} +] +EOF + +"${script_dir}/bless-client-baseline.sh" "${results_dir}" "${baseline_file}" + +cat > "${state_dir}/expected.yml" <<'EOF' +# Generated by `make conformance-bless` from scoped client findings. +# OAuth client scenarios are control-plane responsibilities and are not run here. +client: + - http-custom-headers:encodes + - http-custom-headers:mirrors +EOF +diff -u "${state_dir}/expected.yml" "${baseline_file}" + +second_output="$("${script_dir}/bless-client-baseline.sh" "${results_dir}" "${baseline_file}")" +grep --fixed-strings --quiet -- 'was already current' <<< "${second_output}" + +echo 'client conformance baseline tests passed' diff --git a/tests/conformance/bless-client-baseline.sh b/tests/conformance/bless-client-baseline.sh new file mode 100755 index 0000000..d27af6e --- /dev/null +++ b/tests/conformance/bless-client-baseline.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -gt 2 ]; then + echo "Usage: bless-client-baseline.sh [results-dir [baseline-file]]" >&2 + exit 2 +fi + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "${script_dir}/../.." && pwd)" +results_dir="${1:-${repo_root}/conformance-results/client}" +baseline_file="${2:-${script_dir}/client-expected-failures.yml}" +scenarios=(tools_call request-metadata http-standard-headers http-custom-headers) + +for command in awk cmp cp jq mktemp sed sort; do + if ! command -v "${command}" > /dev/null 2>&1; then + echo "Required command not found: ${command}" >&2 + exit 2 + fi +done +if [ ! -d "${results_dir}" ]; then + echo "No client conformance results directory: ${results_dir}" >&2 + exit 2 +fi +if [ ! -f "${baseline_file}" ]; then + echo "Client conformance baseline not found: ${baseline_file}" >&2 + exit 2 +fi + +state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-client-baseline.XXXXXX")" +findings="${state_dir}/findings.txt" +baseline_entries="${state_dir}/baseline-entries.txt" +candidate="${state_dir}/client-expected-failures.yml" + +cleanup() { + rm -f -- "${findings}" "${baseline_entries}" "${candidate}" + rmdir -- "${state_dir}" +} +trap cleanup EXIT INT TERM + +: > "${findings}" +for scenario in "${scenarios[@]}"; do + matches=("${results_dir}/${scenario}-"*/checks.json) + if [ ! -e "${matches[0]}" ]; then + echo "No client conformance results found for scenario: ${scenario}" >&2 + exit 2 + fi + checks_file="${matches[$((${#matches[@]} - 1))]}" + jq --raw-output --arg scenario "${scenario}" ' + def severity($status): + if $status == "FAILURE" then 3 + elif $status == "WARNING" then 2 + elif $status == "SUCCESS" then 1 + else 0 + end; + + reduce (.[] | select(.status != "INFO")) as $check + ({}; + ($check.id) as $id | + if .[$id] == null or severity($check.status) >= severity(.[$id].status) + then .[$id] = $check + else . + end) | + to_entries[] | + select(.value.status == "FAILURE" or .value.status == "WARNING") | + ($scenario + ":" + .key) + ' "${checks_file}" >> "${findings}" +done +LC_ALL=C sort -u -o "${findings}" "${findings}" + +awk ' + /^client:$/ { in_client = 1; next } + in_client && /^[^[:space:]]/ { exit } + in_client && /^[[:space:]]*-[[:space:]]+/ { + line = $0 + sub(/^[[:space:]]*-[[:space:]]+/, "", line) + sub(/[[:space:]]+#.*$/, "", line) + print line + } +' "${baseline_file}" | LC_ALL=C sort -u > "${baseline_entries}" + +{ + # shellcheck disable=SC2016 # Backticks are literal Markdown. + echo '# Generated by `make conformance-bless` from scoped client findings.' + echo '# OAuth client scenarios are control-plane responsibilities and are not run here.' + if [ -s "${findings}" ]; then + echo 'client:' + sed 's/^/ - /' "${findings}" + else + echo 'client: []' + fi +} > "${candidate}" + +if cmp --silent "${findings}" "${baseline_entries}"; then + echo "Client conformance baseline was already current: ${baseline_file}" +else + cp "${candidate}" "${baseline_file}" + echo "Client conformance baseline updated: ${baseline_file}" +fi diff --git a/tests/conformance/client-expected-failures.yml b/tests/conformance/client-expected-failures.yml new file mode 100644 index 0000000..288387c --- /dev/null +++ b/tests/conformance/client-expected-failures.yml @@ -0,0 +1,10 @@ +# Dataplane-owned upstream MCP client findings for the scoped client lane. +# OAuth scenarios are control-plane responsibilities and are not run here. +client: + # The upstream client does not yet mirror x-mcp-header tool arguments into + # Mcp-Param-* request headers. Keep the null/omission checks as required + # passes by baselining only the affected checks, not the whole scenario. + - http-custom-headers:sep-2243-client-supports-custom-headers + - http-custom-headers:sep-2243-client-mirrors-designated-params + - http-custom-headers:sep-2243-client-encode-values + - http-custom-headers:sep-2243-client-base64-unsafe diff --git a/tests/conformance/client-under-test-test.sh b/tests/conformance/client-under-test-test.sh new file mode 100755 index 0000000..3387f24 --- /dev/null +++ b/tests/conformance/client-under-test-test.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-client-adapter-test.XXXXXX")" +fake_bin="${state_dir}/bin" +docker_args="${state_dir}/docker-args" +curl_bodies="${state_dir}/curl-bodies" + +cleanup() { + rm -rf -- "${state_dir}" +} +trap cleanup EXIT INT TERM + +mkdir -p "${fake_bin}" +cat > "${fake_bin}/docker" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$*" > "${FAKE_DOCKER_ARGS}" +EOF +cat > "${fake_bin}/curl" <<'EOF' +#!/usr/bin/env bash +while [ "$#" -gt 0 ]; do + if [ "$1" = "--data" ]; then + shift + printf '%s\n' "$1" >> "${FAKE_CURL_BODIES}" + fi + shift +done +printf '%s\n' 'data: {"jsonrpc":"2.0","id":1,"result":{"content":[]}}' +EOF +chmod +x "${fake_bin}/docker" "${fake_bin}/curl" + +export PATH="${fake_bin}:${PATH}" +export FAKE_DOCKER_ARGS="${docker_args}" +export FAKE_CURL_BODIES="${curl_bodies}" +export MCP_CONFORMANCE_PROTOCOL_VERSION=2026-07-28 +export MCP_CONFORMANCE_SUBJECT=test-subject +export MCP_CONFORMANCE_CLIENT_SERVER_ID=test-client-server +export MCP_CONFORMANCE_PORT=18080 +export MCP_CONFORMANCE_SCENARIO=http-custom-headers +export MCP_CONFORMANCE_CONTEXT='{ + "name": "http-custom-headers", + "toolCalls": [ + {"name": "first", "arguments": {"region": "west"}}, + {"name": "second", "arguments": {"verbose": null}} + ] +}' + +"${script_dir}/client-under-test.sh" "http://localhost:43123/mcp" + +grep --fixed-strings --quiet -- 'http://host.docker.internal:43123/mcp' "${docker_args}" +grep --fixed-strings --quiet -- '["first","second"]' "${docker_args}" +test "$(wc -l < "${curl_bodies}" | tr -d '[:space:]')" -eq 2 +jq --exit-status --slurp ' + length == 2 and + .[0].method == "tools/call" and + .[0].params.name == "first" and + .[0].params.arguments.region == "west" and + .[0].params._meta["io.modelcontextprotocol/protocolVersion"] == "2026-07-28" and + .[1].params.name == "second" and + .[1].params.arguments.verbose == null +' "${curl_bodies}" > /dev/null + +echo 'client conformance adapter tests passed' diff --git a/tests/conformance/client-under-test.sh b/tests/conformance/client-under-test.sh new file mode 100755 index 0000000..909ee2f --- /dev/null +++ b/tests/conformance/client-under-test.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${MCP_CONFORMANCE_SCENARIO:?MCP_CONFORMANCE_SCENARIO must be set by the conformance runner}" +: "${MCP_CONFORMANCE_PROTOCOL_VERSION:?MCP_CONFORMANCE_PROTOCOL_VERSION must be set by the conformance runner}" +: "${MCP_CONFORMANCE_SUBJECT:?MCP_CONFORMANCE_SUBJECT must be set}" + +if [ "$#" -ne 1 ]; then + echo "Usage: client-under-test.sh " >&2 + exit 2 +fi +if [ "${MCP_CONFORMANCE_PROTOCOL_VERSION}" != "2026-07-28" ]; then + echo "Only MCP 2026-07-28 client conformance is supported" >&2 + exit 2 +fi + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +compose_file="${script_dir}/docker-compose.yml" +virtual_host_id="${MCP_CONFORMANCE_CLIENT_SERVER_ID:-dataplane-client-conformance}" +conformance_port="${MCP_CONFORMANCE_PORT:-8080}" +scenario_server_url="$1" +backend_url="${scenario_server_url/\/\/localhost:/\/\/host.docker.internal:}" +backend_url="${backend_url/\/\/127.0.0.1:/\/\/host.docker.internal:}" + +case "${MCP_CONFORMANCE_SCENARIO}" in + tools_call) + tool_calls='[{"name":"add_numbers","arguments":{"a":2,"b":3}}]' + ;; + request-metadata) + tool_calls='[{"name":"metadata_probe","arguments":{}}]' + ;; + http-standard-headers) + tool_calls='[{"name":"test_headers","arguments":{}}]' + ;; + http-custom-headers) + : "${MCP_CONFORMANCE_CONTEXT:?MCP_CONFORMANCE_CONTEXT is required for http-custom-headers}" + tool_calls="$(jq --exit-status --compact-output '.toolCalls' <<< "${MCP_CONFORMANCE_CONTEXT}")" + ;; + *) + echo "Unsupported dataplane client conformance scenario: ${MCP_CONFORMANCE_SCENARIO}" >&2 + exit 2 + ;; +esac + +tool_names="$(jq --exit-status --compact-output '[.[].name] | unique' <<< "${tool_calls}")" +docker compose -f "${compose_file}" run --rm --no-deps \ + --entrypoint python3 control-plane \ + /opt/contextforge-conformance/write_client_config.py \ + "${MCP_CONFORMANCE_SUBJECT}" \ + "${virtual_host_id}" \ + "${backend_url}" \ + "${tool_names}" \ + > /dev/null + +endpoint="http://127.0.0.1:${conformance_port}/servers/${virtual_host_id}/mcp" +while IFS= read -r tool_call; do + tool_name="$(jq --exit-status --raw-output '.name' <<< "${tool_call}")" + arguments="$(jq --exit-status --compact-output '.arguments' <<< "${tool_call}")" + request="$(jq --null-input --compact-output \ + --arg name "${tool_name}" \ + --argjson arguments "${arguments}" \ + --arg version "${MCP_CONFORMANCE_PROTOCOL_VERSION}" \ + '{ + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { + name: $name, + arguments: $arguments, + _meta: { + "io.modelcontextprotocol/protocolVersion": $version, + "io.modelcontextprotocol/clientInfo": { + name: "dataplane-client-conformance-driver", + version: "1.0.0" + }, + "io.modelcontextprotocol/clientCapabilities": {} + } + } + }')" + + response="$(curl --silent --show-error --fail-with-body \ + --request POST \ + --header 'Content-Type: application/json' \ + --header 'Accept: application/json, text/event-stream' \ + --header "MCP-Protocol-Version: ${MCP_CONFORMANCE_PROTOCOL_VERSION}" \ + --header 'MCP-Method: tools/call' \ + --header "MCP-Name: ${tool_name}" \ + --data "${request}" \ + "${endpoint}")" + + response_json="$(sed -n 's/^data: //p' <<< "${response}" | head -n 1)" + if [ -z "${response_json}" ]; then + response_json="${response}" + fi + if ! jq --exit-status '.result != null and .error == null' <<< "${response_json}" > /dev/null; then + echo "Dataplane rejected client conformance tool call ${tool_name}:" >&2 + echo "${response}" >&2 + exit 1 + fi +done < <(jq --compact-output '.[]' <<< "${tool_calls}") diff --git a/tests/conformance/docker-compose.yml b/tests/conformance/docker-compose.yml index 21e978d..d16e8fc 100644 --- a/tests/conformance/docker-compose.yml +++ b/tests/conformance/docker-compose.yml @@ -70,6 +70,7 @@ services: LOG_LEVEL: INFO volumes: - ../../assets:/keys:ro + - ./write_client_config.py:/opt/contextforge-conformance/write_client_config.py:ro depends_on: redis: condition: service_healthy @@ -83,6 +84,8 @@ services: data-plane: image: ${CF_DATAPLANE_IMAGE:-ghcr.io/contextforge-org/contextforge-data-plane:latest} networks: [contextforge] + extra_hosts: + - host.docker.internal:host-gateway environment: CONTEXTFORGE_DATA_PLANE_ADDRESS: 0.0.0.0:4445 CONTEXTFORGE_DATA_PLANE_REDIS_HOSTNAME: redis diff --git a/tests/conformance/register-fixture.sh b/tests/conformance/register-fixture.sh index 2919276..dca91dc 100755 --- a/tests/conformance/register-fixture.sh +++ b/tests/conformance/register-fixture.sh @@ -109,4 +109,15 @@ conformance_token="$(jq --exit-status --raw-output '.access_token' <<< "${token_ if [ "${GITHUB_ACTIONS:-}" = "true" ]; then echo "::add-mask::${conformance_token}" fi + +jwt_payload="$(cut -d. -f2 <<< "${conformance_token}")" +jwt_payload="${jwt_payload//-/+}" +jwt_payload="${jwt_payload//_/\/}" +case $((${#jwt_payload} % 4)) in + 2) jwt_payload="${jwt_payload}==" ;; + 3) jwt_payload="${jwt_payload}=" ;; +esac +conformance_subject="$(jq --raw-input --exit-status --raw-output '@base64d | fromjson | .sub' <<< "${jwt_payload}")" + echo "MCP_CONFORMANCE_TOKEN=${conformance_token}" >> "${GITHUB_ENV}" +echo "MCP_CONFORMANCE_SUBJECT=${conformance_subject}" >> "${GITHUB_ENV}" diff --git a/tests/conformance/run-client-conformance.sh b/tests/conformance/run-client-conformance.sh new file mode 100755 index 0000000..289c2d4 --- /dev/null +++ b/tests/conformance/run-client-conformance.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${MCP_CONFORMANCE_SPEC_VERSION:?MCP_CONFORMANCE_SPEC_VERSION must be set}" +: "${MCP_CONFORMANCE_SUBJECT:?MCP_CONFORMANCE_SUBJECT must be set}" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "${script_dir}/../.." && pwd)" +suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" +results_root="${MCP_CONFORMANCE_RESULTS_DIR:-${repo_root}/conformance-results}" +results_dir="${MCP_CONFORMANCE_CLIENT_RESULTS_DIR:-${results_root}/client}" +compose_file="${script_dir}/docker-compose.yml" +baseline_file="${script_dir}/client-expected-failures.yml" +client_command="${script_dir}/client-under-test.sh" +scenarios=(tools_call request-metadata http-standard-headers http-custom-headers) + +mkdir -p "${results_dir}" + +# The control-plane publisher would overwrite the scenario-specific Redis +# config, and its own backend probes would contaminate the client observations. +docker compose -f "${compose_file}" stop control-plane > /dev/null + +status=0 +for scenario in "${scenarios[@]}"; do + set +e + ( + cd "${suite_dir}" + npm start -- \ + client \ + --command "${client_command}" \ + --scenario "${scenario}" \ + --spec-version "${MCP_CONFORMANCE_SPEC_VERSION}" \ + --expected-failures "${baseline_file}" \ + --timeout 60000 \ + --output-dir "${results_dir}" + ) + scenario_status="$?" + set -e + if [ "${scenario_status}" -ne 0 ]; then + status=1 + fi +done + +exit "${status}" diff --git a/tests/conformance/run-conformance.sh b/tests/conformance/run-conformance.sh index 1727a03..93a8f74 100755 --- a/tests/conformance/run-conformance.sh +++ b/tests/conformance/run-conformance.sh @@ -9,7 +9,10 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd -- "${script_dir}/../.." && pwd)" suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" conformance_port="${MCP_CONFORMANCE_PORT:-8080}" -results_dir="${MCP_CONFORMANCE_RESULTS_DIR:-${repo_root}/conformance-results}" +results_root="${MCP_CONFORMANCE_RESULTS_DIR:-${repo_root}/conformance-results}" +results_dir="${MCP_CONFORMANCE_SERVER_RESULTS_DIR:-${results_root}/server}" + +mkdir -p "${results_dir}" set +e ( diff --git a/tests/conformance/run-local.sh b/tests/conformance/run-local.sh index 72bb5e1..c167402 100755 --- a/tests/conformance/run-local.sh +++ b/tests/conformance/run-local.sh @@ -97,6 +97,12 @@ echo "Starting the dataplane and nginx." echo "Running MCP ${MCP_CONFORMANCE_SPEC_VERSION} conformance." "${script_dir}/run-conformance.sh" +set +e +echo "Running scoped MCP ${MCP_CONFORMANCE_SPEC_VERSION} client conformance." +"${script_dir}/run-client-conformance.sh" +client_status="$?" +set -e + runner_status="$(sed -n 's/^status=//p' "${GITHUB_OUTPUT}" | tail -n 1)" if [ -z "${runner_status}" ]; then echo "Conformance runner did not report a status." >&2 @@ -105,14 +111,29 @@ fi set +e if [ "${MCP_CONFORMANCE_BLESS:-false}" = "true" ]; then - "${script_dir}/report-baseline-diff.sh" --bless "${MCP_CONFORMANCE_RESULTS_DIR}" + "${script_dir}/report-baseline-diff.sh" --bless "${MCP_CONFORMANCE_RESULTS_DIR}/server" + report_status="$?" + "${script_dir}/bless-client-baseline.sh" \ + "${MCP_CONFORMANCE_RESULTS_DIR}/client" \ + "${script_dir}/client-expected-failures.yml" + client_baseline_status="$?" + if [ "${client_baseline_status}" -eq 0 ]; then + client_status=0 + fi else - "${script_dir}/report-baseline-diff.sh" "${MCP_CONFORMANCE_RESULTS_DIR}" + "${script_dir}/report-baseline-diff.sh" "${MCP_CONFORMANCE_RESULTS_DIR}/server" + report_status="$?" + client_baseline_status=0 fi -report_status="$?" set -e if [ "${runner_status}" -ne 0 ] && [ "${report_status}" -eq 0 ]; then echo "Official runner status ${runner_status} contained no dataplane baseline mismatch." fi +if [ "${client_baseline_status}" -ne 0 ]; then + exit "${client_baseline_status}" +fi +if [ "${client_status}" -ne 0 ]; then + exit "${client_status}" +fi exit "${report_status}" diff --git a/tests/conformance/write_client_config.py b/tests/conformance/write_client_config.py new file mode 100755 index 0000000..5d67a7f --- /dev/null +++ b/tests/conformance/write_client_config.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Publish an isolated upstream-client conformance route for the dataplane.""" + +from __future__ import annotations + +import argparse +import json +import os +from urllib.parse import urlparse + +import msgpack +import redis + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("subject") + parser.add_argument("virtual_host_id") + parser.add_argument("backend_url") + parser.add_argument("tool_names_json") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + redis_url = os.environ.get("REDIS_URL") + if not redis_url: + raise SystemExit("REDIS_URL is required") + + parsed_url = urlparse(args.backend_url) + if parsed_url.scheme not in {"http", "https"} or not parsed_url.hostname: + raise SystemExit("backend_url must be an absolute HTTP(S) URL") + + tool_names = json.loads(args.tool_names_json) + if ( + not isinstance(tool_names, list) + or not tool_names + or not all(isinstance(name, str) and name for name in tool_names) + ): + raise SystemExit("tool_names_json must be a non-empty JSON string array") + + backend_name = "conformance-backend" + config = { + "virtual_hosts": { + args.virtual_host_id: { + "backends": { + backend_name: { + "name": backend_name, + "url": args.backend_url, + "passthrough_headers": [], + "add_headers": {}, + "remove_headers": [], + "allowed_tool_names": tool_names, + "tool_name_aliases": {}, + "allowed_resource_names": [], + "allowed_prompt_names": [], + } + } + } + } + } + + key = msgpack.dumps(("UserConfig", args.subject), use_bin_type=True) + value = msgpack.dumps(config, use_bin_type=True) + client = redis.Redis.from_url(redis_url, decode_responses=False) + client.set(key, value, ex=600) + + +if __name__ == "__main__": + main()