diff --git a/.github/workflows/go-logger.lock.yml b/.github/workflows/go-logger.lock.yml index 6b8293d5fd1..7c2b774e79e 100644 --- a/.github/workflows/go-logger.lock.yml +++ b/.github/workflows/go-logger.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"79904d3a21bbbbfc744aa46adafa352cf5009c2a59114ae4ba2caa748627bb20","body_hash":"92fd878ea7c5e85e8ccdeb0628252d4edfb9f12f44b08735e8371627a29d9348","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.227"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"79904d3a21bbbbfc744aa46adafa352cf5009c2a59114ae4ba2caa748627bb20","body_hash":"48f74e30c3942fbadca47b81891d3a38e221bd65cbe8b0ef9e4435ef0c7d95b0","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.227"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.1","digest":"sha256:5e3f6ee27eeae07195838b97ac4aa2f8aea42a7c55f1c0d3e17d8e88e294ad0d","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.1@sha256:5e3f6ee27eeae07195838b97ac4aa2f8aea42a7c55f1c0d3e17d8e88e294ad0d"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.1","digest":"sha256:288e7d2a12d5b430500d739f9c16e20bb1ed51b91f986f3f3eccde189f489f5c","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.1@sha256:288e7d2a12d5b430500d739f9c16e20bb1ed51b91f986f3f3eccde189f489f5c"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.1","digest":"sha256:f931e5e1e13f765605d03ef9511fc755d779a51b76581ea14586e9871506a610","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.1@sha256:f931e5e1e13f765605d03ef9511fc755d779a51b76581ea14586e9871506a610"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.1","digest":"sha256:9d428af47899bf18ef2d5618075777d76ef344c91e76c1f44ec1aaa0ee347e5f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.1@sha256:9d428af47899bf18ef2d5618075777d76ef344c91e76c1f44ec1aaa0ee347e5f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/go-logger.md b/.github/workflows/go-logger.md index 91c3ec6963c..c67a9fc607a 100644 --- a/.github/workflows/go-logger.md +++ b/.github/workflows/go-logger.md @@ -244,9 +244,11 @@ After adding logging to **all selected files**, validate your changes before cre After validating your changes: -1. The safe-outputs create-pull-request will automatically create a PR -2. Ensure your changes follow the guidelines above -3. The PR title will automatically have the "[log] " prefix +1. Choose exactly one terminal outcome: `create_pull_request` after successful changes, `noop` when no changes are needed, or `report_incomplete` when a blocking failure prevents completion. +2. Call the chosen safe-output command exactly once, as your final action. Do not call any other safe-output command before or after it. +3. Do not probe safe outputs with `which`, `type`, `--help`, or schema-inspection commands. +4. If the safe-output gateway rejects the call, stop immediately and surface its exact rejection message. Do not retry the call or switch to another terminal safe output. +5. The PR title will automatically have the "[log] " prefix. ## Quality Checklist diff --git a/pkg/cli/audit_mcp_tool_usage_test.go b/pkg/cli/audit_mcp_tool_usage_test.go index a22c5127267..aa706465428 100644 --- a/pkg/cli/audit_mcp_tool_usage_test.go +++ b/pkg/cli/audit_mcp_tool_usage_test.go @@ -56,6 +56,17 @@ func TestExtractMCPToolUsageData(t *testing.T) { wantToolCalls: 2, wantErr: false, }, + { + name: "tool discovery is not tool usage", + // Discovery traffic identifies the contacted server but does not constitute tool usage. + logContent: `{"timestamp":"2024-01-12T10:00:00Z","level":"info","type":"request","event":"rpc_call","server_name":"safeoutputs","method":"tools/list","duration":50.0,"status":"success"} +{"timestamp":"2024-01-12T10:00:01Z","level":"info","type":"request","event":"request","server_name":"safeoutputs","method":"tools/list","duration":50.0,"status":"success"} +`, + wantServers: 1, + wantTools: 0, + wantToolCalls: 0, + wantErr: false, + }, { name: "no gateway.jsonl file", logContent: "", diff --git a/pkg/cli/audit_report.go b/pkg/cli/audit_report.go index 30b8ecaec0a..bcb2e4184df 100644 --- a/pkg/cli/audit_report.go +++ b/pkg/cli/audit_report.go @@ -845,7 +845,7 @@ func extractGHErrorLines(filePath string) []string { for line := range strings.SplitSeq(string(content), "\n") { if strings.Contains(line, "##[error]") { stripped := stripGHALogTimestamps(line) - if stripped != "" { + if stripped != "" && !isAgentToolResultAnnotation(stripped) { errorLines = append(errorLines, stripped) } } @@ -854,6 +854,33 @@ func extractGHErrorLines(filePath string) []string { return errorLines } +func isAgentToolResultAnnotation(line string) bool { + _, payload, found := strings.Cut(line, "##[error]") + if !found { + return false + } + + var event struct { + Type string `json:"type"` + Message struct { + Content []struct { + Type string `json:"type"` + } `json:"content"` + } `json:"message"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(payload)), &event); err != nil || event.Type != "user" { + return false + } + hasToolResult := false + for _, content := range event.Message.Content { + if content.Type != "tool_result" { + return false + } + hasToolResult = true + } + return hasToolResult +} + func extractAgentFailureError(agentRan bool, agentStdioPath string, maxMessageLen int) []ValidationIssue { if !agentRan { return nil diff --git a/pkg/cli/audit_report_test.go b/pkg/cli/audit_report_test.go index eab7de2be91..6671ce58475 100644 --- a/pkg/cli/audit_report_test.go +++ b/pkg/cli/audit_report_test.go @@ -1448,6 +1448,35 @@ func TestExtractPreAgentStepErrors(t *testing.T) { assert.Contains(t, errors[0].Message, "Lockdown mode is enabled", "Should include actionable ##[error] text") }) + t.Run("ignores annotated tool results and surfaces the runner failure", func(t *testing.T) { + dir := testutil.TempDir(t, "audit-step-*") + require.NoError(t, os.WriteFile(filepath.Join(dir, "agent-stdio.log"), []byte("agent output"), 0600)) + workflowLogsDir := filepath.Join(dir, "workflow-logs", "agent") + require.NoError(t, os.MkdirAll(workflowLogsDir, 0755)) + toolResult := `{"type":"user","message":{"content":[{"type":"tool_result","content":"raw Go source"}]}}` + logContent := "2026-08-13T04:03:11Z ##[error]" + toolResult + "\n" + + "2026-08-13T04:11:07Z ##[error]The action 'Execute Claude Code CLI' has timed out after 15 minutes." + require.NoError(t, os.WriteFile(filepath.Join(workflowLogsDir, "10_Execute Claude Code CLI.txt"), []byte(logContent), 0600)) + + errors := extractPreAgentStepErrors(dir) + require.Len(t, errors, 1) + assert.Contains(t, errors[0].Message, "timed out after 15 minutes") + assert.NotContains(t, errors[0].Message, "raw Go source") + }) + + t.Run("preserves mixed tool result annotations", func(t *testing.T) { + dir := testutil.TempDir(t, "audit-step-*") + workflowLogsDir := filepath.Join(dir, "workflow-logs", "agent") + require.NoError(t, os.MkdirAll(workflowLogsDir, 0755)) + mixedContent := `{"type":"user","message":{"content":[{"type":"tool_result","content":"raw Go source"},{"type":"text","text":"runner failure"}]}}` + logContent := "2026-08-13T04:03:11Z ##[error]" + mixedContent + require.NoError(t, os.WriteFile(filepath.Join(workflowLogsDir, "10_Execute Claude Code CLI.txt"), []byte(logContent), 0600)) + + errors := extractPreAgentStepErrors(dir) + require.Len(t, errors, 1) + assert.Contains(t, errors[0].Message, "runner failure") + }) + t.Run("returns nil when workflow-logs directory missing", func(t *testing.T) { dir := testutil.TempDir(t, "audit-step-*") // No agent-stdio.log and no workflow-logs directory diff --git a/pkg/cli/gateway_logs_mcp.go b/pkg/cli/gateway_logs_mcp.go index 2dc6f3620b9..b4bb355cefb 100644 --- a/pkg/cli/gateway_logs_mcp.go +++ b/pkg/cli/gateway_logs_mcp.go @@ -123,8 +123,8 @@ func extractToolCallsFromGatewayLog(gatewayLogPath string, mcpData *MCPToolUsage continue // Skip malformed lines } - // Only process tool call events - if entry.Event == "tool_call" || entry.Event == "rpc_call" || entry.Event == "request" { + // Only process actual tool invocations, not protocol requests such as tools/list. + if entry.Event == "tool_call" || entry.Method == "tools/call" { toolName := entry.ToolName if toolName == "" { toolName = entry.Method diff --git a/pkg/cli/gateway_logs_parsing.go b/pkg/cli/gateway_logs_parsing.go index f96b1e7b311..55664b82c80 100644 --- a/pkg/cli/gateway_logs_parsing.go +++ b/pkg/cli/gateway_logs_parsing.go @@ -190,8 +190,8 @@ func processGatewayLogEntry(entry *GatewayLogEntry, metrics *GatewayMetrics, ver metrics.TotalDuration += entry.Duration } - // Track tool calls - if entry.ToolName != "" || entry.Method != "" { + // Track only actual tool invocations, not protocol requests such as tools/list. + if entry.Event == "tool_call" || entry.Method == "tools/call" { toolName := entry.ToolName if toolName == "" { toolName = entry.Method diff --git a/pkg/cli/gateway_logs_test.go b/pkg/cli/gateway_logs_test.go index 7e2142b050f..4c67f9b4c63 100644 --- a/pkg/cli/gateway_logs_test.go +++ b/pkg/cli/gateway_logs_test.go @@ -53,7 +53,7 @@ func TestParseGatewayLogs(t *testing.T) { `, wantServers: 3, wantRequests: 3, - wantToolCalls: 3, + wantToolCalls: 0, wantErrors: 0, wantErr: false, }, @@ -447,14 +447,14 @@ func TestGatewayLogsWithMethodField(t *testing.T) { assert.Len(t, metrics.Servers, 1) assert.Equal(t, 2, metrics.TotalRequests) - assert.Equal(t, 2, metrics.TotalToolCalls) + assert.Equal(t, 1, metrics.TotalToolCalls) server := metrics.Servers["github"] require.NotNil(t, server) - assert.Len(t, server.Tools, 2) + assert.Len(t, server.Tools, 1) - // Check that methods were tracked as tools - assert.Contains(t, server.Tools, "tools/list") + // Protocol discovery remains a request but is not counted as tool usage. + assert.NotContains(t, server.Tools, "tools/list") assert.Contains(t, server.Tools, "tools/call") } diff --git a/pkg/workflow/prompts_test.go b/pkg/workflow/prompts_test.go index 62262da35cf..1c81d1792a5 100644 --- a/pkg/workflow/prompts_test.go +++ b/pkg/workflow/prompts_test.go @@ -276,6 +276,34 @@ func TestDailyFunctionNamerUsesConcreteClaudeModelsForExperiment(t *testing.T) { } } +func TestGoLoggerDefinesSingleTerminalSafeOutputContract(t *testing.T) { + repoRoot, err := findRepoRoot() + if err != nil { + t.Fatalf("Failed to find repo root: %v", err) + } + + workflowFile := filepath.Join(repoRoot, ".github", "workflows", "go-logger.md") + content, err := os.ReadFile(workflowFile) + if err != nil { + t.Fatalf("Failed to read workflow file: %v", err) + } + + workflow := string(content) + for _, keyword := range []string{ + "exactly one terminal outcome", + "`create_pull_request`", + "`noop`", + "`report_incomplete`", + "exactly once, as your final action", + "Do not probe safe outputs", + "Do not retry the call or switch to another terminal safe output", + } { + if !strings.Contains(workflow, keyword) { + t.Fatalf("Expected go-logger workflow to include safe-output contract keyword %q", keyword) + } + } +} + func TestDailyCavemanOptimizerUsesConcreteClaudeModelsForExperiment(t *testing.T) { repoRoot, err := findRepoRoot() if err != nil {