[#35] AI 디버깅을 위한 consumer workflow artifact를 구성한다

.github/workflows/merge-risk-watch.yml

@@ -26,6 +26,11 @@ on:
         required: false
         type: string
         default: ""
+      upload_debug_artifact:
+        description: "AI prediction 원인 추적용 debug artifact를 consumer workflow run에 업로드할지 여부"
+        required: false
+        type: boolean
+        default: false
     secrets:
       watcher_github_token:
         description: "감시 대상 repository checkout, fetch, metadata 조회에 사용할 token"
@@ -61,6 +66,11 @@ on:
         required: false
         type: string
         default: ""
+      upload_debug_artifact:
+        description: "AI prediction 원인 추적용 debug artifact를 업로드할지 여부"
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -152,7 +162,17 @@ jobs:
           WATCHER_REPOSITORY: ${{ inputs.repository }}
           WATCHER_GITHUB_API_URL: ${{ github.api_url }}
           WATCHER_REPOSITORY_PATH: ${{ github.workspace }}/watched-repository
+          WATCHER_WORKFLOW_REF: ${{ job.workflow_ref }}
+          WATCHER_DEBUG_ARTIFACT_DIR: ${{ inputs.upload_debug_artifact && format('{0}/watcher/debug', github.workspace) || '' }}
           WATCHER_BASE_BRANCH: ${{ inputs.base_branch }}
           WATCHER_DEFAULT_BRANCH: ${{ inputs.default_branch }}
           WATCHER_CRITICAL_FILE_PATTERNS: ${{ inputs.critical_file_patterns }}
         run: npm run watch
+
+      - name: Upload Watcher debug artifact
+        if: inputs.upload_debug_artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: watcher-debug
+          path: ${{ github.workspace }}/watcher/debug
+          retention-days: 7

README.md

@@ -46,6 +46,7 @@ reusable workflow는 다음 input을 받습니다.
 | `default_branch` | 선택 | 빈 값 | 감시 대상에서 제외할 default branch |
 | `critical_file_patterns` | 선택 | 빈 값 | score에 반영할 critical file wildcard pattern 목록. 줄바꿈으로 구분 |
 | `watcher_version` | 선택 | 빈 값 | 수동 테스트에 사용할 Watcher release tag. 비워두면 workflow ref 기준 |
+| `upload_debug_artifact` | 선택 | `false` | AI prediction 원인 추적용 debug artifact를 consumer workflow run에 업로드할지 여부 |
 
 `critical_file_patterns`에서 `*`는 단일 path segment 내부를 매칭하고 `**`는 path separator를 포함해 매칭합니다.
 
@@ -123,6 +124,33 @@ AI provider가 실패해도 deterministic possibility report는 유지됩니다.
 
 Report에는 `Low` section, AI prediction `skipped` 상태, branch `updated` 시각, provider error 요약을 유지합니다. Pull Request metadata는 내부 evidence로만 사용할 수 있으며 Markdown report에는 출력하지 않습니다.
 
+## Debug artifact
+
+AI prediction 입력이나 provider 응답을 확인해야 할 때 consumer workflow에서 `upload_debug_artifact`를 `true`로 설정합니다.
+
+```yaml
+with:
+  upload_debug_artifact: true
+```
+
+이 옵션을 켜면 consumer repository의 해당 GitHub Actions run에 `watcher-debug` artifact가 업로드됩니다. Watcher repository가 아니라 reusable workflow를 호출한 consumer repository의 Actions 화면에서 다운로드합니다.
+
+artifact에는 다음 파일이 포함됩니다.
+
+| 파일 | 내용 |
+| --- | --- |
+| `run.json` | repository, base branch, default branch, critical file patterns, Watcher workflow ref |
+| `branch-selection.json` | 수집된 branch, 감시 대상 branch, 제외된 branch와 사유 |
+| `deterministic-evidence.json` | git merge signal, changed files, changed hunks, check/PR metadata, deterministic risk 결과 |
+| `ai-target-selection.json` | AI 호출 대상 branch와 skipped branch 사유 |
+| `ai-prompt.json` | OpenAI에 전달한 system prompt, user prompt, response shape |
+| `ai-response.json` | provider가 반환한 raw response |
+| `ai-error.json` | provider 호출 또는 response validation 실패 요약. 실패가 없으면 생성되지 않을 수 있음 |
+| `ai-result.json` | response validation 이후 branch별 AI prediction, skipped, failed 매핑 결과 |
+| `report.md` | 최종 Markdown report |
+
+debug artifact에는 `GITHUB_TOKEN`, `WATCHER_GITHUB_TOKEN`, `OPENAI_API_KEY`, `DISCORD_WEBHOOK_URL`을 기록하지 않습니다. raw file content와 raw diff 전문도 포함하지 않습니다.
+
 ## Report channel
 
 Merge risk report는 Markdown으로 생성됩니다. consumer repository에 `DISCORD_WEBHOOK_URL` secret이 있으면 Discord webhook으로 전송하고, 없으면 stdout으로 출력합니다.
@@ -150,6 +178,7 @@ WATCHER_BASE_BRANCH=develop \
 WATCHER_DEFAULT_BRANCH=main \
 WATCHER_CRITICAL_FILE_PATTERNS='package-lock.json
 .github/workflows/**' \
+WATCHER_DEBUG_ARTIFACT_DIR=/tmp/watcher-debug \
 GITHUB_TOKEN=github-token \
 OPENAI_API_KEY=openai-api-key \
 DISCORD_WEBHOOK_URL=discord-webhook-url \
@@ -176,6 +205,7 @@ npm test
 | `default_branch` | 제외할 default branch. 예: `main` |
 | `watcher_version` | 테스트할 Watcher release tag. 비워두면 workflow ref 기준 |
 | `critical_file_patterns` | 테스트할 critical file pattern. 예: `package-lock.json`, `.github/workflows/**` |
+| `upload_debug_artifact` | 문제 원인 추적이 필요할 때만 `true` |
 
 3. consumer repository secret을 설정합니다.
 

docs/examples/consumer-merge-risk-watch.yml

@@ -10,6 +10,11 @@ on:
         required: false
         type: string
         default: ""
+      upload_debug_artifact:
+        description: "AI prediction 원인 추적용 debug artifact를 업로드할지 여부"
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -25,6 +30,7 @@ jobs:
       base_branch: develop
       default_branch: main
       watcher_version: ${{ inputs.watcher_version }}
+      upload_debug_artifact: ${{ inputs.upload_debug_artifact || false }}
       critical_file_patterns: |
         package-lock.json
         .github/workflows/**

src/ai/predictionRunner.ts

@@ -3,6 +3,8 @@ import { select as selectTargets } from "./predictionTargetSelector.js"
 import { validateBatch as validateBatchResponse } from "./predictionResponseValidator.js"
 import type {
   AiPrediction,
+  AiPredictionDebugObserver,
+  AiPredictionDebugTarget,
   AiPredictionClient,
   AiPredictionEvidencePayload,
   AiPredictionResult,
@@ -36,20 +38,65 @@ async function predictedResultsFor(
   client: AiPredictionClient,
   options: AiPredictionRunOptions
 ): Promise<Map<AiPredictionEvidencePayload, AiPredictionResult>> {
+  const targetBranches = targets.map(debugTargetFor)
+  const prompt = buildBatchPrompt(targets, options)
+  await notifyDebugObserver("onPromptBuilt", () => options.debugObserver?.onPromptBuilt?.({
+    targetBranches,
+    prompt
+  }))
+
+  let response: unknown
+
+  try {
+    response = await client.predict(prompt)
+  } catch (error) {
+    const errorMessage = errorMessageFor(error)
+    await notifyDebugObserver("onPredictionFailed", () => options.debugObserver?.onPredictionFailed?.({
+      targetBranches,
+      errorMessage
+    }))
+
+    return new Map(targets.map(target => [
+      target,
+      failedResultFor(target, errorMessage)
+    ]))
+  }
+
+  await notifyDebugObserver("onResponseReceived", () => options.debugObserver?.onResponseReceived?.({
+    targetBranches,
+    response
+  }))
+
   try {
-    const prompt = buildBatchPrompt(targets, options)
-    const response = await client.predict(prompt)
     const predictions = validateBatchResponse(response)
 
     return resultsByPayloadFor(targets, predictions)
   } catch (error) {
+    const errorMessage = errorMessageFor(error)
+    await notifyDebugObserver("onPredictionFailed", () => options.debugObserver?.onPredictionFailed?.({
+      targetBranches,
+      errorMessage
+    }))
+
     return new Map(targets.map(target => [
       target,
-      failedResultFor(target, errorMessageFor(error))
+      failedResultFor(target, errorMessage)
     ]))
   }
 }
 
+// debug observer 실패가 prediction 흐름을 중단하지 않도록 격리
+async function notifyDebugObserver(
+  eventName: keyof AiPredictionDebugObserver,
+  action: () => Promise<void> | void | undefined
+): Promise<void> {
+  try {
+    await action()
+  } catch (error) {
+    console.warn(`Failed to notify debug observer (${eventName}): ${errorMessageFor(error)}`)
+  }
+}
+
 // unknown error를 report 가능한 문자열로 변환
 function errorMessageFor(error: unknown): string {
   return error instanceof Error ? error.message : String(error)
@@ -122,3 +169,10 @@ function predictionKeyFor(
 ): string {
   return `${baseBranch}\u0000${branchName}`
 }
+
+function debugTargetFor(payload: AiPredictionEvidencePayload): AiPredictionDebugTarget {
+  return {
+    branchName: payload.branch.name,
+    baseBranch: payload.branch.baseBranch
+  }
+}

src/ai/types.ts

@@ -32,8 +32,36 @@ export type AiPredictionClient = {
   predict(prompt: AiPredictionPrompt): Promise<unknown>
 }
 
-// AI prediction runner가 prompt 생성을 조정하기 위한 설정
-export type AiPredictionRunOptions = AiPredictionPromptBuildOptions
+export type AiPredictionDebugTarget = {
+  branchName: string
+  baseBranch: string
+}
+
+export type AiPredictionPromptDebugEvent = {
+  targetBranches: AiPredictionDebugTarget[]
+  prompt: AiPredictionPrompt
+}
+
+export type AiPredictionResponseDebugEvent = {
+  targetBranches: AiPredictionDebugTarget[]
+  response: unknown
+}
+
+export type AiPredictionFailureDebugEvent = {
+  targetBranches: AiPredictionDebugTarget[]
+  errorMessage: string
+}
+
+export type AiPredictionDebugObserver = {
+  onPromptBuilt?(event: AiPredictionPromptDebugEvent): void | Promise<void>
+  onResponseReceived?(event: AiPredictionResponseDebugEvent): void | Promise<void>
+  onPredictionFailed?(event: AiPredictionFailureDebugEvent): void | Promise<void>
+}
+
+// AI prediction runner가 prompt 생성과 debug 기록을 조정하기 위한 설정
+export type AiPredictionRunOptions = AiPredictionPromptBuildOptions & {
+  debugObserver?: AiPredictionDebugObserver
+}
 
 // branch별 AI prediction 실행 결과
 export type AiPredictionResult =

src/branches/branchSelector.ts

@@ -1,34 +1,74 @@
 import type {
   BranchContext,
+  BranchExclusionReason,
+  BranchSelectionResult,
   BranchSelectionOptions,
+  ExcludedBranch,
   RepositoryBranch
 } from "./types.js"
 
 export function select(
   branches: RepositoryBranch[],
   options: BranchSelectionOptions
 ): BranchContext[] {
-  return branches
-    .filter(branch => isWatchableBranch(branch, options))
-    .map(branch => ({
-      baseBranch: options.baseBranch,
-      name: branch.name,
-      headSha: branch.sha,
-      author: branch.author,
-      updatedAt: branch.updatedAt,
-      checks: branch.checks ?? [],
-      pullRequest: branch.pullRequest
-    }))
+  return selectWithReasons(branches, options).selected
 }
 
-function isWatchableBranch(
+export function selectWithReasons(
+  branches: RepositoryBranch[],
+  options: BranchSelectionOptions
+): BranchSelectionResult {
+  const selected: BranchContext[] = []
+  const excluded: ExcludedBranch[] = []
+
+  for (const branch of branches) {
+    const reason = exclusionReasonFor(branch, options)
+
+    if (reason) {
+      excluded.push({
+        name: branch.name,
+        sha: branch.sha,
+        reason
+      })
+      continue
+    }
+
+    selected.push(branchContextFor(branch, options))
+  }
+
+  return {
+    selected,
+    excluded
+  }
+}
+
+function branchContextFor(
+  branch: RepositoryBranch,
+  options: BranchSelectionOptions
+): BranchContext {
+  return {
+    baseBranch: options.baseBranch,
+    name: branch.name,
+    headSha: branch.sha,
+    author: branch.author,
+    updatedAt: branch.updatedAt,
+    checks: branch.checks ?? [],
+    pullRequest: branch.pullRequest
+  }
+}
+
+function exclusionReasonFor(
   branch: RepositoryBranch,
   options: BranchSelectionOptions
-): boolean {
+): BranchExclusionReason | undefined {
   // base, default branch는 비교 기준이므로 감시 대상에서 제외
-  if (branch.name === options.baseBranch || branch.name === options.defaultBranch) {
-    return false
+  if (branch.name === options.baseBranch) {
+    return "base_branch"
+  }
+
+  if (branch.name === options.defaultBranch) {
+    return "default_branch"
   }
 
-  return true
+  return undefined
 }

src/branches/types.ts

@@ -25,6 +25,21 @@ export type BranchSelectionOptions = {
   defaultBranch?: string
 }
 
+export type BranchExclusionReason =
+  | "base_branch"
+  | "default_branch"
+
+export type ExcludedBranch = {
+  name: string
+  sha: string
+  reason: BranchExclusionReason
+}
+
+export type BranchSelectionResult = {
+  selected: BranchContext[]
+  excluded: ExcludedBranch[]
+}
+
 export type BranchContext = {
   baseBranch: string
   name: string