From 4d96a591402c34630f0904b3d98b39eba24b7a0b Mon Sep 17 00:00:00 2001 From: opficdev <162981733+opficdev@users.noreply.github.com> Date: Tue, 23 Jun 2026 22:51:44 +0900 Subject: [PATCH 1/8] =?UTF-8?q?refactor:=20PR=20metadata=20report=20?= =?UTF-8?q?=EC=B6=9C=EB=A0=A5=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reports/markdownFormatter.ts | 10 ---------- tests/reports/markdownFormatter.test.ts | 22 +++++----------------- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/src/reports/markdownFormatter.ts b/src/reports/markdownFormatter.ts index 9a2c60b..7096002 100644 --- a/src/reports/markdownFormatter.ts +++ b/src/reports/markdownFormatter.ts @@ -61,11 +61,6 @@ function metadataLinesFor(item: MergeRiskReportItem): string[] { lines.push(`- updated: ${code(item.updatedAt.toISOString())}`) } - if (item.pullRequest) { - const title = escapeLinkText(item.pullRequest.title) - lines.push(`- pull request: [#${item.pullRequest.number} ${title}](${item.pullRequest.url})`) - } - return lines } @@ -159,11 +154,6 @@ function actionLinesFor(action: AiRecommendedAction): string[] { return lines } -// Markdown link text 안의 대괄호가 링크 경계를 깨지 않도록 escape -function escapeLinkText(value: string): string { - return value.replaceAll("[", "\\[").replaceAll("]", "\\]") -} - // Markdown inline code 안의 backtick보다 긴 delimiter를 사용해 code span을 구성 function code(value: string): string { const backtick = "`" diff --git a/tests/reports/markdownFormatter.test.ts b/tests/reports/markdownFormatter.test.ts index 5338520..de1c3e6 100644 --- a/tests/reports/markdownFormatter.test.ts +++ b/tests/reports/markdownFormatter.test.ts @@ -19,25 +19,14 @@ test("formats merge risk report summary and sections", () => { assert.match(markdown, /- score\/status: `65` \/ `high`/) }) -// branch metadata와 Pull Request link가 Markdown item에 포함되는지 확인 -test("formats branch metadata and pull request link", () => { +// branch metadata는 유지하되 Pull Request metadata는 Markdown item에서 제외되는지 확인 +test("formats branch metadata without pull request link", () => { const markdown = formatMergeRiskReportMarkdown(report()) assert.match(markdown, /- author: `opfic`/) assert.match(markdown, /- updated: `2026-06-22T01:00:00.000Z`/) - assert.match( - markdown, - /- pull request: \[#12 Report item\]\(https:\/\/github.com\/opficdev\/Watcher\/pull\/12\)/ - ) -}) - -// Pull Request 제목의 대괄호가 Markdown link text를 깨지 않도록 escape되는지 확인 -test("escapes pull request title brackets", () => { - const markdown = formatMergeRiskReportMarkdown(report(undefined, { - pullRequestTitle: "[Report] item" - })) - - assert.match(markdown, /- pull request: \[#12 \\\[Report\\\] item\]\(https:\/\/github.com\/opficdev\/Watcher\/pull\/12\)/) + assert.doesNotMatch(markdown, /pull request/) + assert.doesNotMatch(markdown, /github\.com\/opficdev\/Watcher\/pull\/12/) }) // deterministic reason의 관련 파일, branch, check metadata가 표시되는지 확인 @@ -120,7 +109,6 @@ function report( aiPrediction?: AiPredictionResult, options: { branchName?: string - pullRequestTitle?: string reasonFile?: string } = {} ): MergeRiskReport { @@ -142,7 +130,7 @@ function report( updatedAt: new Date("2026-06-22T01:00:00.000Z"), pullRequest: { number: 12, - title: options.pullRequestTitle ?? "Report item", + title: "Report item", url: "https://github.com/opficdev/Watcher/pull/12", author: "opfic" }, From 1aef41579600d558d848183fc1a4dc8e99c993f1 Mon Sep 17 00:00:00 2001 From: opficdev <162981733+opficdev@users.noreply.github.com> Date: Tue, 23 Jun 2026 22:54:32 +0900 Subject: [PATCH 2/8] =?UTF-8?q?refactor:=20AI=20prediction=20report=20?= =?UTF-8?q?=EC=B6=9C=EB=A0=A5=20=EC=B6=95=EC=95=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reports/markdownFormatter.ts | 10 ++-------- tests/reports/markdownFormatter.test.ts | 7 +++---- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/reports/markdownFormatter.ts b/src/reports/markdownFormatter.ts index 7096002..5b7ce66 100644 --- a/src/reports/markdownFormatter.ts +++ b/src/reports/markdownFormatter.ts @@ -102,12 +102,11 @@ function aiPredictionLinesFor(prediction: AiPredictionResult | undefined): strin return failedLinesFor(prediction) } -// AI가 생성한 prediction, confidence, action, false positive note를 표시 +// AI가 생성한 prediction과 action을 표시 function predictedLinesFor(result: AiPredictionPredictedResult): string[] { const lines = [ "- ai prediction:", - ` - prediction: ${result.prediction.prediction}`, - ` - confidence: ${code(result.prediction.confidence.toString())}` + ` - prediction: ${result.prediction.prediction}` ] if (result.prediction.recommendedActions.length) { @@ -115,11 +114,6 @@ function predictedLinesFor(result: AiPredictionPredictedResult): string[] { lines.push(...result.prediction.recommendedActions.flatMap(action => actionLinesFor(action))) } - if (result.prediction.falsePositiveNotes.length) { - lines.push(" - false positive notes:") - lines.push(...result.prediction.falsePositiveNotes.map(note => ` - ${note}`)) - } - return lines } diff --git a/tests/reports/markdownFormatter.test.ts b/tests/reports/markdownFormatter.test.ts index de1c3e6..80f55e3 100644 --- a/tests/reports/markdownFormatter.test.ts +++ b/tests/reports/markdownFormatter.test.ts @@ -50,18 +50,17 @@ test("formats inline code containing backticks", () => { assert.match(markdown, /- files: `` src\/`shared`\.ts ``/) }) -// AI predicted 결과를 deterministic reason과 분리해 표시하는지 확인 +// AI predicted 결과를 action 중심으로 축약해 표시하는지 확인 test("formats predicted AI result", () => { const markdown = formatMergeRiskReportMarkdown(report(predictedAiResult("feature/risk"))) assert.match(markdown, /- ai prediction:/) assert.match(markdown, /- prediction: 공유 파일 변경 의도가 겹칠 가능성 있음/) - assert.match(markdown, /- confidence: `82`/) assert.match(markdown, /- recommended actions:/) assert.match(markdown, /- `high` base branch rebase: 최신 main 기준으로 rebase 후 실제 충돌 여부 확인/) assert.match(markdown, /- files: `src\/shared.ts`/) - assert.match(markdown, /- false positive notes:/) - assert.match(markdown, /- 파일은 같지만 line range가 다르면 false positive 가능성 있음/) + assert.doesNotMatch(markdown, /confidence/) + assert.doesNotMatch(markdown, /false positive notes/) }) // AI skipped 결과를 report에 표시하는지 확인 From 84e80d3b90040b9585b9ce0e3bf6f0182901bb37 Mon Sep 17 00:00:00 2001 From: opficdev <162981733+opficdev@users.noreply.github.com> Date: Tue, 23 Jun 2026 23:01:01 +0900 Subject: [PATCH 3/8] =?UTF-8?q?refactor:=20risk=20reason=20=EC=A4=91?= =?UTF-8?q?=EB=B3=B5=20=EC=B6=9C=EB=A0=A5=20=EC=B6=95=EC=95=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reports/markdownFormatter.ts | 16 +++++++++---- tests/reports/markdownFormatter.test.ts | 31 ++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/reports/markdownFormatter.ts b/src/reports/markdownFormatter.ts index 5b7ce66..98eb8f5 100644 --- a/src/reports/markdownFormatter.ts +++ b/src/reports/markdownFormatter.ts @@ -38,13 +38,18 @@ export function format(report: MergeRiskReport): string { // branch 하나의 score, metadata, reason을 Markdown block으로 구성 function linesForItem(item: MergeRiskReportItem): string[] { + const hasSameHunkOverlap = item.reasons.some(reason => reason.code === "same_hunk_overlap") + return [ "", `#### ${code(item.branchName)}`, `- score/status: ${code(item.score.toString())} / ${code(item.status)}`, ...metadataLinesFor(item), "- reasons:", - ...item.reasons.flatMap(reason => linesForReason(reason)), + ...item.reasons.flatMap(reason => linesForReason( + reason, + hasSameHunkOverlap && reason.code === "same_file_overlap" + )), ...aiPredictionLinesFor(item.aiPrediction) ] } @@ -65,16 +70,19 @@ function metadataLinesFor(item: MergeRiskReportItem): string[] { } // deterministic reason의 code, score 영향, 관련 metadata를 Markdown bullet로 구성 -function linesForReason(reason: BranchRiskReason): string[] { +function linesForReason( + reason: BranchRiskReason, + hidesOverlappedMetadata = false +): string[] { const lines = [ ` - ${code(reason.code)} (+${reason.scoreImpact.toString()}): ${reason.message}` ] - if (reason.files?.length) { + if (!hidesOverlappedMetadata && reason.files?.length) { lines.push(` - files: ${reason.files.map(code).join(", ")}`) } - if (reason.branches?.length) { + if (!hidesOverlappedMetadata && reason.branches?.length) { lines.push(` - branches: ${reason.branches.map(code).join(", ")}`) } diff --git a/tests/reports/markdownFormatter.test.ts b/tests/reports/markdownFormatter.test.ts index 80f55e3..7977102 100644 --- a/tests/reports/markdownFormatter.test.ts +++ b/tests/reports/markdownFormatter.test.ts @@ -4,6 +4,7 @@ import { BranchRiskStatus, formatMergeRiskReportMarkdown, type AiPredictionResult, + type BranchRiskReason, type MergeRiskReport } from "../../src/index.js" @@ -39,6 +40,33 @@ test("formats deterministic reason metadata", () => { assert.match(markdown, /- checks: `build`/) }) +// same hunk와 same file overlap이 같이 있으면 중복 file, branch 목록을 축약하는지 확인 +test("compacts duplicated same file overlap metadata", () => { + const markdown = formatMergeRiskReportMarkdown(report(undefined, { + reasons: [ + { + code: "same_hunk_overlap", + message: "다른 branch와 같은 hunk를 수정함", + scoreImpact: 55, + files: ["src/shared.ts"], + branches: ["feature/other"] + }, + { + code: "same_file_overlap", + message: "다른 branch와 같은 파일을 수정함", + scoreImpact: 30, + files: ["src/shared.ts"], + branches: ["feature/other"] + } + ] + })) + + assert.match(markdown, /- `same_hunk_overlap` \(\+55\): 다른 branch와 같은 hunk를 수정함/) + assert.match(markdown, /- `same_file_overlap` \(\+30\): 다른 branch와 같은 파일을 수정함/) + assert.equal(markdown.match(/- files: `src\/shared\.ts`/g)?.length, 1) + assert.equal(markdown.match(/- branches: `feature\/other`/g)?.length, 1) +}) + // inline code 내부 backtick이 Markdown code span 문법을 깨지 않도록 delimiter를 늘리는지 확인 test("formats inline code containing backticks", () => { const markdown = formatMergeRiskReportMarkdown(report(undefined, { @@ -109,6 +137,7 @@ function report( options: { branchName?: string reasonFile?: string + reasons?: BranchRiskReason[] } = {} ): MergeRiskReport { const branchName = options.branchName ?? "feature/risk" @@ -139,7 +168,7 @@ function report( headSha: "feature-risk-sha", checks: [] }, - reasons: [{ + reasons: options.reasons ?? [{ code: "same_hunk_overlap", message: "다른 branch와 같은 hunk를 수정함", scoreImpact: 35, From 5f9e7f68f125f1056981b56fa94a13d1e7266d15 Mon Sep 17 00:00:00 2001 From: opficdev <162981733+opficdev@users.noreply.github.com> Date: Tue, 23 Jun 2026 23:12:53 +0900 Subject: [PATCH 4/8] =?UTF-8?q?refactor:=20AI=20prediction=20batch=20?= =?UTF-8?q?=ED=98=B8=EC=B6=9C=20=EA=B5=AC=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ai/geminiPredictionClient.ts | 22 ++- src/ai/predictionPromptBuilder.ts | 22 ++- src/ai/predictionResponseValidator.ts | 28 ++- src/ai/predictionRunner.ts | 131 ++++++++------ src/ai/promptTemplates.ts | 29 ++++ src/ai/types.ts | 6 + src/index.ts | 16 +- tests/ai/geminiPredictionClient.test.ts | 50 ++++++ tests/ai/predictionPromptBuilder.test.ts | 46 +++-- tests/ai/predictionResponseValidator.test.ts | 24 ++- tests/ai/predictionRunner.test.ts | 169 ++++++------------- 11 files changed, 353 insertions(+), 190 deletions(-) diff --git a/src/ai/geminiPredictionClient.ts b/src/ai/geminiPredictionClient.ts index a18efd1..789abaa 100644 --- a/src/ai/geminiPredictionClient.ts +++ b/src/ai/geminiPredictionClient.ts @@ -161,6 +161,10 @@ export function createDefaultAiPredictionClient( // Watcher가 검증할 AiPrediction shape를 Gemini structured output schema로 전달 function geminiRequestBodyFor(prompt: AiPredictionPrompt): Record { + const schema = prompt.responseShape === "predictionBatch" + ? aiPredictionBatchSchema() + : aiPredictionSchema() + return { systemInstruction: { parts: [{ text: prompt.systemPrompt }] @@ -173,7 +177,7 @@ function geminiRequestBodyFor(prompt: AiPredictionPrompt): Record { + return { + type: "object", + properties: { + predictions: { + type: "array", + items: aiPredictionSchema() + } + }, + required: [ + "predictions" + ] + } +} + +// Gemini가 Watcher prediction 하나의 contract에 맞는 JSON을 반환하도록 요청하는 schema function aiPredictionSchema(): Record { return { type: "object", diff --git a/src/ai/predictionPromptBuilder.ts b/src/ai/predictionPromptBuilder.ts index c0d3d8b..637f0b0 100644 --- a/src/ai/predictionPromptBuilder.ts +++ b/src/ai/predictionPromptBuilder.ts @@ -1,4 +1,7 @@ -import { DEFAULT_AI_PREDICTION_SYSTEM_PROMPT } from "./promptTemplates.js" +import { + DEFAULT_AI_PREDICTION_BATCH_SYSTEM_PROMPT, + DEFAULT_AI_PREDICTION_SYSTEM_PROMPT +} from "./promptTemplates.js" import type { AiPredictionEvidencePayload, AiPredictionPrompt, @@ -12,7 +15,22 @@ export function build( ): AiPredictionPrompt { return { systemPrompt: options.systemPrompt ?? DEFAULT_AI_PREDICTION_SYSTEM_PROMPT, - userPrompt: JSON.stringify(promptEvidenceFor(payload), null, 2) + userPrompt: JSON.stringify(promptEvidenceFor(payload), null, 2), + responseShape: "prediction" + } +} + +// 여러 branch evidence를 한 번의 AI provider 호출에 전달할 batch prompt로 구성 +export function buildBatch( + payloads: AiPredictionEvidencePayload[], + options: AiPredictionPromptBuildOptions = {} +): AiPredictionPrompt { + return { + systemPrompt: options.systemPrompt ?? DEFAULT_AI_PREDICTION_BATCH_SYSTEM_PROMPT, + userPrompt: JSON.stringify({ + branches: payloads.map(promptEvidenceFor) + }, null, 2), + responseShape: "predictionBatch" } } diff --git a/src/ai/predictionResponseValidator.ts b/src/ai/predictionResponseValidator.ts index 3616a43..e0464e6 100644 --- a/src/ai/predictionResponseValidator.ts +++ b/src/ai/predictionResponseValidator.ts @@ -12,17 +12,33 @@ const actionPriorities = new Set([ // AI가 반환한 unknown JSON을 Watcher가 사용하는 prediction 모델로 검증 export function validate(response: unknown): AiPrediction { + return predictionFor(response, "response") +} + +// AI가 반환한 batch JSON을 branch별 prediction 배열로 검증 +export function validateBatch(response: unknown): AiPrediction[] { const value = objectFor(response, "response") + return arrayFor(value.predictions, "predictions") + .map((prediction, index) => predictionFor(prediction, `predictions[${index}]`)) +} + +// AI prediction 하나가 Watcher report에 사용할 수 있는 shape인지 검증 +function predictionFor( + response: unknown, + path: string +): AiPrediction { + const value = objectFor(response, path) + return { - branchName: stringFor(value.branchName, "branchName"), - baseBranch: stringFor(value.baseBranch, "baseBranch"), - prediction: stringFor(value.prediction, "prediction"), + branchName: stringFor(value.branchName, `${path}.branchName`), + baseBranch: stringFor(value.baseBranch, `${path}.baseBranch`), + prediction: stringFor(value.prediction, `${path}.prediction`), confidence: confidenceFor(value.confidence), - recommendedActions: arrayFor(value.recommendedActions ?? [], "recommendedActions") + recommendedActions: arrayFor(value.recommendedActions ?? [], `${path}.recommendedActions`) .map((action, index) => recommendedActionFor(action, index)), - falsePositiveNotes: arrayFor(value.falsePositiveNotes ?? [], "falsePositiveNotes") - .map((note, index) => stringFor(note, `falsePositiveNotes[${index}]`)) + falsePositiveNotes: arrayFor(value.falsePositiveNotes ?? [], `${path}.falsePositiveNotes`) + .map((note, index) => stringFor(note, `${path}.falsePositiveNotes[${index}]`)) } } diff --git a/src/ai/predictionRunner.ts b/src/ai/predictionRunner.ts index 0ba8c4e..8bfd712 100644 --- a/src/ai/predictionRunner.ts +++ b/src/ai/predictionRunner.ts @@ -1,71 +1,52 @@ -import { build as buildPrompt } from "./predictionPromptBuilder.js" +import { buildBatch as buildBatchPrompt } from "./predictionPromptBuilder.js" import { select as selectTargets } from "./predictionTargetSelector.js" -import { validate as validateResponse } from "./predictionResponseValidator.js" +import { validateBatch as validateBatchResponse } from "./predictionResponseValidator.js" import type { + AiPrediction, AiPredictionClient, AiPredictionEvidencePayload, AiPredictionResult, AiPredictionRunOptions } from "./types.js" -// Gemini free tier rate limit을 줄이기 위해 선택된 AI 호출 사이에 둘 내부 대기 시간 -const DEFAULT_AI_PREDICTION_DELAY_MS = 60000 - -// 대상 선택, prompt 생성, provider 호출, 응답 검증을 branch별 AI prediction 결과로 연결 +// 대상 선택, batch prompt 생성, provider 호출, 응답 검증을 branch별 AI prediction 결과로 연결 export async function predict( payloads: AiPredictionEvidencePayload[], client: AiPredictionClient, options: AiPredictionRunOptions = {} ): Promise { - const targets = new Set(selectTargets(payloads)) - const results: AiPredictionResult[] = [] - - for (const payload of payloads) { - if (!targets.has(payload)) { - results.push({ - status: "skipped", - branchName: payload.branch.name, - baseBranch: payload.branch.baseBranch, - reason: skippedReasonFor(payload) - }) - continue - } + const targets = selectTargets(payloads) + const targetSet = new Set(targets) - results.push(await predictedResultFor(payload, client, options)) - targets.delete(payload) - - if (0 < targets.size) { - await delay(DEFAULT_AI_PREDICTION_DELAY_MS) - } + if (targets.length === 0) { + return payloads.map(skippedResultFor) } - return results + const predictedResults = await predictedResultsFor(targets, client, options) + + return payloads.map(payload => targetSet.has(payload) + ? predictedResults.get(payload) ?? failedResultFor(payload, "AI prediction response is missing") + : skippedResultFor(payload) + ) } -// provider 호출과 schema 검증 실패를 branch 단위 failed 결과로 격리 -async function predictedResultFor( - payload: AiPredictionEvidencePayload, +// provider 호출과 schema 검증 실패를 선택된 branch 단위 failed 결과로 격리 +async function predictedResultsFor( + targets: AiPredictionEvidencePayload[], client: AiPredictionClient, options: AiPredictionRunOptions -): Promise { +): Promise> { try { - const prompt = buildPrompt(payload, options) + const prompt = buildBatchPrompt(targets, options) const response = await client.predict(prompt) - const prediction = validateResponse(response) + const predictions = validateBatchResponse(response) - return { - status: "predicted", - branchName: payload.branch.name, - baseBranch: payload.branch.baseBranch, - prediction - } + return resultsByPayloadFor(targets, predictions) } catch (error) { - return { - status: "failed", - branchName: payload.branch.name, - baseBranch: payload.branch.baseBranch, - errorMessage: errorMessageFor(error) - } + return new Map(targets.map(target => [ + target, + failedResultFor(target, errorMessageFor(error)) + ])) } } @@ -81,11 +62,63 @@ function skippedReasonFor(payload: AiPredictionEvidencePayload): "not_target" | : "not_target" } -// 선택된 AI provider 호출 사이에 간격을 두어 rate limit 진입 가능성을 낮춤 -function delay(delayMs: number): Promise { - if (delayMs <= 0) { - return Promise.resolve() +// AI prediction 대상이 아닌 branch를 skipped 결과로 변환 +function skippedResultFor(payload: AiPredictionEvidencePayload): AiPredictionResult { + return { + status: "skipped", + branchName: payload.branch.name, + baseBranch: payload.branch.baseBranch, + reason: skippedReasonFor(payload) + } +} + +// provider 실패나 응답 누락을 branch별 failed 결과로 변환 +function failedResultFor( + payload: AiPredictionEvidencePayload, + errorMessage: string +): AiPredictionResult { + return { + status: "failed", + branchName: payload.branch.name, + baseBranch: payload.branch.baseBranch, + errorMessage } +} + +// batch 응답을 원래 target payload와 매칭해 branch별 결과로 복원 +function resultsByPayloadFor( + targets: AiPredictionEvidencePayload[], + predictions: AiPrediction[] +): Map { + const predictionsByBranch = new Map(predictions.map(prediction => [ + predictionKeyFor(prediction.branchName, prediction.baseBranch), + prediction + ])) + + return new Map(targets.map(target => { + const prediction = predictionsByBranch.get(predictionKeyFor( + target.branch.name, + target.branch.baseBranch + )) + + return [ + target, + prediction + ? { + status: "predicted", + branchName: target.branch.name, + baseBranch: target.branch.baseBranch, + prediction + } + : failedResultFor(target, "AI prediction response is missing") + ] + })) +} - return new Promise(resolve => setTimeout(resolve, delayMs)) +// branch 이름만 같은 다른 base branch와 섞이지 않도록 base branch까지 포함해 매칭 +function predictionKeyFor( + branchName: string, + baseBranch: string +): string { + return `${baseBranch}\u0000${branchName}` } diff --git a/src/ai/promptTemplates.ts b/src/ai/promptTemplates.ts index 37920ea..4602c35 100644 --- a/src/ai/promptTemplates.ts +++ b/src/ai/promptTemplates.ts @@ -23,3 +23,32 @@ export const DEFAULT_AI_PREDICTION_SYSTEM_PROMPT = [ " \"falsePositiveNotes\": string[]", "}" ].join("\n") + +// 여러 branch evidence를 한 번에 판단할 때 사용하는 batch system prompt +export const DEFAULT_AI_PREDICTION_BATCH_SYSTEM_PROMPT = [ + "You are Watcher's merge risk prediction assistant.", + "Use only the provided deterministic evidence.", + "Do not recalculate or overwrite the possibility score, status, or reasons.", + "Do not describe the deterministic score as a probability or percentage.", + "Use probabilistic wording. Avoid phrases such as guaranteed, will cause, or will result for possibility-based risks.", + "Compare the provided branches together and predict practical merge risk impact.", + "Recommend next actions for each branch.", + "Write prediction, recommended action titles, descriptions, and false positive notes in Korean.", + "If the evidence is weak, explain possible false positives.", + "Return only JSON with this shape:", + "{", + " \"predictions\": [{", + " \"branchName\": string,", + " \"baseBranch\": string,", + " \"prediction\": string,", + " \"confidence\": integer from 0 to 100. Use 98 for high confidence, not 0.98 or 1,", + " \"recommendedActions\": [{", + " \"title\": string,", + " \"description\": string,", + " \"priority\": \"low\" | \"medium\" | \"high\",", + " \"files\": string[]", + " }],", + " \"falsePositiveNotes\": string[]", + " }]", + "}" +].join("\n") diff --git a/src/ai/types.ts b/src/ai/types.ts index bae00ef..14178e5 100644 --- a/src/ai/types.ts +++ b/src/ai/types.ts @@ -14,8 +14,14 @@ export type AiPredictionEvidencePayload = { export type AiPredictionPrompt = { systemPrompt: string userPrompt: string + responseShape?: AiPredictionPromptResponseShape } +// provider가 structured output schema를 고를 때 사용할 응답 형태 +export type AiPredictionPromptResponseShape = + | "prediction" + | "predictionBatch" + // prompt 호출자가 provider나 실행 환경에 맞게 system prompt를 교체하기 위한 설정 export type AiPredictionPromptBuildOptions = { systemPrompt?: string diff --git a/src/index.ts b/src/index.ts index 787b754..d40a05c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,14 +13,23 @@ export { GEMINI_API_KEY_ENV_NAME, GeminiPredictionClient } from "./ai/geminiPredictionClient.js" -export { build as buildAiPredictionPrompt } from "./ai/predictionPromptBuilder.js" -export { DEFAULT_AI_PREDICTION_SYSTEM_PROMPT } from "./ai/promptTemplates.js" +export { + build as buildAiPredictionPrompt, + buildBatch as buildAiPredictionBatchPrompt +} from "./ai/predictionPromptBuilder.js" +export { + DEFAULT_AI_PREDICTION_BATCH_SYSTEM_PROMPT, + DEFAULT_AI_PREDICTION_SYSTEM_PROMPT +} from "./ai/promptTemplates.js" export { predict as predictMergeRisksWithAi } from "./ai/predictionRunner.js" export { DEFAULT_AI_PREDICTION_TARGET_STATUS, select as selectAiPredictionTargets } from "./ai/predictionTargetSelector.js" -export { validate as validateAiPredictionResponse } from "./ai/predictionResponseValidator.js" +export { + validate as validateAiPredictionResponse, + validateBatch as validateAiPredictionBatchResponse +} from "./ai/predictionResponseValidator.js" export { collectGitMergeSignal } from "./git/gitMergeSignalCollector.js" export { send as sendMergeRiskReport } from "./reportChannels/reportChannel.js" export { analyze as analyzeBranchMergeRisks } from "./risks/riskAnalyzer.js" @@ -41,6 +50,7 @@ export type { AiPredictionFailedResult, AiPredictionPrompt, AiPredictionPromptBuildOptions, + AiPredictionPromptResponseShape, AiPredictionPredictedResult, AiPredictionResult, AiPredictionRunOptions, diff --git a/tests/ai/geminiPredictionClient.test.ts b/tests/ai/geminiPredictionClient.test.ts index c7a587f..510510f 100644 --- a/tests/ai/geminiPredictionClient.test.ts +++ b/tests/ai/geminiPredictionClient.test.ts @@ -84,6 +84,34 @@ test("sends prompt to Gemini generateContent endpoint", async () => { assert.equal(body.generationConfig.responseFormat.text.mimeType, "APPLICATION_JSON") }) +// batch prompt는 Gemini structured output schema도 predictions 배열로 요청하는지 확인 +test("sends batch response schema to Gemini", async () => { + const fetcher = fetchSpy(validGeminiBatchResponse()) + const client = new GeminiPredictionClient({ + apiKey: "gemini-key", + fetch: fetcher + }) + + await client.predict(batchPrompt()) + + const request = fetcher.requests[0] + const body = JSON.parse(request?.init.body as string) as { + generationConfig: { + responseFormat: { + text: { + schema: { + properties: { + predictions?: unknown + } + } + } + } + } + } + + assert.notEqual(body.generationConfig.responseFormat.text.schema.properties.predictions, undefined) +}) + // Gemini 503 계열 일시 실패는 exponential backoff 후 재시도하는지 확인 test("retries Gemini high demand failures with exponential backoff", async () => { const fetcher = fetchSequenceSpy([ @@ -354,6 +382,14 @@ function prompt(): AiPredictionPrompt { } } +function batchPrompt(): AiPredictionPrompt { + return { + systemPrompt: "Return JSON only.", + userPrompt: "{\"branches\":[{\"branch\":\"feature/a\"}]}", + responseShape: "predictionBatch" + } +} + // Gemini generateContent가 반환하는 JSON text 응답 fixture function validGeminiResponse(): unknown { return { @@ -367,6 +403,20 @@ function validGeminiResponse(): unknown { } } +function validGeminiBatchResponse(): unknown { + return { + candidates: [{ + content: { + parts: [{ + text: JSON.stringify({ + predictions: [validPrediction()] + }) + }] + } + }] + } +} + // Watcher AI prediction schema를 만족하는 parsed JSON fixture function validPrediction(): unknown { return { diff --git a/tests/ai/predictionPromptBuilder.test.ts b/tests/ai/predictionPromptBuilder.test.ts index 3ab8e25..fa680b3 100644 --- a/tests/ai/predictionPromptBuilder.test.ts +++ b/tests/ai/predictionPromptBuilder.test.ts @@ -1,7 +1,9 @@ import test from "node:test" import assert from "node:assert/strict" import { + DEFAULT_AI_PREDICTION_BATCH_SYSTEM_PROMPT, DEFAULT_AI_PREDICTION_SYSTEM_PROMPT, + buildAiPredictionBatchPrompt, buildAiPredictionPrompt, type AiPredictionEvidencePayload, type BranchContext, @@ -39,6 +41,28 @@ test("builds user prompt with structured evidence", () => { assert.equal(changedHunks[0]?.filePath, "src/shared.ts") }) +// batch prompt는 여러 branch evidence를 하나의 provider 호출 입력으로 묶는지 확인 +test("builds batch user prompt with structured evidence list", () => { + const prompt = buildAiPredictionBatchPrompt([ + payload("feature/a"), + payload("feature/b") + ]) + const evidence = JSON.parse(prompt.userPrompt) as { + branches: Array<{ + branch: { + name: string + } + }> + } + + assert.equal(prompt.systemPrompt, DEFAULT_AI_PREDICTION_BATCH_SYSTEM_PROMPT) + assert.equal(prompt.responseShape, "predictionBatch") + assert.deepEqual(evidence.branches.map(branch => branch.branch.name), [ + "feature/a", + "feature/b" + ]) +}) + // prompt 출력이 provider와 무관한 system/user 문자열로만 구성되는지 확인 test("keeps prompt provider agnostic", () => { const prompt = buildAiPredictionPrompt(payload()) @@ -57,11 +81,11 @@ test("allows caller provided system prompt", () => { assert.equal(prompt.systemPrompt, "Return only compact JSON.") }) -function payload(): AiPredictionEvidencePayload { +function payload(branchName = "feature/watch"): AiPredictionEvidencePayload { return { - branch: branch(), - possibility: possibility(), - gitSignal: gitSignal(), + branch: branch(branchName), + possibility: possibility(branchName), + gitSignal: gitSignal(branchName), changedHunks: [{ filePath: "src/shared.ts", startLine: 12, @@ -70,11 +94,11 @@ function payload(): AiPredictionEvidencePayload { } } -function branch(): BranchContext { +function branch(name = "feature/watch"): BranchContext { return { baseBranch: "main", - name: "feature/watch", - headSha: "feature-watch-sha", + name, + headSha: `${name}-sha`, author: "opfic", updatedAt: new Date("2026-06-22T00:00:00.000Z"), checks: [{ @@ -90,9 +114,9 @@ function branch(): BranchContext { } } -function possibility(): BranchRisk { +function possibility(branchName = "feature/watch"): BranchRisk { return { - branchName: "feature/watch", + branchName, baseBranch: "main", score: 55, status: "high", @@ -105,11 +129,11 @@ function possibility(): BranchRisk { } } -function gitSignal(): GitMergeSignal { +function gitSignal(branchName = "feature/watch"): GitMergeSignal { return { status: "clean", baseBranch: "main", - branchName: "feature/watch", + branchName, mergeBaseSha: "merge-base-sha", changedFiles: ["src/shared.ts"], conflictFiles: [] diff --git a/tests/ai/predictionResponseValidator.test.ts b/tests/ai/predictionResponseValidator.test.ts index 1c4bfd6..af402f1 100644 --- a/tests/ai/predictionResponseValidator.test.ts +++ b/tests/ai/predictionResponseValidator.test.ts @@ -1,6 +1,9 @@ import test from "node:test" import assert from "node:assert/strict" -import { validateAiPredictionResponse } from "../../src/index.js" +import { + validateAiPredictionBatchResponse, + validateAiPredictionResponse +} from "../../src/index.js" // AI prediction JSON이 기대한 모델이면 그대로 통과하는지 확인 test("validates ai prediction response", () => { @@ -26,6 +29,21 @@ test("validates ai prediction response", () => { ]) }) +// batch AI prediction JSON이 branch별 prediction 배열이면 그대로 통과하는지 확인 +test("validates ai prediction batch response", () => { + const predictions = validateAiPredictionBatchResponse({ + predictions: [ + validResponse("feature/a"), + validResponse("feature/b") + ] + }) + + assert.deepEqual(predictions.map(prediction => prediction.branchName), [ + "feature/a", + "feature/b" + ]) +}) + // confidence가 0-100 범위를 벗어나면 AI 응답을 거부하는지 확인 test("rejects confidence outside report range", () => { assert.throws( @@ -119,9 +137,9 @@ test("rejects non-string action files", () => { ) }) -function validResponse(): Record { +function validResponse(branchName = "feature/watch"): Record { return { - branchName: "feature/watch", + branchName, baseBranch: "main", prediction: "shared module 변경 의도가 겹쳐 rebase 우선 확인이 필요함", confidence: 82, diff --git a/tests/ai/predictionRunner.test.ts b/tests/ai/predictionRunner.test.ts index 5b28486..8e33cdc 100644 --- a/tests/ai/predictionRunner.test.ts +++ b/tests/ai/predictionRunner.test.ts @@ -1,4 +1,4 @@ -import test, { mock } from "node:test" +import test from "node:test" import assert from "node:assert/strict" import { BranchRiskStatus, @@ -12,7 +12,7 @@ import { type GitMergeSignal } from "../../src/index.js" -// critical branch만 AI client로 prediction을 요청하는지 확인 +// critical branch만 한 번의 batch AI client 호출로 prediction을 요청하는지 확인 test("predicts selected critical merge risk payloads", async () => { const client = new AiPredictionClientSpy() const results = await predictMergeRisksWithAi([ @@ -22,6 +22,7 @@ test("predicts selected critical merge risk payloads", async () => { assert.deepEqual(results.map(result => result.status), ["skipped", "predicted"]) assert.equal(client.prompts.length, 1) + assert.equal(client.prompts[0]?.responseShape, "predictionBatch") const predicted = results[1] assert.equal( predicted?.status === "predicted" ? predicted.prediction.branchName : undefined, @@ -53,85 +54,55 @@ test("records confirmed conflict skip reason", async () => { assert.equal(client.prompts.length, 0) }) -// Gemini 무료 등급의 일시 실패를 줄이기 위해 선택된 branch prediction을 순차 실행하는지 확인 -test("runs selected predictions sequentially", async () => { - mock.timers.enable({ apis: ["setTimeout"] }) - const client = new DeferredAiPredictionClient() - - try { - const running = predictMergeRisksWithAi([ - payload("feature/a", 100, BranchRiskStatus.Critical), - payload("feature/b", 100, BranchRiskStatus.Critical) - ], client) - - await client.waitForPrompts(1) - assert.equal(client.prompts.length, 1) - - client.resolveNext() - await flushTasks() - mock.timers.tick(60000) - await flushTasks() - assert.equal(client.prompts.length, 2) - - client.resolveNext() - const results = await running - assert.deepEqual(results.map(result => result.status), ["predicted", "predicted"]) - } finally { - mock.timers.reset() - } -}) - -// 선택된 AI 호출 사이에 내부 기본 간격을 두는지 확인 -test("waits between selected predictions", async () => { - mock.timers.enable({ apis: ["setTimeout"] }) - const client = new DeferredAiPredictionClient() - - try { - const running = predictMergeRisksWithAi([ - payload("feature/a", 100, BranchRiskStatus.Critical), - payload("feature/b", 100, BranchRiskStatus.Critical) - ], client) - - await client.waitForPrompts(1) - client.resolveNext() - await flushTasks() - - mock.timers.tick(59999) - await flushTasks() - assert.equal(client.prompts.length, 1) - - mock.timers.tick(1) - await flushTasks() - assert.equal(client.prompts.length, 2) +// 여러 critical branch를 한 번의 AI 응답으로 다시 branch별 결과에 매칭하는지 확인 +test("maps batch prediction response to selected payload order", async () => { + const client = new AiPredictionClientSpy({ + predictions: [ + validResponse("feature/b"), + validResponse("feature/a") + ] + }) + const results = await predictMergeRisksWithAi([ + payload("feature/a", 100, BranchRiskStatus.Critical), + payload("feature/b", 100, BranchRiskStatus.Critical) + ], client) - client.resolveNext() - await running - assert.equal(client.prompts.length, 2) - } finally { - mock.timers.reset() - } + assert.deepEqual(results.map(result => result.status), ["predicted", "predicted"]) + assert.equal( + results[0]?.status === "predicted" ? results[0].prediction.branchName : undefined, + "feature/a" + ) + assert.equal( + results[1]?.status === "predicted" ? results[1].prediction.branchName : undefined, + "feature/b" + ) + assert.equal(client.prompts.length, 1) }) -// AI client 오류가 전체 실행 실패가 아니라 branch 단위 failed 결과로 기록되는지 확인 +// AI client 오류가 전체 실행 실패가 아니라 선택된 branch 단위 failed 결과로 기록되는지 확인 test("records failed result when client throws", async () => { const client = new AiPredictionClientSpy(new Error("provider failed")) - const [result] = await predictMergeRisksWithAi([ - payload("feature/critical", 100, BranchRiskStatus.Critical) + const results = await predictMergeRisksWithAi([ + payload("feature/a", 100, BranchRiskStatus.Critical), + payload("feature/b", 100, BranchRiskStatus.Critical) ], client) - assert.equal(result?.status, "failed") - assert.match(result?.status === "failed" ? result.errorMessage : "", /provider failed/) + assert.deepEqual(results.map(result => result.status), ["failed", "failed"]) + assert.match(results[0]?.status === "failed" ? results[0].errorMessage : "", /provider failed/) + assert.match(results[1]?.status === "failed" ? results[1].errorMessage : "", /provider failed/) }) // schema validation 실패가 branch 단위 failed 결과로 기록되는지 확인 test("records failed result when response is invalid", async () => { const client = new AiPredictionClientSpy({ - branchName: "feature/high", - baseBranch: "main", - prediction: "invalid confidence", - confidence: 120, - recommendedActions: [], - falsePositiveNotes: [] + predictions: [{ + branchName: "feature/critical", + baseBranch: "main", + prediction: "invalid confidence", + confidence: 120, + recommendedActions: [], + falsePositiveNotes: [] + }] }) const [result] = await predictMergeRisksWithAi([ payload("feature/critical", 100, BranchRiskStatus.Critical) @@ -165,58 +136,20 @@ class AiPredictionClientSpy implements AiPredictionClient { throw this.response } - return this.response ?? validResponse(branchNameFrom(prompt)) + return this.response ?? validBatchResponse(branchNamesFrom(prompt)) } } -class DeferredAiPredictionClient implements AiPredictionClient { - prompts: AiPredictionPrompt[] = [] - private promptWaiters: Array<() => void> = [] - private pending: Array<{ - prompt: AiPredictionPrompt - resolve: (value: unknown) => void - }> = [] - - async predict(prompt: AiPredictionPrompt): Promise { - this.prompts.push(prompt) - this.promptWaiters.splice(0).forEach(resolve => resolve()) - - return new Promise(resolve => { - this.pending.push({ prompt, resolve }) - }) - } - - async waitForPrompts(count: number): Promise { - while (this.prompts.length < count) { - await new Promise(resolve => { - this.promptWaiters.push(resolve) - }) - } - } - - resolveNext(): void { - const pending = this.pending.shift() - - if (!pending) { - throw new Error("No pending AI prediction") - } - - pending.resolve(validResponse(branchNameFrom(pending.prompt))) - } -} - -function flushTasks(): Promise { - return new Promise(resolve => setImmediate(resolve)) -} - -function branchNameFrom(prompt: AiPredictionPrompt): string { +function branchNamesFrom(prompt: AiPredictionPrompt): string[] { const evidence = JSON.parse(prompt.userPrompt) as { - branch: { - name: string - } + branches: Array<{ + branch: { + name: string + } + }> } - return evidence.branch.name + return evidence.branches.map(branch => branch.branch.name) } function payload( @@ -271,6 +204,12 @@ function gitSignal(branchName: string): GitMergeSignal { } } +function validBatchResponse(branchNames: string[]): unknown { + return { + predictions: branchNames.map(validResponse) + } +} + function validResponse(branchName: string): unknown { return { branchName, From 1e191111d4b36864f1723aba8cfe491307a9fb3f Mon Sep 17 00:00:00 2001 From: opficdev <162981733+opficdev@users.noreply.github.com> Date: Tue, 23 Jun 2026 23:16:12 +0900 Subject: [PATCH 5/8] =?UTF-8?q?docs:=20report=20=EC=B6=9C=EB=A0=A5=20?= =?UTF-8?q?=EC=A0=95=EC=B1=85=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7f15d0c..afb56fd 100644 --- a/README.md +++ b/README.md @@ -99,17 +99,17 @@ Watcher는 merge 가능/불가능을 단정하지 않고 branch별 signal을 충 `confirmed_conflict`는 최상위 signal입니다. 이 signal이 있으면 다른 reason을 추가로 합산하지 않고 `critical` risk로 처리합니다. -각 reason은 report에 code, message, score impact, 관련 file, 관련 branch, 관련 check metadata로 표시됩니다. 이 정보가 AI prediction에 전달되는 정제된 evidence입니다. +각 reason은 report에 code, message, score impact, 관련 file, 관련 branch, 관련 check metadata로 표시됩니다. 다만 `same_hunk_overlap`이 있는 branch에서는 중복되는 `same_file_overlap`의 file, branch 목록을 다시 반복하지 않습니다. 이 정보가 AI prediction에 전달되는 정제된 evidence입니다. ## AI-assisted prediction -AI prediction은 deterministic possibility score를 대체하지 않습니다. Watcher는 deterministic evidence를 Gemini API에 전달하고 AI는 실무 관점의 prediction, confidence, recommended actions, false positive notes를 추가합니다. +AI prediction은 deterministic possibility score를 대체하지 않습니다. Watcher는 deterministic evidence를 Gemini API에 전달하고 AI는 실무 관점의 prediction과 recommended actions를 추가합니다. 기본 AI provider는 Gemini API입니다. consumer repository에는 `GEMINI_API_KEY` secret을 설정해야 합니다. AI prediction 대상은 기본적으로 `critical` possibility입니다. 이미 virtual merge에서 conflict가 확정된 branch는 AI 호출 없이 deterministic report만 사용합니다. 그 외 낮은 status의 branch는 AI 호출을 생략하고 `skipped` 상태로 report에 표시됩니다. -Gemini free tier rate limit을 줄이기 위해 선택된 branch prediction은 순차 실행하며 기본적으로 호출 사이에 60초를 대기합니다. +Gemini free tier의 RPM/RPD 사용량을 줄이기 위해 선택된 branch prediction은 report 단위 batch 요청으로 한 번에 실행합니다. AI prediction 결과는 다음 상태 중 하나입니다. @@ -121,6 +121,8 @@ AI prediction 결과는 다음 상태 중 하나입니다. AI provider가 실패해도 deterministic possibility report는 유지됩니다. 실패한 branch는 `failed` 상태와 error message를 report에 포함합니다. +Report에는 `Low` section, AI prediction `skipped` 상태, branch `updated` 시각, Gemini error 요약을 유지합니다. Pull Request metadata는 내부 evidence로만 사용할 수 있으며 Markdown report에는 출력하지 않습니다. + ## Report channel Merge risk report는 Markdown으로 생성됩니다. consumer repository에 `DISCORD_WEBHOOK_URL` secret이 있으면 Discord webhook으로 전송하고, 없으면 stdout으로 출력합니다. From 5eb75827f0f1547fd406f53a1833d957a603d249 Mon Sep 17 00:00:00 2001 From: opficdev <162981733+opficdev@users.noreply.github.com> Date: Tue, 23 Jun 2026 23:26:54 +0900 Subject: [PATCH 6/8] =?UTF-8?q?refactor:=20same=5Ffile=5Foverlap=20?= =?UTF-8?q?=EC=A4=91=EB=B3=B5=20=ED=91=9C=EC=8B=9C=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reports/markdownFormatter.ts | 38 ++++++++++++++++++------- tests/reports/markdownFormatter.test.ts | 26 +++++++++++++++++ 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/reports/markdownFormatter.ts b/src/reports/markdownFormatter.ts index 98eb8f5..3c2b77a 100644 --- a/src/reports/markdownFormatter.ts +++ b/src/reports/markdownFormatter.ts @@ -38,7 +38,9 @@ export function format(report: MergeRiskReport): string { // branch 하나의 score, metadata, reason을 Markdown block으로 구성 function linesForItem(item: MergeRiskReportItem): string[] { - const hasSameHunkOverlap = item.reasons.some(reason => reason.code === "same_hunk_overlap") + const sameHunkFiles = new Set(item.reasons + .filter(reason => reason.code === "same_hunk_overlap") + .flatMap(reason => reason.files ?? [])) return [ "", @@ -46,10 +48,9 @@ function linesForItem(item: MergeRiskReportItem): string[] { `- score/status: ${code(item.score.toString())} / ${code(item.status)}`, ...metadataLinesFor(item), "- reasons:", - ...item.reasons.flatMap(reason => linesForReason( - reason, - hasSameHunkOverlap && reason.code === "same_file_overlap" - )), + ...item.reasons + .map(reason => compactSameFileOverlapReason(reason, sameHunkFiles)) + .flatMap(reason => linesForReason(reason)), ...aiPredictionLinesFor(item.aiPrediction) ] } @@ -69,20 +70,35 @@ function metadataLinesFor(item: MergeRiskReportItem): string[] { return lines } -// deterministic reason의 code, score 영향, 관련 metadata를 Markdown bullet로 구성 -function linesForReason( +// same hunk로 이미 설명된 file만 same file reason에서 제거해 중복 표시를 줄임 +function compactSameFileOverlapReason( reason: BranchRiskReason, - hidesOverlappedMetadata = false -): string[] { + sameHunkFiles: Set +): BranchRiskReason { + if (reason.code !== "same_file_overlap" || !reason.files?.length || sameHunkFiles.size === 0) { + return reason + } + + const files = reason.files.filter(file => !sameHunkFiles.has(file)) + + return { + ...reason, + files, + branches: 0 < files.length ? reason.branches : undefined + } +} + +// deterministic reason의 code, score 영향, 관련 metadata를 Markdown bullet로 구성 +function linesForReason(reason: BranchRiskReason): string[] { const lines = [ ` - ${code(reason.code)} (+${reason.scoreImpact.toString()}): ${reason.message}` ] - if (!hidesOverlappedMetadata && reason.files?.length) { + if (reason.files?.length) { lines.push(` - files: ${reason.files.map(code).join(", ")}`) } - if (!hidesOverlappedMetadata && reason.branches?.length) { + if (reason.branches?.length) { lines.push(` - branches: ${reason.branches.map(code).join(", ")}`) } diff --git a/tests/reports/markdownFormatter.test.ts b/tests/reports/markdownFormatter.test.ts index 7977102..2fa2dd2 100644 --- a/tests/reports/markdownFormatter.test.ts +++ b/tests/reports/markdownFormatter.test.ts @@ -67,6 +67,32 @@ test("compacts duplicated same file overlap metadata", () => { assert.equal(markdown.match(/- branches: `feature\/other`/g)?.length, 1) }) +// hunk와 겹치지 않는 same file overlap 정보는 축약 과정에서도 보존되는지 확인 +test("keeps non-hunk same file overlap metadata", () => { + const markdown = formatMergeRiskReportMarkdown(report(undefined, { + reasons: [ + { + code: "same_hunk_overlap", + message: "다른 branch와 같은 hunk를 수정함", + scoreImpact: 55, + files: ["src/a.ts"], + branches: ["feature/a"] + }, + { + code: "same_file_overlap", + message: "다른 branch와 같은 파일을 수정함", + scoreImpact: 30, + files: ["src/a.ts", "src/b.ts"], + branches: ["feature/a", "feature/b"] + } + ] + })) + + assert.equal(markdown.match(/`src\/a\.ts`/g)?.length, 1) + assert.match(markdown, /- files: `src\/b.ts`/) + assert.match(markdown, /- branches: `feature\/a`, `feature\/b`/) +}) + // inline code 내부 backtick이 Markdown code span 문법을 깨지 않도록 delimiter를 늘리는지 확인 test("formats inline code containing backticks", () => { const markdown = formatMergeRiskReportMarkdown(report(undefined, { From 5962180b4f4765ed09739fddf5edcc8929a7827d Mon Sep 17 00:00:00 2001 From: opficdev <162981733+opficdev@users.noreply.github.com> Date: Tue, 23 Jun 2026 23:28:51 +0900 Subject: [PATCH 7/8] =?UTF-8?q?fix:=20batch=20prediction=20=EB=B6=80?= =?UTF-8?q?=EB=B6=84=20=EC=8B=A4=ED=8C=A8=20=EA=B2=A9=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ai/predictionResponseValidator.ts | 9 +++++++- tests/ai/predictionResponseValidator.test.ts | 15 +++++++++++++ tests/ai/predictionRunner.test.ts | 23 +++++++++++++++++++- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/ai/predictionResponseValidator.ts b/src/ai/predictionResponseValidator.ts index e0464e6..2e29284 100644 --- a/src/ai/predictionResponseValidator.ts +++ b/src/ai/predictionResponseValidator.ts @@ -20,7 +20,14 @@ export function validateBatch(response: unknown): AiPrediction[] { const value = objectFor(response, "response") return arrayFor(value.predictions, "predictions") - .map((prediction, index) => predictionFor(prediction, `predictions[${index}]`)) + .map((prediction, index) => { + try { + return predictionFor(prediction, `predictions[${index}]`) + } catch { + return undefined + } + }) + .filter((prediction): prediction is AiPrediction => prediction !== undefined) } // AI prediction 하나가 Watcher report에 사용할 수 있는 shape인지 검증 diff --git a/tests/ai/predictionResponseValidator.test.ts b/tests/ai/predictionResponseValidator.test.ts index af402f1..2a83f84 100644 --- a/tests/ai/predictionResponseValidator.test.ts +++ b/tests/ai/predictionResponseValidator.test.ts @@ -44,6 +44,21 @@ test("validates ai prediction batch response", () => { ]) }) +// batch prediction 일부가 잘못되어도 유효한 prediction은 유지되는지 확인 +test("keeps valid predictions when batch contains invalid item", () => { + const predictions = validateAiPredictionBatchResponse({ + predictions: [ + validResponse("feature/a"), + { + ...validResponse("feature/b"), + confidence: 120 + } + ] + }) + + assert.deepEqual(predictions.map(prediction => prediction.branchName), ["feature/a"]) +}) + // confidence가 0-100 범위를 벗어나면 AI 응답을 거부하는지 확인 test("rejects confidence outside report range", () => { assert.throws( diff --git a/tests/ai/predictionRunner.test.ts b/tests/ai/predictionRunner.test.ts index 8e33cdc..f9c145e 100644 --- a/tests/ai/predictionRunner.test.ts +++ b/tests/ai/predictionRunner.test.ts @@ -109,7 +109,28 @@ test("records failed result when response is invalid", async () => { ], client) assert.equal(result?.status, "failed") - assert.match(result?.status === "failed" ? result.errorMessage : "", /confidence/) + assert.match(result?.status === "failed" ? result.errorMessage : "", /AI prediction response is missing/) +}) + +// batch 응답 중 일부만 검증에 실패하면 해당 branch만 failed 처리하는지 확인 +test("keeps valid batch predictions when one response item is invalid", async () => { + const client = new AiPredictionClientSpy({ + predictions: [ + validResponse("feature/a"), + { + ...(validResponse("feature/b") as Record), + confidence: 120 + } + ] + }) + const results = await predictMergeRisksWithAi([ + payload("feature/a", 100, BranchRiskStatus.Critical), + payload("feature/b", 100, BranchRiskStatus.Critical) + ], client) + + assert.equal(results[0]?.status, "predicted") + assert.equal(results[1]?.status, "failed") + assert.match(results[1]?.status === "failed" ? results[1].errorMessage : "", /AI prediction response is missing/) }) // prompt builder 옵션이 runner를 통해 AI client까지 전달되는지 확인 From 26e37551485eb0c386175e91908918302c440c78 Mon Sep 17 00:00:00 2001 From: opficdev <162981733+opficdev@users.noreply.github.com> Date: Tue, 23 Jun 2026 23:32:24 +0900 Subject: [PATCH 8/8] =?UTF-8?q?refactor:=20AI=20prediction=20=EC=9D=91?= =?UTF-8?q?=EB=8B=B5=20=ED=95=84=EB=93=9C=20=EC=B6=95=EC=95=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ai/geminiPredictionClient.ts | 8 +-- src/ai/predictionResponseValidator.ts | 20 +------ src/ai/promptTemplates.ts | 14 ++--- src/ai/types.ts | 2 - tests/ai/geminiPredictionClient.test.ts | 13 +++-- tests/ai/predictionPromptBuilder.test.ts | 5 +- tests/ai/predictionResponseValidator.test.ts | 58 ++++++-------------- tests/ai/predictionRunner.test.ts | 20 ++++--- tests/ai/types.test.ts | 5 +- tests/reports/markdownFormatter.test.ts | 4 +- tests/reports/reportBuilder.test.ts | 4 +- 11 files changed, 49 insertions(+), 104 deletions(-) diff --git a/src/ai/geminiPredictionClient.ts b/src/ai/geminiPredictionClient.ts index 789abaa..c5af5c1 100644 --- a/src/ai/geminiPredictionClient.ts +++ b/src/ai/geminiPredictionClient.ts @@ -319,21 +319,15 @@ function aiPredictionSchema(): Record { branchName: { type: "string" }, baseBranch: { type: "string" }, prediction: { type: "string" }, - confidence: { type: "number" }, recommendedActions: { type: "array", items: recommendedActionSchema() - }, - falsePositiveNotes: { - type: "array", - items: { type: "string" } } }, required: [ "branchName", "baseBranch", - "prediction", - "confidence" + "prediction" ] } } diff --git a/src/ai/predictionResponseValidator.ts b/src/ai/predictionResponseValidator.ts index 2e29284..d52febf 100644 --- a/src/ai/predictionResponseValidator.ts +++ b/src/ai/predictionResponseValidator.ts @@ -41,11 +41,8 @@ function predictionFor( branchName: stringFor(value.branchName, `${path}.branchName`), baseBranch: stringFor(value.baseBranch, `${path}.baseBranch`), prediction: stringFor(value.prediction, `${path}.prediction`), - confidence: confidenceFor(value.confidence), recommendedActions: arrayFor(value.recommendedActions ?? [], `${path}.recommendedActions`) - .map((action, index) => recommendedActionFor(action, index)), - falsePositiveNotes: arrayFor(value.falsePositiveNotes ?? [], `${path}.falsePositiveNotes`) - .map((note, index) => stringFor(note, `${path}.falsePositiveNotes[${index}]`)) + .map((action, index) => recommendedActionFor(action, index)) } } @@ -69,21 +66,6 @@ function recommendedActionFor( } } -// confidence는 report에서 비교할 수 있도록 0-100 범위의 정수로 제한 -function confidenceFor(value: unknown): number { - if ( - typeof value !== "number" || - !Number.isFinite(value) || - !Number.isInteger(value) || - value < 0 || - 100 < value - ) { - throw new Error("AI prediction response confidence must be an integer between 0 and 100") - } - - return value -} - // action priority가 Watcher가 표시할 수 있는 허용 값인지 검증 function priorityFor( value: unknown, diff --git a/src/ai/promptTemplates.ts b/src/ai/promptTemplates.ts index 4602c35..17da04b 100644 --- a/src/ai/promptTemplates.ts +++ b/src/ai/promptTemplates.ts @@ -6,21 +6,18 @@ export const DEFAULT_AI_PREDICTION_SYSTEM_PROMPT = [ "Do not describe the deterministic score as a probability or percentage.", "Use probabilistic wording. Avoid phrases such as guaranteed, will cause, or will result for possibility-based risks.", "Predict practical merge risk impact and recommend next actions.", - "Write prediction, recommended action titles, descriptions, and false positive notes in Korean.", - "If the evidence is weak, explain possible false positives.", + "Write prediction, recommended action titles, and descriptions in Korean.", "Return only JSON with this shape:", "{", " \"branchName\": string,", " \"baseBranch\": string,", " \"prediction\": string,", - " \"confidence\": integer from 0 to 100. Use 98 for high confidence, not 0.98 or 1,", " \"recommendedActions\": [{", " \"title\": string,", " \"description\": string,", " \"priority\": \"low\" | \"medium\" | \"high\",", " \"files\": string[]", - " }],", - " \"falsePositiveNotes\": string[]", + " }]", "}" ].join("\n") @@ -33,22 +30,19 @@ export const DEFAULT_AI_PREDICTION_BATCH_SYSTEM_PROMPT = [ "Use probabilistic wording. Avoid phrases such as guaranteed, will cause, or will result for possibility-based risks.", "Compare the provided branches together and predict practical merge risk impact.", "Recommend next actions for each branch.", - "Write prediction, recommended action titles, descriptions, and false positive notes in Korean.", - "If the evidence is weak, explain possible false positives.", + "Write prediction, recommended action titles, and descriptions in Korean.", "Return only JSON with this shape:", "{", " \"predictions\": [{", " \"branchName\": string,", " \"baseBranch\": string,", " \"prediction\": string,", - " \"confidence\": integer from 0 to 100. Use 98 for high confidence, not 0.98 or 1,", " \"recommendedActions\": [{", " \"title\": string,", " \"description\": string,", " \"priority\": \"low\" | \"medium\" | \"high\",", " \"files\": string[]", - " }],", - " \"falsePositiveNotes\": string[]", + " }]", " }]", "}" ].join("\n") diff --git a/src/ai/types.ts b/src/ai/types.ts index 14178e5..8477d02 100644 --- a/src/ai/types.ts +++ b/src/ai/types.ts @@ -67,9 +67,7 @@ export type AiPrediction = { branchName: string baseBranch: string prediction: string - confidence: number recommendedActions: AiRecommendedAction[] - falsePositiveNotes: string[] } // AI가 제안하는 다음 action과 그 action이 필요한 근거 diff --git a/tests/ai/geminiPredictionClient.test.ts b/tests/ai/geminiPredictionClient.test.ts index 510510f..89efaa9 100644 --- a/tests/ai/geminiPredictionClient.test.ts +++ b/tests/ai/geminiPredictionClient.test.ts @@ -101,7 +101,11 @@ test("sends batch response schema to Gemini", async () => { text: { schema: { properties: { - predictions?: unknown + predictions?: { + items?: { + properties?: Record + } + } } } } @@ -110,6 +114,9 @@ test("sends batch response schema to Gemini", async () => { } assert.notEqual(body.generationConfig.responseFormat.text.schema.properties.predictions, undefined) + const properties = body.generationConfig.responseFormat.text.schema.properties.predictions?.items?.properties + assert.equal(properties?.confidence, undefined) + assert.equal(properties?.falsePositiveNotes, undefined) }) // Gemini 503 계열 일시 실패는 exponential backoff 후 재시도하는지 확인 @@ -423,13 +430,11 @@ function validPrediction(): unknown { branchName: "feature/a", baseBranch: "main", prediction: "shared file 변경이 겹쳐 rebase 확인이 필요함", - confidence: 82, recommendedActions: [{ title: "base branch rebase", description: "shared file 변경을 먼저 rebase해 실제 conflict 여부를 확인함", priority: "high", files: ["src/shared.ts"] - }], - falsePositiveNotes: [] + }] } } diff --git a/tests/ai/predictionPromptBuilder.test.ts b/tests/ai/predictionPromptBuilder.test.ts index fa680b3..9e1eec2 100644 --- a/tests/ai/predictionPromptBuilder.test.ts +++ b/tests/ai/predictionPromptBuilder.test.ts @@ -19,10 +19,11 @@ test("builds prompt that preserves deterministic possibility", () => { assert.match(prompt.systemPrompt, /Do not recalculate or overwrite/) assert.match(prompt.systemPrompt, /Do not describe the deterministic score as a probability/) assert.match(prompt.systemPrompt, /Avoid phrases such as guaranteed, will cause, or will result/) - assert.match(prompt.systemPrompt, /Write prediction, recommended action titles, descriptions, and false positive notes in Korean/) + assert.match(prompt.systemPrompt, /Write prediction, recommended action titles, and descriptions in Korean/) assert.match(prompt.systemPrompt, /Return only JSON/) - assert.match(prompt.systemPrompt, /Use 98 for high confidence, not 0\.98 or 1/) assert.match(prompt.systemPrompt, /recommendedActions/) + assert.doesNotMatch(prompt.systemPrompt, /confidence/) + assert.doesNotMatch(prompt.systemPrompt, /falsePositiveNotes/) }) // user prompt가 raw diff 대신 정제된 evidence만 JSON으로 전달하는지 확인 diff --git a/tests/ai/predictionResponseValidator.test.ts b/tests/ai/predictionResponseValidator.test.ts index 2a83f84..718cdf5 100644 --- a/tests/ai/predictionResponseValidator.test.ts +++ b/tests/ai/predictionResponseValidator.test.ts @@ -11,22 +11,16 @@ test("validates ai prediction response", () => { branchName: "feature/watch", baseBranch: "main", prediction: "shared module 변경 의도가 겹쳐 rebase 우선 확인이 필요함", - confidence: 82, recommendedActions: [{ title: "base branch rebase", description: "shared.ts 변경을 먼저 rebase해 실제 conflict 여부를 확인함", priority: "high", files: ["src/shared.ts"] - }], - falsePositiveNotes: ["서로 다른 export만 수정했다면 실제 conflict 가능성은 낮아질 수 있음"] + }] }) assert.equal(prediction.branchName, "feature/watch") - assert.equal(prediction.confidence, 82) assert.equal(prediction.recommendedActions[0]?.priority, "high") - assert.deepEqual(prediction.falsePositiveNotes, [ - "서로 다른 export만 수정했다면 실제 conflict 가능성은 낮아질 수 있음" - ]) }) // batch AI prediction JSON이 branch별 prediction 배열이면 그대로 통과하는지 확인 @@ -50,8 +44,14 @@ test("keeps valid predictions when batch contains invalid item", () => { predictions: [ validResponse("feature/a"), { - ...validResponse("feature/b"), - confidence: 120 + branchName: "feature/b", + baseBranch: "main", + prediction: "shared module 변경 의도가 겹쳐 rebase 우선 확인이 필요함", + recommendedActions: [{ + title: "base branch rebase", + description: "shared.ts 변경을 먼저 rebase해 실제 conflict 여부를 확인함", + priority: "urgent" + }] } ] }) @@ -59,28 +59,6 @@ test("keeps valid predictions when batch contains invalid item", () => { assert.deepEqual(predictions.map(prediction => prediction.branchName), ["feature/a"]) }) -// confidence가 0-100 범위를 벗어나면 AI 응답을 거부하는지 확인 -test("rejects confidence outside report range", () => { - assert.throws( - () => validateAiPredictionResponse({ - ...validResponse(), - confidence: 120 - }), - /confidence/ - ) -}) - -// confidence가 소수면 report 의미가 모호하므로 AI 응답을 거부하는지 확인 -test("rejects fractional confidence", () => { - assert.throws( - () => validateAiPredictionResponse({ - ...validResponse(), - confidence: 0.98 - }), - /integer/ - ) -}) - // action priority가 허용된 값이 아니면 AI 응답을 거부하는지 확인 test("rejects invalid action priority", () => { assert.throws( @@ -96,28 +74,26 @@ test("rejects invalid action priority", () => { ) }) -// optional 배열 field가 누락되거나 null이면 빈 배열로 보정되는지 확인 -test("defaults nullish optional arrays", () => { +// recommendedActions가 누락되거나 null이면 빈 배열로 보정되는지 확인 +test("defaults nullish recommended actions", () => { const prediction = validateAiPredictionResponse({ branchName: "feature/watch", baseBranch: "main", prediction: "shared module 변경 의도가 겹쳐 rebase 우선 확인이 필요함", - confidence: 82, recommendedActions: null }) assert.deepEqual(prediction.recommendedActions, []) - assert.deepEqual(prediction.falsePositiveNotes, []) }) -// optional 배열 field가 배열이 아닌 값이면 AI 응답을 거부하는지 확인 -test("rejects non-array optional arrays", () => { +// recommendedActions가 배열이 아닌 값이면 AI 응답을 거부하는지 확인 +test("rejects non-array recommended actions", () => { assert.throws( () => validateAiPredictionResponse({ ...validResponse(), - falsePositiveNotes: "none" + recommendedActions: "none" }), - /falsePositiveNotes/ + /recommendedActions/ ) }) @@ -157,13 +133,11 @@ function validResponse(branchName = "feature/watch"): Record { branchName, baseBranch: "main", prediction: "shared module 변경 의도가 겹쳐 rebase 우선 확인이 필요함", - confidence: 82, recommendedActions: [{ title: "base branch rebase", description: "shared.ts 변경을 먼저 rebase해 실제 conflict 여부를 확인함", priority: "high", files: ["src/shared.ts"] - }], - falsePositiveNotes: [] + }] } } diff --git a/tests/ai/predictionRunner.test.ts b/tests/ai/predictionRunner.test.ts index f9c145e..4763d92 100644 --- a/tests/ai/predictionRunner.test.ts +++ b/tests/ai/predictionRunner.test.ts @@ -98,10 +98,12 @@ test("records failed result when response is invalid", async () => { predictions: [{ branchName: "feature/critical", baseBranch: "main", - prediction: "invalid confidence", - confidence: 120, - recommendedActions: [], - falsePositiveNotes: [] + prediction: "invalid priority", + recommendedActions: [{ + title: "base branch rebase", + description: "shared.ts 확인", + priority: "urgent" + }] }] }) const [result] = await predictMergeRisksWithAi([ @@ -119,7 +121,11 @@ test("keeps valid batch predictions when one response item is invalid", async () validResponse("feature/a"), { ...(validResponse("feature/b") as Record), - confidence: 120 + recommendedActions: [{ + title: "base branch rebase", + description: "shared.ts 확인", + priority: "urgent" + }] } ] }) @@ -236,13 +242,11 @@ function validResponse(branchName: string): unknown { branchName, baseBranch: "main", prediction: "shared module 변경 의도가 겹쳐 rebase 우선 확인이 필요함", - confidence: 82, recommendedActions: [{ title: "base branch rebase", description: "shared.ts 변경을 먼저 rebase해 실제 conflict 여부를 확인함", priority: "high", files: ["src/shared.ts"] - }], - falsePositiveNotes: [] + }] } } diff --git a/tests/ai/types.test.ts b/tests/ai/types.test.ts index d1bcd7b..49b3b1b 100644 --- a/tests/ai/types.test.ts +++ b/tests/ai/types.test.ts @@ -14,18 +14,15 @@ test("models ai prediction separately from deterministic possibility", () => { branchName: "feature/watch", baseBranch: "main", prediction: "shared module 변경 의도가 겹쳐 rebase 우선 확인이 필요함", - confidence: 82, recommendedActions: [{ title: "base branch rebase", description: "shared.ts 변경을 먼저 rebase해 실제 conflict 여부를 확인함", priority: "high", files: ["src/shared.ts"] - }], - falsePositiveNotes: ["서로 다른 export만 수정했다면 실제 conflict 가능성은 낮아질 수 있음"] + }] } assert.equal(prediction.branchName, "feature/watch") - assert.equal(prediction.confidence, 82) assert.equal(prediction.recommendedActions[0]?.priority, "high") }) diff --git a/tests/reports/markdownFormatter.test.ts b/tests/reports/markdownFormatter.test.ts index 2fa2dd2..14fc931 100644 --- a/tests/reports/markdownFormatter.test.ts +++ b/tests/reports/markdownFormatter.test.ts @@ -217,14 +217,12 @@ function predictedAiResult(branchName: string): AiPredictionResult { branchName, baseBranch: "main", prediction: "공유 파일 변경 의도가 겹칠 가능성 있음", - confidence: 82, recommendedActions: [{ title: "base branch rebase", description: "최신 main 기준으로 rebase 후 실제 충돌 여부 확인", priority: "high", files: ["src/shared.ts"] - }], - falsePositiveNotes: ["파일은 같지만 line range가 다르면 false positive 가능성 있음"] + }] } } } diff --git a/tests/reports/reportBuilder.test.ts b/tests/reports/reportBuilder.test.ts index 829153a..b07c5c0 100644 --- a/tests/reports/reportBuilder.test.ts +++ b/tests/reports/reportBuilder.test.ts @@ -219,14 +219,12 @@ function predictedAiResult(branchName: string): AiPredictionResult { branchName, baseBranch: "main", prediction: "공유 파일 변경 의도가 겹칠 가능성 있음", - confidence: 82, recommendedActions: [{ title: "base branch rebase", description: "최신 main 기준으로 rebase 후 실제 충돌 여부 확인", priority: "high", files: ["src/shared.ts"] - }], - falsePositiveNotes: ["파일은 같지만 line range가 다르면 false positive 가능성 있음"] + }] } } }