diff --git a/README.md b/README.md index 99f3c8b..7f15d0c 100644 --- a/README.md +++ b/README.md @@ -107,14 +107,16 @@ AI prediction은 deterministic possibility score를 대체하지 않습니다. W 기본 AI provider는 Gemini API입니다. consumer repository에는 `GEMINI_API_KEY` secret을 설정해야 합니다. -AI prediction 대상은 기본적으로 `medium` 이상 possibility입니다. 현재 기준으로 score가 25점 이상인 branch만 AI prediction 대상으로 선택됩니다. 낮은 score의 branch는 AI 호출을 생략하고 `skipped` 상태로 report에 표시됩니다. +AI prediction 대상은 기본적으로 `critical` possibility입니다. 이미 virtual merge에서 conflict가 확정된 branch는 AI 호출 없이 deterministic report만 사용합니다. 그 외 낮은 status의 branch는 AI 호출을 생략하고 `skipped` 상태로 report에 표시됩니다. + +Gemini free tier rate limit을 줄이기 위해 선택된 branch prediction은 순차 실행하며 기본적으로 호출 사이에 60초를 대기합니다. AI prediction 결과는 다음 상태 중 하나입니다. | status | 의미 | | --- | --- | | `predicted` | Gemini API 응답을 검증했고 prediction과 recommended actions를 report에 포함함 | -| `skipped` | deterministic possibility score가 threshold보다 낮아 AI 호출을 생략함 | +| `skipped` | AI prediction 대상이 아니어서 provider 호출을 생략함 | | `failed` | provider 호출이나 응답 검증에 실패해 branch 단위 실패로 격리함 | AI provider가 실패해도 deterministic possibility report는 유지됩니다. 실패한 branch는 `failed` 상태와 error message를 report에 포함합니다. @@ -187,7 +189,7 @@ npm test 5. stdout report가 정상일 때만 consumer repository secret에 `DISCORD_WEBHOOK_URL`을 추가하고 같은 workflow를 다시 수동 실행합니다. -Discord 메시지가 정상적으로 도착하면 consumer repository의 예시 workflow에 `schedule`과 `pull_request` trigger를 유지해 실서비스 실행으로 전환합니다. +Discord 메시지가 정상적으로 도착하면 consumer repository의 예시 workflow에 `schedule` trigger를 유지해 실서비스 실행으로 전환합니다. ## Local test와 scheduled run 차이 @@ -199,12 +201,11 @@ scheduled run은 consumer repository의 실제 remote branch를 fetch하고, `ba | 증상 | 확인할 항목 | | --- | --- | -| workflow가 시작되지 않음 | consumer workflow가 `pull_request`, `schedule`, `workflow_dispatch` 중 필요한 trigger를 가지고 있는지 확인 | -| fork PR에서 실행되지 않음 | 예시 workflow의 `github.event.pull_request.head.repo.fork == false` 조건 확인 | +| workflow가 시작되지 않음 | consumer workflow가 `schedule`, `workflow_dispatch` 중 필요한 trigger를 가지고 있는지 확인 | | checkout 또는 fetch 실패 | `WATCHER_GITHUB_TOKEN` 권한과 `contents: read` permission 확인 | | PR metadata가 비어 있음 | `pull-requests: read` permission과 commit에 연결된 PR 존재 여부 확인 | | check metadata가 비어 있음 | `checks: read` permission과 해당 branch head SHA의 check run 존재 여부 확인 | -| AI prediction이 `skipped`로 표시됨 | deterministic possibility score가 기본 threshold인 25점 미만인지 확인 | -| AI prediction이 `failed`로 표시됨 | `GEMINI_API_KEY` secret과 Gemini API 응답 형식 확인 | +| AI prediction이 `skipped`로 표시됨 | deterministic possibility status가 `critical`인지와 `confirmed_conflict`가 아닌지 확인 | +| AI prediction이 `failed`로 표시됨 | `GEMINI_API_KEY` secret, Gemini API 응답 형식, rate limit 상태 확인 | | Discord 전송이 되지 않음 | `DISCORD_WEBHOOK_URL` secret과 webhook channel 권한 확인 | | merge된 branch가 계속 감시됨 | GitHub `Automatically delete head branches` 설정과 원격 branch 삭제 상태 확인 | diff --git a/src/ai/geminiPredictionClient.ts b/src/ai/geminiPredictionClient.ts index d09fdde..a18efd1 100644 --- a/src/ai/geminiPredictionClient.ts +++ b/src/ai/geminiPredictionClient.ts @@ -6,18 +6,37 @@ import type { // workflow log가 과도하게 커지지 않도록 Gemini 실패 응답 본문을 제한 const MAX_GEMINI_ERROR_DETAIL_LENGTH = 1000 +// Gemini 서버 capacity나 일시적 내부 오류는 backoff 후 재시도 가능 +const RETRYABLE_GEMINI_STATUS_CODES = new Set([429, 500, 502, 503, 504]) + +// Gemini 일시 실패가 장기 장애인지 판단하기 전까지 허용할 총 요청 횟수 +const DEFAULT_GEMINI_MAX_ATTEMPTS = 5 + +// 503 계열 일시 실패의 exponential backoff 시작 대기 시간 +const DEFAULT_GEMINI_RETRY_BASE_DELAY_MS = 2000 + +// 429 rate limit은 서버 capacity와 별도로 더 길게 대기 +const DEFAULT_GEMINI_RATE_LIMIT_DELAY_MS = 60000 + +// 같은 시점에 retry가 몰리지 않도록 추가할 최대 jitter +const DEFAULT_GEMINI_RETRY_JITTER_MS = 1000 + // Watcher가 Gemini prediction에 기본으로 사용할 model 이름 export const DEFAULT_GEMINI_PREDICTION_MODEL = "gemini-3.5-flash" // consumer repository에서 Gemini API key를 주입할 environment variable 이름 export const GEMINI_API_KEY_ENV_NAME = "GEMINI_API_KEY" -// 테스트와 운영 환경에서 Gemini 연결값을 바꿔 끼우기 위한 설정 +// 호출 환경에 맞게 Gemini 연결값을 바꿔 끼우기 위한 설정 export type GeminiPredictionClientOptions = { apiKey?: string model?: string endpoint?: string fetch?: typeof fetch + maxAttempts?: number + retryBaseDelayMs?: number + rateLimitDelayMs?: number + retryJitterMs?: number } // Watcher가 사용하는 Gemini generateContent 응답 중 JSON text 추출에 필요한 최소 shape @@ -39,12 +58,24 @@ export class GeminiPredictionClient implements AiPredictionClient { // generateContent endpoint에 포함할 Gemini model 이름 private readonly model: string - // 테스트와 API version 교체를 위해 분리한 Gemini REST base URL + // API version 교체와 proxy 구성을 위해 분리한 Gemini REST base URL private readonly endpoint: string - // 테스트에서 network 없이 검증할 수 있도록 주입 가능한 fetch 구현 + // 실행 환경별 HTTP 호출 구현을 바꿔 끼우기 위한 fetch 구현 private readonly fetcher: typeof fetch + // 일시 실패가 지속될 때 최대 몇 번까지 요청할지 결정 + private readonly maxAttempts: number + + // 503 계열 일시 실패의 exponential backoff 시작 대기 시간 + private readonly retryBaseDelayMs: number + + // 429 rate limit 응답 후 다음 요청까지 기다릴 시간 + private readonly rateLimitDelayMs: number + + // retry 요청이 동시에 몰리는 것을 피하기 위한 최대 jitter + private readonly retryJitterMs: number + // 명시 options 또는 GEMINI_API_KEY 환경 변수로 Gemini client를 구성 constructor(options: GeminiPredictionClientOptions = {}) { const apiKey = options.apiKey ?? process.env[GEMINI_API_KEY_ENV_NAME] @@ -57,31 +88,68 @@ export class GeminiPredictionClient implements AiPredictionClient { this.model = options.model ?? DEFAULT_GEMINI_PREDICTION_MODEL this.endpoint = options.endpoint ?? "https://generativelanguage.googleapis.com/v1beta" this.fetcher = options.fetch ?? fetch + this.maxAttempts = Math.max(1, options.maxAttempts ?? DEFAULT_GEMINI_MAX_ATTEMPTS) + this.retryBaseDelayMs = options.retryBaseDelayMs ?? DEFAULT_GEMINI_RETRY_BASE_DELAY_MS + this.rateLimitDelayMs = options.rateLimitDelayMs ?? DEFAULT_GEMINI_RATE_LIMIT_DELAY_MS + this.retryJitterMs = options.retryJitterMs ?? DEFAULT_GEMINI_RETRY_JITTER_MS } // system/user prompt를 Gemini JSON 응답 요청으로 변환하고 parsed JSON을 반환 async predict(prompt: AiPredictionPrompt): Promise { - const response = await this.fetcher(this.url(), { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-goog-api-key": this.apiKey - }, - body: JSON.stringify(geminiRequestBodyFor(prompt)) - }) + const body = JSON.stringify(geminiRequestBodyFor(prompt)) + let lastResponse: Response | undefined + + for (let attempt = 0; attempt < this.maxAttempts; attempt += 1) { + const response = await this.fetcher(this.url(), { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-goog-api-key": this.apiKey + }, + body + }) + + if (response.ok) { + const value = await response.json() as GeminiGenerateContentResponse + return JSON.parse(geminiTextFor(value)) + } + + lastResponse = response - if (!response.ok) { - throw new Error(await geminiRequestErrorMessageFor(response)) + if (!this.shouldRetry(response.status, attempt)) { + break + } + + await delay(this.retryDelayFor(response.status, attempt)) + } + + if (!lastResponse) { + throw new Error("Gemini prediction request failed without response") } - const value = await response.json() as GeminiGenerateContentResponse - return JSON.parse(geminiTextFor(value)) + throw new Error(await geminiRequestErrorMessageFor(lastResponse)) } // Gemini model별 generateContent 호출 URL을 구성 private url(): string { return `${this.endpoint}/models/${this.model}:generateContent` } + + // retry 가능한 status이고 다음 시도 횟수가 남아 있는지 확인 + private shouldRetry(status: number, attempt: number): boolean { + return attempt + 1 < this.maxAttempts && RETRYABLE_GEMINI_STATUS_CODES.has(status) + } + + // 429는 더 길게 대기하고 503 계열은 exponential backoff로 간격을 늘림 + private retryDelayFor(status: number, attempt: number): number { + const jitter = Math.floor(Math.random() * this.retryJitterMs) + + if (status === 429) { + return this.rateLimitDelayMs + jitter + } + + return (2 ** attempt) * this.retryBaseDelayMs + jitter + } } // Watcher 기본 AI provider를 Gemini client로 조립 @@ -112,6 +180,15 @@ function geminiRequestBodyFor(prompt: AiPredictionPrompt): Record { + if (delayMs <= 0) { + return Promise.resolve() + } + + return new Promise(resolve => setTimeout(resolve, delayMs)) +} + // Gemini API가 반환한 HTTP 실패 원문을 보존해 workflow log에서 원인을 확인 async function geminiRequestErrorMessageFor(response: Response): Promise { const detail = await geminiErrorDetailFor(response) @@ -125,6 +202,13 @@ async function geminiErrorDetailFor(response: Response): Promise value !== undefined).join(", ") +} + +// JSON error body에서 Gemini가 제공한 message만 추출 +function geminiErrorMessageFor(text: string): string | undefined { + try { + const value = JSON.parse(text) as { + error?: { + message?: unknown + } + } + + return typeof value.error?.message === "string" ? value.error.message : undefined + } catch { + return undefined + } +} + +// "key: value" 형태의 Gemini quota message에서 한 항목만 추출 +function valueAfter(message: string, key: string): string | undefined { + const index = message.indexOf(key) + + if (index === -1) { + return undefined + } + + const value = message.slice(index + key.length).trim() + const end = value.search(/[,\n]/) + const trimmed = (end === -1 ? value : value.slice(0, end)).trim() + + return trimmed || undefined +} + +// Gemini quota message의 retry 안내 시간을 추출 +function retryDelayTextFor(message: string): string | undefined { + const match = message.match(/Please retry in ([0-9]+(?:\.[0-9]+)?s)/) + + return match?.[1] +} + // Gemini 응답 part가 여러 개로 나뉘어도 하나의 JSON 문자열로 합침 function geminiTextFor(response: GeminiGenerateContentResponse): string { const text = response.candidates?.[0]?.content?.parts diff --git a/src/ai/predictionResponseValidator.ts b/src/ai/predictionResponseValidator.ts index f291617..3616a43 100644 --- a/src/ai/predictionResponseValidator.ts +++ b/src/ai/predictionResponseValidator.ts @@ -46,10 +46,16 @@ function recommendedActionFor( } } -// confidence는 report에서 비교할 수 있도록 0-100 범위의 숫자로 제한 +// confidence는 report에서 비교할 수 있도록 0-100 범위의 정수로 제한 function confidenceFor(value: unknown): number { - if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || 100 < value) { - throw new Error("AI prediction response confidence must be a number between 0 and 100") + if ( + typeof value !== "number" || + !Number.isFinite(value) || + !Number.isInteger(value) || + value < 0 || + 100 < value + ) { + throw new Error("AI prediction response confidence must be an integer between 0 and 100") } return value diff --git a/src/ai/predictionRunner.ts b/src/ai/predictionRunner.ts index 4649976..0ba8c4e 100644 --- a/src/ai/predictionRunner.ts +++ b/src/ai/predictionRunner.ts @@ -8,26 +8,38 @@ import type { AiPredictionRunOptions } from "./types.js" +// Gemini free tier rate limit을 줄이기 위해 선택된 AI 호출 사이에 둘 내부 대기 시간 +const DEFAULT_AI_PREDICTION_DELAY_MS = 60000 + // 대상 선택, prompt 생성, provider 호출, 응답 검증을 branch별 AI prediction 결과로 연결 export async function predict( payloads: AiPredictionEvidencePayload[], client: AiPredictionClient, options: AiPredictionRunOptions = {} ): Promise { - const targets = new Set(selectTargets(payloads, options)) + const targets = new Set(selectTargets(payloads)) + const results: AiPredictionResult[] = [] - return Promise.all(payloads.map(async (payload): Promise => { + for (const payload of payloads) { if (!targets.has(payload)) { - return { + results.push({ status: "skipped", branchName: payload.branch.name, baseBranch: payload.branch.baseBranch, - reason: "below_threshold" - } + reason: skippedReasonFor(payload) + }) + continue + } + + results.push(await predictedResultFor(payload, client, options)) + targets.delete(payload) + + if (0 < targets.size) { + await delay(DEFAULT_AI_PREDICTION_DELAY_MS) } + } - return predictedResultFor(payload, client, options) - })) + return results } // provider 호출과 schema 검증 실패를 branch 단위 failed 결과로 격리 @@ -61,3 +73,19 @@ async function predictedResultFor( function errorMessageFor(error: unknown): string { return error instanceof Error ? error.message : String(error) } + +// AI prediction 대상이 아닌 branch의 생략 사유를 report 가능한 값으로 변환 +function skippedReasonFor(payload: AiPredictionEvidencePayload): "not_target" | "confirmed_conflict" { + return payload.possibility.reasons.some(reason => reason.code === "confirmed_conflict") + ? "confirmed_conflict" + : "not_target" +} + +// 선택된 AI provider 호출 사이에 간격을 두어 rate limit 진입 가능성을 낮춤 +function delay(delayMs: number): Promise { + if (delayMs <= 0) { + return Promise.resolve() + } + + return new Promise(resolve => setTimeout(resolve, delayMs)) +} diff --git a/src/ai/predictionTargetSelector.ts b/src/ai/predictionTargetSelector.ts index f43f33c..715d6ba 100644 --- a/src/ai/predictionTargetSelector.ts +++ b/src/ai/predictionTargetSelector.ts @@ -1,17 +1,17 @@ import type { - AiPredictionEvidencePayload, - AiPredictionTargetSelectionOptions + AiPredictionEvidencePayload } from "./types.js" +import { BranchRiskStatus } from "../risks/types.js" -// medium 이상 possibility를 기본 AI prediction 대상으로 삼기 위한 최소 score -export const DEFAULT_AI_PREDICTION_MINIMUM_SCORE = 25 +// Gemini 호출량을 줄이기 위해 기본 AI prediction 대상은 critical possibility로 제한 +export const DEFAULT_AI_PREDICTION_TARGET_STATUS = BranchRiskStatus.Critical -// deterministic possibility score 기준으로 AI prediction 대상 evidence만 선택 +// deterministic possibility status 기준으로 AI prediction 대상 evidence만 선택 export function select( - payloads: AiPredictionEvidencePayload[], - options: AiPredictionTargetSelectionOptions = {} + payloads: AiPredictionEvidencePayload[] ): AiPredictionEvidencePayload[] { - const minimumScore = options.minimumScore ?? DEFAULT_AI_PREDICTION_MINIMUM_SCORE - - return payloads.filter(payload => minimumScore <= payload.possibility.score) + return payloads.filter(payload => + payload.possibility.status === DEFAULT_AI_PREDICTION_TARGET_STATUS && + !payload.possibility.reasons.some(reason => reason.code === "confirmed_conflict") + ) } diff --git a/src/ai/promptTemplates.ts b/src/ai/promptTemplates.ts index 7346f23..37920ea 100644 --- a/src/ai/promptTemplates.ts +++ b/src/ai/promptTemplates.ts @@ -3,14 +3,17 @@ export const DEFAULT_AI_PREDICTION_SYSTEM_PROMPT = [ "You are Watcher's merge risk prediction assistant.", "Use only the provided deterministic evidence.", "Do not recalculate or overwrite the possibility score, status, or reasons.", + "Do not describe the deterministic score as a probability or percentage.", + "Use probabilistic wording. Avoid phrases such as guaranteed, will cause, or will result for possibility-based risks.", "Predict practical merge risk impact and recommend next actions.", + "Write prediction, recommended action titles, descriptions, and false positive notes in Korean.", "If the evidence is weak, explain possible false positives.", "Return only JSON with this shape:", "{", " \"branchName\": string,", " \"baseBranch\": string,", " \"prediction\": string,", - " \"confidence\": number,", + " \"confidence\": integer from 0 to 100. Use 98 for high confidence, not 0.98 or 1,", " \"recommendedActions\": [{", " \"title\": string,", " \"description\": string,", diff --git a/src/ai/types.ts b/src/ai/types.ts index 9f8aca7..bae00ef 100644 --- a/src/ai/types.ts +++ b/src/ai/types.ts @@ -10,11 +10,6 @@ export type AiPredictionEvidencePayload = { changedHunks: BranchChangedHunk[] } -// AI prediction 비용을 줄이기 위해 deterministic score 기준으로 대상 branch를 제한하는 설정 -export type AiPredictionTargetSelectionOptions = { - minimumScore?: number -} - // AI provider에 전달할 system/user prompt 묶음 export type AiPredictionPrompt = { systemPrompt: string @@ -31,10 +26,8 @@ export type AiPredictionClient = { predict(prompt: AiPredictionPrompt): Promise } -// AI prediction runner가 대상 선택과 prompt 생성을 조정하기 위한 설정 -export type AiPredictionRunOptions = - AiPredictionTargetSelectionOptions & - AiPredictionPromptBuildOptions +// AI prediction runner가 prompt 생성을 조정하기 위한 설정 +export type AiPredictionRunOptions = AiPredictionPromptBuildOptions // branch별 AI prediction 실행 결과 export type AiPredictionResult = @@ -53,7 +46,7 @@ export type AiPredictionSkippedResult = { status: "skipped" branchName: string baseBranch: string - reason: "below_threshold" + reason: "not_target" | "confirmed_conflict" } export type AiPredictionFailedResult = { diff --git a/src/index.ts b/src/index.ts index 0bf811c..787b754 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,7 +17,7 @@ export { build as buildAiPredictionPrompt } from "./ai/predictionPromptBuilder.j export { DEFAULT_AI_PREDICTION_SYSTEM_PROMPT } from "./ai/promptTemplates.js" export { predict as predictMergeRisksWithAi } from "./ai/predictionRunner.js" export { - DEFAULT_AI_PREDICTION_MINIMUM_SCORE, + DEFAULT_AI_PREDICTION_TARGET_STATUS, select as selectAiPredictionTargets } from "./ai/predictionTargetSelector.js" export { validate as validateAiPredictionResponse } from "./ai/predictionResponseValidator.js" @@ -45,7 +45,6 @@ export type { AiPredictionResult, AiPredictionRunOptions, AiPredictionSkippedResult, - AiPredictionTargetSelectionOptions, AiRecommendedAction, AiRecommendedActionPriority } from "./ai/types.js" diff --git a/src/reports/markdownFormatter.ts b/src/reports/markdownFormatter.ts index db050cd..9a2c60b 100644 --- a/src/reports/markdownFormatter.ts +++ b/src/reports/markdownFormatter.ts @@ -128,7 +128,7 @@ function predictedLinesFor(result: AiPredictionPredictedResult): string[] { return lines } -// threshold 미달로 AI prediction을 생략한 이유를 표시 +// AI prediction을 생략한 이유를 표시 function skippedLinesFor(result: AiPredictionSkippedResult): string[] { return [ "- ai prediction:", diff --git a/tests/ai/geminiPredictionClient.test.ts b/tests/ai/geminiPredictionClient.test.ts index 9889695..c7a587f 100644 --- a/tests/ai/geminiPredictionClient.test.ts +++ b/tests/ai/geminiPredictionClient.test.ts @@ -84,10 +84,125 @@ test("sends prompt to Gemini generateContent endpoint", async () => { assert.equal(body.generationConfig.responseFormat.text.mimeType, "APPLICATION_JSON") }) +// Gemini 503 계열 일시 실패는 exponential backoff 후 재시도하는지 확인 +test("retries Gemini high demand failures with exponential backoff", async () => { + const fetcher = fetchSequenceSpy([ + { + body: {}, + ok: false, + status: 503, + text: "high demand" + }, + { + body: {}, + ok: false, + status: 503, + text: "high demand" + }, + { + body: validGeminiResponse() + } + ]) + const client = new GeminiPredictionClient({ + apiKey: "gemini-key", + fetch: fetcher, + maxAttempts: 3, + retryBaseDelayMs: 0, + retryJitterMs: 0 + }) + + const response = await client.predict(prompt()) + + assert.deepEqual(response, validPrediction()) + assert.equal(fetcher.requests.length, 3) +}) + +// Gemini 429 rate limit은 일반 503보다 긴 대기 후 재시도하는지 확인 +test("retries Gemini rate limit failures with longer delay", async () => { + const fetcher = fetchSequenceSpy([ + { + body: {}, + ok: false, + status: 429, + text: "rate limit" + }, + { + body: validGeminiResponse() + } + ]) + const client = new GeminiPredictionClient({ + apiKey: "gemini-key", + fetch: fetcher, + maxAttempts: 2, + rateLimitDelayMs: 0, + retryJitterMs: 0 + }) + + const response = await client.predict(prompt()) + + assert.deepEqual(response, validPrediction()) + assert.equal(fetcher.requests.length, 2) +}) + +// request body 오류처럼 non-retryable 실패는 retry 없이 즉시 실패하는지 확인 +test("does not retry non-retryable Gemini request failure", async () => { + const fetcher = fetchSequenceSpy([ + { + body: {}, + ok: false, + status: 400, + text: "invalid request" + }, + { + body: validGeminiResponse() + } + ]) + const client = new GeminiPredictionClient({ + apiKey: "gemini-key", + fetch: fetcher, + maxAttempts: 5 + }) + + await assert.rejects( + client.predict(prompt()), + /Gemini prediction request failed with status 400: invalid request/ + ) + assert.equal(fetcher.requests.length, 1) +}) + +// Gemini quota 실패는 report가 길어지지 않도록 metric과 retry 시간만 요약하는지 확인 +test("summarizes Gemini quota failure", async () => { + const client = new GeminiPredictionClient({ + apiKey: "gemini-key", + maxAttempts: 1, + fetch: fetchSpy({}, { + ok: false, + status: 429, + text: JSON.stringify({ + error: { + code: 429, + message: [ + "You exceeded your current quota.", + "Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.5-flash", + "Please retry in 51.095224543s." + ].join("\n"), + status: "RESOURCE_EXHAUSTED" + } + }) + }) + }) + + await assert.rejects( + client.predict(prompt()), + /Gemini prediction request failed with status 429: Gemini quota exceeded, metric: generativelanguage.googleapis.com\/generate_content_free_tier_requests, limit: 20, model: gemini-3.5-flash, retry after: 51.095224543s/ + ) +}) + // Gemini HTTP 실패가 branch 단위 failed result로 격리될 수 있도록 Error로 노출되는지 확인 test("throws when Gemini request fails", async () => { const client = new GeminiPredictionClient({ apiKey: "gemini-key", + maxAttempts: 1, fetch: fetchSpy({}, { ok: false, status: 429, @@ -152,6 +267,13 @@ type FetchSpy = typeof fetch & { }> } +type FetchResponse = { + body: unknown + ok?: boolean + status?: number + text?: string +} + // network 호출 없이 Gemini response와 HTTP 상태를 주입하는 fetch 대역 function fetchSpy( body: unknown, @@ -186,6 +308,44 @@ function fetchSpy( return spy } +// 여러 Gemini 응답을 순서대로 반환해 retry 흐름을 검증하는 fetch 대역 +function fetchSequenceSpy(responses: FetchResponse[]): FetchSpy { + const requests: FetchSpy["requests"] = [] + const spy = (async ( + input: string | URL | Request, + init?: RequestInit + ): Promise => { + requests.push({ + url: String(input), + init: { + headers: init?.headers as Record, + body: init?.body + } + }) + + const response = responses.shift() + + if (!response) { + throw new Error("Unexpected Gemini request") + } + + return responseFor(response) + }) as FetchSpy + + spy.requests = requests + return spy +} + +// 테스트용 response 정의를 fetch Response 최소 구현으로 변환 +function responseFor(response: FetchResponse): Response { + return { + ok: response.ok ?? true, + status: response.status ?? 200, + json: async () => response.body, + text: async () => response.text ?? JSON.stringify(response.body) + } as Response +} + // Gemini client가 전송할 system/user prompt fixture function prompt(): AiPredictionPrompt { return { diff --git a/tests/ai/predictionPromptBuilder.test.ts b/tests/ai/predictionPromptBuilder.test.ts index 15262b5..3ab8e25 100644 --- a/tests/ai/predictionPromptBuilder.test.ts +++ b/tests/ai/predictionPromptBuilder.test.ts @@ -15,7 +15,11 @@ test("builds prompt that preserves deterministic possibility", () => { assert.equal(prompt.systemPrompt, DEFAULT_AI_PREDICTION_SYSTEM_PROMPT) assert.match(prompt.systemPrompt, /Do not recalculate or overwrite/) + assert.match(prompt.systemPrompt, /Do not describe the deterministic score as a probability/) + assert.match(prompt.systemPrompt, /Avoid phrases such as guaranteed, will cause, or will result/) + assert.match(prompt.systemPrompt, /Write prediction, recommended action titles, descriptions, and false positive notes in Korean/) assert.match(prompt.systemPrompt, /Return only JSON/) + assert.match(prompt.systemPrompt, /Use 98 for high confidence, not 0\.98 or 1/) assert.match(prompt.systemPrompt, /recommendedActions/) }) diff --git a/tests/ai/predictionResponseValidator.test.ts b/tests/ai/predictionResponseValidator.test.ts index 044c907..1c4bfd6 100644 --- a/tests/ai/predictionResponseValidator.test.ts +++ b/tests/ai/predictionResponseValidator.test.ts @@ -37,6 +37,17 @@ test("rejects confidence outside report range", () => { ) }) +// confidence가 소수면 report 의미가 모호하므로 AI 응답을 거부하는지 확인 +test("rejects fractional confidence", () => { + assert.throws( + () => validateAiPredictionResponse({ + ...validResponse(), + confidence: 0.98 + }), + /integer/ + ) +}) + // action priority가 허용된 값이 아니면 AI 응답을 거부하는지 확인 test("rejects invalid action priority", () => { assert.throws( diff --git a/tests/ai/predictionRunner.test.ts b/tests/ai/predictionRunner.test.ts index 54989e9..5b28486 100644 --- a/tests/ai/predictionRunner.test.ts +++ b/tests/ai/predictionRunner.test.ts @@ -1,21 +1,23 @@ -import test from "node:test" +import test, { mock } from "node:test" import assert from "node:assert/strict" import { + BranchRiskStatus, predictMergeRisksWithAi, type AiPredictionClient, type AiPredictionEvidencePayload, type AiPredictionPrompt, type BranchContext, type BranchRisk, + type BranchRiskReasonCode, type GitMergeSignal } from "../../src/index.js" -// threshold 이상 branch만 AI client로 prediction을 요청하는지 확인 -test("predicts selected merge risk payloads", async () => { +// critical branch만 AI client로 prediction을 요청하는지 확인 +test("predicts selected critical merge risk payloads", async () => { const client = new AiPredictionClientSpy() const results = await predictMergeRisksWithAi([ - payload("feature/low", 20), - payload("feature/high", 55) + payload("feature/high", 55, BranchRiskStatus.High), + payload("feature/critical", 100, BranchRiskStatus.Critical) ], client) assert.deepEqual(results.map(result => result.status), ["skipped", "predicted"]) @@ -23,50 +25,98 @@ test("predicts selected merge risk payloads", async () => { const predicted = results[1] assert.equal( predicted?.status === "predicted" ? predicted.prediction.branchName : undefined, - "feature/high" + "feature/critical" ) }) -// custom threshold를 runner 옵션으로 전달할 수 있는지 확인 -test("uses custom prediction threshold", async () => { +// score가 높아도 critical status가 아니면 AI prediction을 생략하는지 확인 +test("skips non-critical payloads", async () => { const client = new AiPredictionClientSpy() const results = await predictMergeRisksWithAi([ - payload("feature/high", 55), - payload("feature/critical", 100) - ], client, { - minimumScore: 80 - }) + payload("feature/high", 90, BranchRiskStatus.High) + ], client) - assert.deepEqual(results.map(result => result.status), ["skipped", "predicted"]) - assert.equal(client.prompts.length, 1) - const predicted = results[1] - assert.equal( - predicted?.status === "predicted" ? predicted.prediction.branchName : undefined, - "feature/critical" - ) + assert.deepEqual(results.map(result => result.status), ["skipped"]) + assert.equal(results[0]?.status === "skipped" ? results[0].reason : undefined, "not_target") + assert.equal(client.prompts.length, 0) }) -// 선택된 branch prediction들이 서로 기다리지 않고 병렬로 시작되는지 확인 -test("starts selected predictions in parallel", async () => { - const client = new DeferredAiPredictionClient() - const running = predictMergeRisksWithAi([ - payload("feature/a", 55), - payload("feature/b", 80) +// 이미 Git conflict가 확정된 branch는 AI prediction을 생략하고 확정 충돌 사유를 기록하는지 확인 +test("records confirmed conflict skip reason", async () => { + const client = new AiPredictionClientSpy() + const [result] = await predictMergeRisksWithAi([ + payload("feature/conflict", 100, BranchRiskStatus.Critical, "confirmed_conflict") ], client) - await client.waitForPrompts(2) - assert.equal(client.prompts.length, 2) + assert.equal(result?.status, "skipped") + assert.equal(result?.status === "skipped" ? result.reason : undefined, "confirmed_conflict") + assert.equal(client.prompts.length, 0) +}) + +// Gemini 무료 등급의 일시 실패를 줄이기 위해 선택된 branch prediction을 순차 실행하는지 확인 +test("runs selected predictions sequentially", async () => { + mock.timers.enable({ apis: ["setTimeout"] }) + const client = new DeferredAiPredictionClient() + + try { + const running = predictMergeRisksWithAi([ + payload("feature/a", 100, BranchRiskStatus.Critical), + payload("feature/b", 100, BranchRiskStatus.Critical) + ], client) + + await client.waitForPrompts(1) + assert.equal(client.prompts.length, 1) + + client.resolveNext() + await flushTasks() + mock.timers.tick(60000) + await flushTasks() + assert.equal(client.prompts.length, 2) - client.resolveAll() - const results = await running - assert.deepEqual(results.map(result => result.status), ["predicted", "predicted"]) + client.resolveNext() + const results = await running + assert.deepEqual(results.map(result => result.status), ["predicted", "predicted"]) + } finally { + mock.timers.reset() + } +}) + +// 선택된 AI 호출 사이에 내부 기본 간격을 두는지 확인 +test("waits between selected predictions", async () => { + mock.timers.enable({ apis: ["setTimeout"] }) + const client = new DeferredAiPredictionClient() + + try { + const running = predictMergeRisksWithAi([ + payload("feature/a", 100, BranchRiskStatus.Critical), + payload("feature/b", 100, BranchRiskStatus.Critical) + ], client) + + await client.waitForPrompts(1) + client.resolveNext() + await flushTasks() + + mock.timers.tick(59999) + await flushTasks() + assert.equal(client.prompts.length, 1) + + mock.timers.tick(1) + await flushTasks() + assert.equal(client.prompts.length, 2) + + client.resolveNext() + await running + assert.equal(client.prompts.length, 2) + } finally { + mock.timers.reset() + } }) // AI client 오류가 전체 실행 실패가 아니라 branch 단위 failed 결과로 기록되는지 확인 test("records failed result when client throws", async () => { const client = new AiPredictionClientSpy(new Error("provider failed")) const [result] = await predictMergeRisksWithAi([ - payload("feature/high", 55) + payload("feature/critical", 100, BranchRiskStatus.Critical) ], client) assert.equal(result?.status, "failed") @@ -84,7 +134,7 @@ test("records failed result when response is invalid", async () => { falsePositiveNotes: [] }) const [result] = await predictMergeRisksWithAi([ - payload("feature/high", 55) + payload("feature/critical", 100, BranchRiskStatus.Critical) ], client) assert.equal(result?.status, "failed") @@ -95,7 +145,7 @@ test("records failed result when response is invalid", async () => { test("passes custom system prompt to client", async () => { const client = new AiPredictionClientSpy() await predictMergeRisksWithAi([ - payload("feature/high", 55) + payload("feature/critical", 100, BranchRiskStatus.Critical) ], client, { systemPrompt: "Return compact JSON." }) @@ -144,13 +194,21 @@ class DeferredAiPredictionClient implements AiPredictionClient { } } - resolveAll(): void { - for (const pending of this.pending.splice(0)) { - pending.resolve(validResponse(branchNameFrom(pending.prompt))) + resolveNext(): void { + const pending = this.pending.shift() + + if (!pending) { + throw new Error("No pending AI prediction") } + + pending.resolve(validResponse(branchNameFrom(pending.prompt))) } } +function flushTasks(): Promise { + return new Promise(resolve => setImmediate(resolve)) +} + function branchNameFrom(prompt: AiPredictionPrompt): string { const evidence = JSON.parse(prompt.userPrompt) as { branch: { @@ -163,11 +221,13 @@ function branchNameFrom(prompt: AiPredictionPrompt): string { function payload( branchName: string, - score: number + score: number, + status: BranchRiskStatus, + reasonCode: BranchRiskReasonCode = "same_hunk_overlap" ): AiPredictionEvidencePayload { return { branch: branch(branchName), - possibility: possibility(branchName, score), + possibility: possibility(branchName, score, status, reasonCode), gitSignal: gitSignal(branchName), changedHunks: [] } @@ -184,15 +244,17 @@ function branch(name: string): BranchContext { function possibility( branchName: string, - score: number + score: number, + status: BranchRiskStatus, + reasonCode: BranchRiskReasonCode ): BranchRisk { return { branchName, baseBranch: "main", score, - status: "high", + status, reasons: [{ - code: "same_hunk_overlap", + code: reasonCode, message: "다른 branch와 같은 hunk를 수정함", scoreImpact: score }] diff --git a/tests/ai/predictionTargetSelector.test.ts b/tests/ai/predictionTargetSelector.test.ts index f9e667b..50a2f1d 100644 --- a/tests/ai/predictionTargetSelector.test.ts +++ b/tests/ai/predictionTargetSelector.test.ts @@ -1,59 +1,63 @@ import test from "node:test" import assert from "node:assert/strict" import { - DEFAULT_AI_PREDICTION_MINIMUM_SCORE, + BranchRiskStatus, + DEFAULT_AI_PREDICTION_TARGET_STATUS, selectAiPredictionTargets, type AiPredictionEvidencePayload, type BranchContext, type BranchRisk, type GitMergeSignal } from "../../src/index.js" +import type { BranchRiskReasonCode } from "../../src/risks/types.js" -// 기본 기준으로 medium 이상 score만 AI prediction 대상으로 선택하는지 확인 -test("selects medium or higher score targets by default", () => { +// 기본 기준으로 critical possibility만 AI prediction 대상으로 선택하는지 확인 +test("selects critical targets by default", () => { const selected = selectAiPredictionTargets([ - payload("feature/low", 20), - payload("feature/medium", DEFAULT_AI_PREDICTION_MINIMUM_SCORE), - payload("feature/high", 55) + payload("feature/low", 20, BranchRiskStatus.Low), + payload("feature/high", 55, BranchRiskStatus.High), + payload("feature/critical", 100, DEFAULT_AI_PREDICTION_TARGET_STATUS) ]) - assert.deepEqual(selected.map(target => target.branch.name), [ - "feature/medium", - "feature/high" + assert.deepEqual(selected.map(target => target.branch.name), ["feature/critical"]) +}) + +// score가 높아도 critical status가 아니면 Gemini 호출 대상에서 제외되는지 확인 +test("skips high score non-critical targets", () => { + const selected = selectAiPredictionTargets([ + payload("feature/high", 90, BranchRiskStatus.High) ]) + + assert.deepEqual(selected, []) }) -// caller가 지정한 minimumScore 기준으로 AI prediction 대상을 제한하는지 확인 -test("selects targets with custom minimum score", () => { +// critical status면 score 값과 별개로 AI prediction 대상에 포함되는지 확인 +test("includes critical status targets", () => { const selected = selectAiPredictionTargets([ - payload("feature/medium", 25), - payload("feature/high", 55), - payload("feature/critical", 100) - ], { - minimumScore: 80 - }) + payload("feature/critical", 80, BranchRiskStatus.Critical) + ]) assert.deepEqual(selected.map(target => target.branch.name), ["feature/critical"]) }) -// threshold와 같은 score는 AI prediction 대상에 포함되는지 확인 -test("includes targets at the minimum score boundary", () => { +// 이미 Git conflict가 확정된 branch는 AI prediction 없이 deterministic report만 사용하는지 확인 +test("skips confirmed conflict targets", () => { const selected = selectAiPredictionTargets([ - payload("feature/boundary", 50) - ], { - minimumScore: 50 - }) + payload("feature/conflict", 100, BranchRiskStatus.Critical, "confirmed_conflict") + ]) - assert.deepEqual(selected.map(target => target.branch.name), ["feature/boundary"]) + assert.deepEqual(selected, []) }) function payload( branchName: string, - score: number + score: number, + status: BranchRiskStatus, + reasonCode: BranchRiskReasonCode = "merge_check_failed" ): AiPredictionEvidencePayload { return { branch: branch(branchName), - possibility: possibility(branchName, score), + possibility: possibility(branchName, score, status, reasonCode), gitSignal: gitSignal(branchName), changedHunks: [] } @@ -70,15 +74,17 @@ function branch(name: string): BranchContext { function possibility( branchName: string, - score: number + score: number, + status: BranchRiskStatus, + reasonCode: BranchRiskReasonCode ): BranchRisk { return { branchName, baseBranch: "main", score, - status: "medium", + status, reasons: [{ - code: "merge_check_failed", + code: reasonCode, message: "virtual merge 확인에 실패함", scoreImpact: score }] diff --git a/tests/reports/markdownFormatter.test.ts b/tests/reports/markdownFormatter.test.ts index 68a3cbf..5338520 100644 --- a/tests/reports/markdownFormatter.test.ts +++ b/tests/reports/markdownFormatter.test.ts @@ -81,12 +81,12 @@ test("formats skipped AI result", () => { status: "skipped", branchName: "feature/risk", baseBranch: "main", - reason: "below_threshold" + reason: "not_target" })) assert.match(markdown, /- ai prediction:/) assert.match(markdown, /- status: `skipped`/) - assert.match(markdown, /- reason: `below_threshold`/) + assert.match(markdown, /- reason: `not_target`/) }) // AI failed 결과를 deterministic report 유지 상태로 표시하는지 확인 diff --git a/tests/reports/reportBuilder.test.ts b/tests/reports/reportBuilder.test.ts index aad50a4..829153a 100644 --- a/tests/reports/reportBuilder.test.ts +++ b/tests/reports/reportBuilder.test.ts @@ -143,7 +143,7 @@ test("keeps skipped AI result when present", () => { status: "skipped", branchName: "feature/skipped", baseBranch: "main", - reason: "below_threshold" + reason: "not_target" } const report = buildMergeRiskReport([ input("feature/skipped", BranchRiskStatus.Low, 0, {}, prediction)