Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,17 +99,17 @@ Watcher는 merge 가능/불가능을 단정하지 않고 branch별 signal을 충

`confirmed_conflict`는 최상위 signal입니다. 이 signal이 있으면 다른 reason을 추가로 합산하지 않고 `critical` risk로 처리합니다.

각 reason은 report에 code, message, score impact, 관련 file, 관련 branch, 관련 check metadata로 표시됩니다. 이 정보가 AI prediction에 전달되는 정제된 evidence입니다.
각 reason은 report에 code, message, score impact, 관련 file, 관련 branch, 관련 check metadata로 표시됩니다. 다만 `same_hunk_overlap`이 있는 branch에서는 중복되는 `same_file_overlap`의 file, branch 목록을 다시 반복하지 않습니다. 이 정보가 AI prediction에 전달되는 정제된 evidence입니다.

## AI-assisted prediction

AI prediction은 deterministic possibility score를 대체하지 않습니다. Watcher는 deterministic evidence를 Gemini API에 전달하고 AI는 실무 관점의 prediction, confidence, recommended actions, false positive notes를 추가합니다.
AI prediction은 deterministic possibility score를 대체하지 않습니다. Watcher는 deterministic evidence를 Gemini API에 전달하고 AI는 실무 관점의 prediction과 recommended actions를 추가합니다.

기본 AI provider는 Gemini API입니다. consumer repository에는 `GEMINI_API_KEY` secret을 설정해야 합니다.

AI prediction 대상은 기본적으로 `critical` possibility입니다. 이미 virtual merge에서 conflict가 확정된 branch는 AI 호출 없이 deterministic report만 사용합니다. 그 외 낮은 status의 branch는 AI 호출을 생략하고 `skipped` 상태로 report에 표시됩니다.

Gemini free tier rate limit을 줄이기 위해 선택된 branch prediction은 순차 실행하며 기본적으로 호출 사이에 60초를 대기합니다.
Gemini free tier의 RPM/RPD 사용량을 줄이기 위해 선택된 branch prediction은 report 단위 batch 요청으로 한 번에 실행합니다.

AI prediction 결과는 다음 상태 중 하나입니다.

Expand All @@ -121,6 +121,8 @@ AI prediction 결과는 다음 상태 중 하나입니다.

AI provider가 실패해도 deterministic possibility report는 유지됩니다. 실패한 branch는 `failed` 상태와 error message를 report에 포함합니다.

Report에는 `Low` section, AI prediction `skipped` 상태, branch `updated` 시각, Gemini error 요약을 유지합니다. Pull Request metadata는 내부 evidence로만 사용할 수 있으며 Markdown report에는 출력하지 않습니다.

## Report channel

Merge risk report는 Markdown으로 생성됩니다. consumer repository에 `DISCORD_WEBHOOK_URL` secret이 있으면 Discord webhook으로 전송하고, 없으면 stdout으로 출력합니다.
Expand Down
30 changes: 22 additions & 8 deletions src/ai/geminiPredictionClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,10 @@ export function createDefaultAiPredictionClient(

// Watcher가 검증할 AiPrediction shape를 Gemini structured output schema로 전달
function geminiRequestBodyFor(prompt: AiPredictionPrompt): Record<string, unknown> {
const schema = prompt.responseShape === "predictionBatch"
? aiPredictionBatchSchema()
: aiPredictionSchema()

return {
systemInstruction: {
parts: [{ text: prompt.systemPrompt }]
Expand All @@ -173,7 +177,7 @@ function geminiRequestBodyFor(prompt: AiPredictionPrompt): Record<string, unknow
responseFormat: {
text: {
mimeType: "APPLICATION_JSON",
schema: aiPredictionSchema()
schema
}
}
}
Expand Down Expand Up @@ -292,28 +296,38 @@ function geminiTextFor(response: GeminiGenerateContentResponse): string {
}

// Gemini가 Watcher prediction contract에 맞는 JSON을 반환하도록 요청하는 schema
function aiPredictionBatchSchema(): Record<string, unknown> {
return {
type: "object",
properties: {
predictions: {
type: "array",
items: aiPredictionSchema()
}
},
required: [
"predictions"
]
}
}

// Gemini가 Watcher prediction 하나의 contract에 맞는 JSON을 반환하도록 요청하는 schema
function aiPredictionSchema(): Record<string, unknown> {
return {
type: "object",
properties: {
branchName: { type: "string" },
baseBranch: { type: "string" },
prediction: { type: "string" },
confidence: { type: "number" },
recommendedActions: {
type: "array",
items: recommendedActionSchema()
},
falsePositiveNotes: {
type: "array",
items: { type: "string" }
}
},
required: [
"branchName",
"baseBranch",
"prediction",
"confidence"
"prediction"
]
}
}
Expand Down
22 changes: 20 additions & 2 deletions src/ai/predictionPromptBuilder.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import { DEFAULT_AI_PREDICTION_SYSTEM_PROMPT } from "./promptTemplates.js"
import {
DEFAULT_AI_PREDICTION_BATCH_SYSTEM_PROMPT,
DEFAULT_AI_PREDICTION_SYSTEM_PROMPT
} from "./promptTemplates.js"
import type {
AiPredictionEvidencePayload,
AiPredictionPrompt,
Expand All @@ -12,7 +15,22 @@ export function build(
): AiPredictionPrompt {
return {
systemPrompt: options.systemPrompt ?? DEFAULT_AI_PREDICTION_SYSTEM_PROMPT,
userPrompt: JSON.stringify(promptEvidenceFor(payload), null, 2)
userPrompt: JSON.stringify(promptEvidenceFor(payload), null, 2),
responseShape: "prediction"
}
}

// 여러 branch evidence를 한 번의 AI provider 호출에 전달할 batch prompt로 구성
export function buildBatch(
payloads: AiPredictionEvidencePayload[],
options: AiPredictionPromptBuildOptions = {}
): AiPredictionPrompt {
return {
systemPrompt: options.systemPrompt ?? DEFAULT_AI_PREDICTION_BATCH_SYSTEM_PROMPT,
userPrompt: JSON.stringify({
branches: payloads.map(promptEvidenceFor)
}, null, 2),
responseShape: "predictionBatch"
}
}

Expand Down
51 changes: 28 additions & 23 deletions src/ai/predictionResponseValidator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,37 @@ const actionPriorities = new Set<AiRecommendedActionPriority>([

// AI가 반환한 unknown JSON을 Watcher가 사용하는 prediction 모델로 검증
export function validate(response: unknown): AiPrediction {
return predictionFor(response, "response")
}

// AI가 반환한 batch JSON을 branch별 prediction 배열로 검증
export function validateBatch(response: unknown): AiPrediction[] {
const value = objectFor(response, "response")

return arrayFor(value.predictions, "predictions")
.map((prediction, index) => {
try {
return predictionFor(prediction, `predictions[${index}]`)
} catch {
return undefined
}
})
.filter((prediction): prediction is AiPrediction => prediction !== undefined)
}
Comment thread
opficdev marked this conversation as resolved.

// AI prediction 하나가 Watcher report에 사용할 수 있는 shape인지 검증
function predictionFor(
response: unknown,
path: string
): AiPrediction {
const value = objectFor(response, path)

return {
branchName: stringFor(value.branchName, "branchName"),
baseBranch: stringFor(value.baseBranch, "baseBranch"),
prediction: stringFor(value.prediction, "prediction"),
confidence: confidenceFor(value.confidence),
recommendedActions: arrayFor(value.recommendedActions ?? [], "recommendedActions")
.map((action, index) => recommendedActionFor(action, index)),
falsePositiveNotes: arrayFor(value.falsePositiveNotes ?? [], "falsePositiveNotes")
.map((note, index) => stringFor(note, `falsePositiveNotes[${index}]`))
branchName: stringFor(value.branchName, `${path}.branchName`),
baseBranch: stringFor(value.baseBranch, `${path}.baseBranch`),
prediction: stringFor(value.prediction, `${path}.prediction`),
recommendedActions: arrayFor(value.recommendedActions ?? [], `${path}.recommendedActions`)
.map((action, index) => recommendedActionFor(action, index))
}
}

Expand All @@ -46,21 +66,6 @@ function recommendedActionFor(
}
}

// confidence는 report에서 비교할 수 있도록 0-100 범위의 정수로 제한
function confidenceFor(value: unknown): number {
if (
typeof value !== "number" ||
!Number.isFinite(value) ||
!Number.isInteger(value) ||
value < 0 ||
100 < value
) {
throw new Error("AI prediction response confidence must be an integer between 0 and 100")
}

return value
}

// action priority가 Watcher가 표시할 수 있는 허용 값인지 검증
function priorityFor(
value: unknown,
Expand Down
131 changes: 82 additions & 49 deletions src/ai/predictionRunner.ts
Original file line number Diff line number Diff line change
@@ -1,71 +1,52 @@
import { build as buildPrompt } from "./predictionPromptBuilder.js"
import { buildBatch as buildBatchPrompt } from "./predictionPromptBuilder.js"
import { select as selectTargets } from "./predictionTargetSelector.js"
import { validate as validateResponse } from "./predictionResponseValidator.js"
import { validateBatch as validateBatchResponse } from "./predictionResponseValidator.js"
import type {
AiPrediction,
AiPredictionClient,
AiPredictionEvidencePayload,
AiPredictionResult,
AiPredictionRunOptions
} from "./types.js"

// Gemini free tier rate limit을 줄이기 위해 선택된 AI 호출 사이에 둘 내부 대기 시간
const DEFAULT_AI_PREDICTION_DELAY_MS = 60000

// 대상 선택, prompt 생성, provider 호출, 응답 검증을 branch별 AI prediction 결과로 연결
// 대상 선택, batch prompt 생성, provider 호출, 응답 검증을 branch별 AI prediction 결과로 연결
export async function predict(
payloads: AiPredictionEvidencePayload[],
client: AiPredictionClient,
options: AiPredictionRunOptions = {}
): Promise<AiPredictionResult[]> {
const targets = new Set(selectTargets(payloads))
const results: AiPredictionResult[] = []

for (const payload of payloads) {
if (!targets.has(payload)) {
results.push({
status: "skipped",
branchName: payload.branch.name,
baseBranch: payload.branch.baseBranch,
reason: skippedReasonFor(payload)
})
continue
}
const targets = selectTargets(payloads)
const targetSet = new Set(targets)

results.push(await predictedResultFor(payload, client, options))
targets.delete(payload)

if (0 < targets.size) {
await delay(DEFAULT_AI_PREDICTION_DELAY_MS)
}
if (targets.length === 0) {
return payloads.map(skippedResultFor)
}

return results
const predictedResults = await predictedResultsFor(targets, client, options)

return payloads.map(payload => targetSet.has(payload)
? predictedResults.get(payload) ?? failedResultFor(payload, "AI prediction response is missing")
: skippedResultFor(payload)
)
}

// provider 호출과 schema 검증 실패를 branch 단위 failed 결과로 격리
async function predictedResultFor(
payload: AiPredictionEvidencePayload,
// provider 호출과 schema 검증 실패를 선택된 branch 단위 failed 결과로 격리
async function predictedResultsFor(
targets: AiPredictionEvidencePayload[],
client: AiPredictionClient,
options: AiPredictionRunOptions
): Promise<AiPredictionResult> {
): Promise<Map<AiPredictionEvidencePayload, AiPredictionResult>> {
try {
const prompt = buildPrompt(payload, options)
const prompt = buildBatchPrompt(targets, options)
const response = await client.predict(prompt)
const prediction = validateResponse(response)
const predictions = validateBatchResponse(response)

return {
status: "predicted",
branchName: payload.branch.name,
baseBranch: payload.branch.baseBranch,
prediction
}
return resultsByPayloadFor(targets, predictions)
} catch (error) {
return {
status: "failed",
branchName: payload.branch.name,
baseBranch: payload.branch.baseBranch,
errorMessage: errorMessageFor(error)
}
return new Map(targets.map(target => [
target,
failedResultFor(target, errorMessageFor(error))
]))
}
}

Expand All @@ -81,11 +62,63 @@ function skippedReasonFor(payload: AiPredictionEvidencePayload): "not_target" |
: "not_target"
}

// 선택된 AI provider 호출 사이에 간격을 두어 rate limit 진입 가능성을 낮춤
function delay(delayMs: number): Promise<void> {
if (delayMs <= 0) {
return Promise.resolve()
// AI prediction 대상이 아닌 branch를 skipped 결과로 변환
function skippedResultFor(payload: AiPredictionEvidencePayload): AiPredictionResult {
return {
status: "skipped",
branchName: payload.branch.name,
baseBranch: payload.branch.baseBranch,
reason: skippedReasonFor(payload)
}
}

// provider 실패나 응답 누락을 branch별 failed 결과로 변환
function failedResultFor(
payload: AiPredictionEvidencePayload,
errorMessage: string
): AiPredictionResult {
return {
status: "failed",
branchName: payload.branch.name,
baseBranch: payload.branch.baseBranch,
errorMessage
}
}

// batch 응답을 원래 target payload와 매칭해 branch별 결과로 복원
function resultsByPayloadFor(
targets: AiPredictionEvidencePayload[],
predictions: AiPrediction[]
): Map<AiPredictionEvidencePayload, AiPredictionResult> {
const predictionsByBranch = new Map(predictions.map(prediction => [
predictionKeyFor(prediction.branchName, prediction.baseBranch),
prediction
]))

return new Map(targets.map(target => {
const prediction = predictionsByBranch.get(predictionKeyFor(
target.branch.name,
target.branch.baseBranch
))

return [
target,
prediction
? {
status: "predicted",
branchName: target.branch.name,
baseBranch: target.branch.baseBranch,
prediction
}
: failedResultFor(target, "AI prediction response is missing")
]
}))
}

return new Promise(resolve => setTimeout(resolve, delayMs))
// branch 이름만 같은 다른 base branch와 섞이지 않도록 base branch까지 포함해 매칭
function predictionKeyFor(
branchName: string,
baseBranch: string
): string {
return `${baseBranch}\u0000${branchName}`
}
Loading
Loading