Skip to content

Reduce chat request retention #689

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jahooma merged 1 commit into from
May 15, 2026
Merged

Reduce chat request retention #689

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions web/src/app/api/v1/chat/completions/_post.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import type { NextRequest } from 'next/server'

import type { ChatCompletionRequestBody } from '@/llm-api/types'

import { createRequestAuditRecord } from '@/llm-api/helpers'
import {
CanopyWaveError,
handleCanopyWaveNonStream,
Expand Down Expand Up @@ -874,9 +875,7 @@ export async function postChatCompletions(params: {

// Log detailed error information for debugging
const errorDetails = openrouterError?.toJSON()
const shouldRecordMessages = freebuffAccessTier !== 'limited'
const { messages: _messages, ...bodyWithoutMessages } = body
const telemetryBody = shouldRecordMessages ? body : bodyWithoutMessages
const telemetryBody = createRequestAuditRecord(body)
const providerLabel = siliconflowError
? 'SiliconFlow'
: opencodeZenError
Expand Down Expand Up @@ -904,9 +903,8 @@ export async function postChatCompletions(params: {
messageCount: Array.isArray(typedBody.messages)
? typedBody.messages.length
: 0,
...(shouldRecordMessages
? { messages: typedBody.messages }
: { messagesOmitted: true, accessTier: freebuffAccessTier }),
messagesOmitted: true,
accessTier: freebuffAccessTier,
providerStatusCode: (
openrouterError ??
fireworksError ??
Expand Down
7 changes: 5 additions & 2 deletions web/src/llm-api/canopywave.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { env } from '@codebuff/internal/env'

import {
consumeCreditsForMessage,
createRequestAuditRecord,
extractRequestMetadata,
insertMessageToBigQuery,
} from './helpers'
Expand Down Expand Up @@ -165,6 +166,7 @@ export async function handleCanopyWaveNonStream({
const originalModel = body.model
const startTime = new Date()
const { clientId, clientRequestId, costMode } = extractRequestMetadata({ body, logger })
const auditRequest = createRequestAuditRecord(body)

const response = await createCanopyWaveRequest({ body, originalModel, fetch })

Expand All @@ -181,7 +183,7 @@ export async function handleCanopyWaveNonStream({
messageId: data.id,
userId,
startTime,
request: body,
request: auditRequest,
reasoningText,
responseText: content,
usageData,
Expand Down Expand Up @@ -242,6 +244,7 @@ export async function handleCanopyWaveStream({
const originalModel = body.model
const startTime = new Date()
const { clientId, clientRequestId, costMode } = extractRequestMetadata({ body, logger })
const auditRequest = createRequestAuditRecord(body)

const response = await createCanopyWaveRequest({ body, originalModel, fetch })

Expand Down Expand Up @@ -305,7 +308,7 @@ export async function handleCanopyWaveStream({
clientRequestId,
costMode,
startTime,
request: body,
request: auditRequest,
originalModel,
line,
state,
Expand Down
7 changes: 5 additions & 2 deletions web/src/llm-api/deepseek.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { env } from '@codebuff/internal/env'

import {
consumeCreditsForMessage,
createRequestAuditRecord,
extractRequestMetadata,
insertMessageToBigQuery,
} from './helpers'
Expand Down Expand Up @@ -203,6 +204,7 @@ export async function handleDeepSeekNonStream({
body,
logger,
})
const auditRequest = createRequestAuditRecord(body)

const response = await createDeepSeekRequest({ body, originalModel, fetch })

Expand All @@ -222,7 +224,7 @@ export async function handleDeepSeekNonStream({
messageId: data.id,
userId,
startTime,
request: body,
request: auditRequest,
reasoningText,
responseText: content,
usageData,
Expand Down Expand Up @@ -286,6 +288,7 @@ export async function handleDeepSeekStream({
body,
logger,
})
const auditRequest = createRequestAuditRecord(body)
const skipDisconnectedBilling = isDeepSeekV4FlashModel(body.model)

const response = await createDeepSeekRequest({ body, originalModel, fetch })
Expand Down Expand Up @@ -355,7 +358,7 @@ export async function handleDeepSeekStream({
clientRequestId,
costMode,
startTime,
request: body,
request: auditRequest,
originalModel,
line,
state,
Expand Down
7 changes: 5 additions & 2 deletions web/src/llm-api/fireworks.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { env } from '@codebuff/internal/env'
import { FIREWORKS_DEPLOYMENT_MAP } from './fireworks-config'
import {
consumeCreditsForMessage,
createRequestAuditRecord,
extractRequestMetadata,
insertMessageToBigQuery,
} from './helpers'
Expand Down Expand Up @@ -273,6 +274,7 @@ export async function handleFireworksNonStream({
body,
logger,
})
const auditRequest = createRequestAuditRecord(body)

const response = await createFireworksRequestWithFallback({
body,
Expand All @@ -298,7 +300,7 @@ export async function handleFireworksNonStream({
messageId: data.id,
userId,
startTime,
request: body,
request: auditRequest,
reasoningText,
responseText: content,
usageData,
Expand Down Expand Up @@ -362,6 +364,7 @@ export async function handleFireworksStream({
body,
logger,
})
const auditRequest = createRequestAuditRecord(body)

const response = await createFireworksRequestWithFallback({
body,
Expand Down Expand Up @@ -431,7 +434,7 @@ export async function handleFireworksStream({
clientRequestId,
costMode,
startTime,
request: body,
request: auditRequest,
originalModel,
line,
state,
Expand Down
89 changes: 87 additions & 2 deletions web/src/llm-api/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,85 @@ export type UsageData = {
cost: number
}

export function createRequestAuditRecord(body: unknown) {
// TODO: Add a separate append-only message_request BigQuery table for full
// raw request bodies, inserted before streaming starts. Keeping only this
// summary here avoids retaining huge chat requests until provider streams end.
if (typeof body !== 'object' || body === null || Array.isArray(body)) {
return { invalid_request_shape: true }
}

const typedBody = body as Partial<ChatCompletionRequestBody>
const messages = Array.isArray(typedBody.messages)
? typedBody.messages
: undefined
const tools = Array.isArray(typedBody.tools) ? typedBody.tools : undefined

const messageRoleCounts = messages?.reduce<Record<string, number>>(
(counts, message) => {
const role =
typeof message === 'object' && message !== null && 'role' in message
? String(message.role)
: 'unknown'
counts[role] = (counts[role] ?? 0) + 1
return counts
},
{},
)

return {
model: typeof typedBody.model === 'string' ? typedBody.model : undefined,
stream:
typeof typedBody.stream === 'boolean' ? typedBody.stream : undefined,
temperature:
typeof typedBody.temperature === 'number'
? typedBody.temperature
: undefined,
max_tokens:
typeof typedBody.max_tokens === 'number'
? typedBody.max_tokens
: undefined,
max_completion_tokens:
typeof typedBody.max_completion_tokens === 'number'
? typedBody.max_completion_tokens
: undefined,
top_p: typeof typedBody.top_p === 'number' ? typedBody.top_p : undefined,
reasoning_effort:
typeof typedBody.reasoning_effort === 'string'
? typedBody.reasoning_effort
: undefined,
reasoning_enabled:
typeof typedBody.reasoning?.enabled === 'boolean'
? typedBody.reasoning.enabled
: undefined,
reasoning_effort_nested:
typeof typedBody.reasoning?.effort === 'string'
? typedBody.reasoning.effort
: undefined,
usage_include:
typeof typedBody.usage?.include === 'boolean'
? typedBody.usage.include
: undefined,
codebuff_metadata:
typeof typedBody.codebuff_metadata === 'object' &&
typedBody.codebuff_metadata !== null
? { ...typedBody.codebuff_metadata }
: undefined,
message_count: messages?.length ?? 0,
message_role_counts: messageRoleCounts,
messages_omitted: !!messages,
tool_count: tools?.length ?? 0,
tool_names: tools
?.map((tool) =>
typeof tool === 'object' && tool !== null
? tool.function?.name
: undefined,
)
.filter((name): name is string => typeof name === 'string'),
tools_omitted: !!tools,
}
}

export function extractRequestMetadata(params: {
body: unknown
logger: Logger
Expand All @@ -35,14 +114,20 @@ export function extractRequestMetadata(params: {
const rawClientId = metadata?.client_id
const clientId = typeof rawClientId === 'string' ? rawClientId : null
if (!clientId) {
logger.warn({ body }, 'Received request without client_id')
logger.warn(
{ request: createRequestAuditRecord(body) },
'Received request without client_id',
)
}

const rawRunId = metadata?.run_id
const clientRequestId: string | null =
typeof rawRunId === 'string' ? rawRunId : null
if (!clientRequestId) {
logger.warn({ body }, 'Received request without run_id')
logger.warn(
{ request: createRequestAuditRecord(body) },
'Received request without run_id',
)
}

const n = metadata?.n
Expand Down
7 changes: 5 additions & 2 deletions web/src/llm-api/moonshot.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { env } from '@codebuff/internal/env'

import {
consumeCreditsForMessage,
createRequestAuditRecord,
extractRequestMetadata,
insertMessageToBigQuery,
} from './helpers'
Expand Down Expand Up @@ -288,6 +289,7 @@ export async function handleMoonshotNonStream({
body,
logger,
})
const auditRequest = createRequestAuditRecord(body)

const response = await createMoonshotRequest({ body, originalModel, fetch })
if (!response.ok) {
Expand All @@ -306,7 +308,7 @@ export async function handleMoonshotNonStream({
messageId: data.id,
userId,
startTime,
request: body,
request: auditRequest,
reasoningText,
responseText: content,
usageData,
Expand Down Expand Up @@ -368,6 +370,7 @@ export async function handleMoonshotStream({
body,
logger,
})
const auditRequest = createRequestAuditRecord(body)

const response = await createMoonshotRequest({ body, originalModel, fetch })
if (!response.ok) {
Expand Down Expand Up @@ -435,7 +438,7 @@ export async function handleMoonshotStream({
clientRequestId,
costMode,
startTime,
request: body,
request: auditRequest,
originalModel,
line,
state,
Expand Down
Loading
Loading