Skip to content

Route embeddings and transcription through Vercel #4490

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chrarnoldus wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Route embeddings and transcription through Vercel #4490

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 93 additions & 47 deletions apps/web/src/app/admin/gateway/RoutingContent.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,41 @@ import { Input } from '@/components/ui/input';
import { Textarea } from '@/components/ui/textarea';
import { Label } from '@/components/ui/label';
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card';
import { DEFAULT_VERCEL_PERCENTAGE, NOTE_MAX_LENGTH } from '@/lib/ai-gateway/gateway-config';
import {
DEFAULT_VERCEL_PERCENTAGE,
NOTE_MAX_LENGTH,
type VercelRoutingApiType,
} from '@/lib/ai-gateway/gateway-config';

const ROUTING_API_LABELS: Record<VercelRoutingApiType, string> = {
chat: 'Chat',
embeddings: 'Embeddings',
transcription: 'Transcription',
};

const EMPTY_PERCENTAGES: Record<VercelRoutingApiType, string> = {
chat: '',
embeddings: '',
transcription: '',
};

export function RoutingContent() {
const trpc = useTRPC();
const queryClient = useQueryClient();

const { data, isLoading } = useQuery(trpc.admin.gatewayConfig.get.queryOptions());

const [inputValue, setInputValue] = useState('');
const [percentages, setPercentages] = useState(EMPTY_PERCENTAGES);
const [noteValue, setNoteValue] = useState('');
const [hasChanges, setHasChanges] = useState(false);

useEffect(() => {
if (data) {
setInputValue(data.vercel_routing_percentage?.toString() ?? '');
setPercentages({
chat: data.vercel_chat_routing_percentage?.toString() ?? '',
embeddings: data.vercel_embeddings_routing_percentage?.toString() ?? '',
transcription: data.vercel_transcription_routing_percentage?.toString() ?? '',
});
setNoteValue('');
setHasChanges(false);
}
Expand All @@ -35,7 +55,7 @@ export function RoutingContent() {
void queryClient.invalidateQueries({
queryKey: trpc.admin.gatewayConfig.get.queryKey(),
});
toast.success('Vercel routing percentage updated');
toast.success('Vercel routing percentages updated');
},
onError: error => {
toast.error(error.message || 'Failed to update');
Expand All @@ -49,61 +69,85 @@ export function RoutingContent() {
}

function handleSave() {
const trimmed = inputValue.trim();
const note = noteInput();
if (trimmed === '') {
mutation.mutate({ vercel_routing_percentage: null, note });
} else {
const num = Number(trimmed);
if (!Number.isInteger(num) || num < 0 || num > 100) {
toast.error('Please enter a whole number between 0 and 100, or leave empty for default');
return;
}
mutation.mutate({ vercel_routing_percentage: num, note });
function parsePercentage(input: string): number | null | undefined {
const trimmed = input.trim();
if (trimmed === '') return null;
const value = Number(trimmed);
return Number.isInteger(value) && value >= 0 && value <= 100 ? value : undefined;
}

const chat = parsePercentage(percentages.chat);
const embeddings = parsePercentage(percentages.embeddings);
const transcription = parsePercentage(percentages.transcription);
if (chat === undefined || embeddings === undefined || transcription === undefined) {
toast.error('Please enter a whole number between 0 and 100, or leave empty for default');
return;
}
mutation.mutate({
vercel_chat_routing_percentage: chat,
vercel_embeddings_routing_percentage: embeddings,
vercel_transcription_routing_percentage: transcription,
note: noteInput(),
});
}

function handleClear() {
mutation.mutate({ vercel_routing_percentage: null, note: noteInput() });
mutation.mutate({
vercel_chat_routing_percentage: null,
vercel_embeddings_routing_percentage: null,
vercel_transcription_routing_percentage: null,
note: noteInput(),
});
}

if (isLoading) {
return <div className="text-muted-foreground py-8 text-sm">Loading...</div>;
}

const currentOverride = data?.vercel_routing_percentage;
const isOverrideActive = currentOverride !== null && currentOverride !== undefined;
const currentPercentages = {
chat: data?.vercel_chat_routing_percentage,
embeddings: data?.vercel_embeddings_routing_percentage,
transcription: data?.vercel_transcription_routing_percentage,
};
const isOverrideActive = Object.values(currentPercentages).some(value => value != null);

return (
<div className="flex w-full flex-col gap-y-6">
<Card>
<CardHeader>
<CardTitle>Vercel Routing Percentage</CardTitle>
<CardTitle>Vercel Routing Percentages</CardTitle>
<CardDescription>
For models available on the Vercel AI Gateway, controls the percentage of traffic routed
to Vercel (vs OpenRouter). Models not available on Vercel always go to OpenRouter, so
overall traffic may still be skewed towards OpenRouter. Leave empty to use the default (
{DEFAULT_VERCEL_PERCENTAGE}%).
Controls the Vercel traffic split independently for each API. Unsupported models always
use OpenRouter. Leave a field empty to use the {DEFAULT_VERCEL_PERCENTAGE}% default.
</CardDescription>
</CardHeader>
<CardContent className="flex flex-col gap-4">
<div className="grid gap-3 md:grid-cols-3">
{Object.entries(ROUTING_API_LABELS).map(([apiType, label]) => (
<div className="flex flex-col gap-2" key={apiType}>
<Label htmlFor={`routing-${apiType}`}>{label}</Label>
<div className="flex items-center gap-2">
<Input
id={`routing-${apiType}`}
type="number"
min={0}
max={100}
placeholder={`Default: ${DEFAULT_VERCEL_PERCENTAGE}%`}
value={percentages[apiType as VercelRoutingApiType]}
onChange={e => {
setPercentages(current => ({ ...current, [apiType]: e.target.value }));
setHasChanges(true);
}}
onKeyDown={e => {
if (e.key === 'Enter') handleSave();
}}
/>
<span className="text-muted-foreground text-sm">%</span>
</div>
</div>
))}
</div>
<div className="flex items-center gap-3">
<Input
type="number"
min={0}
max={100}
placeholder={`Default: ${DEFAULT_VERCEL_PERCENTAGE}%`}
value={inputValue}
onChange={e => {
setInputValue(e.target.value);
setHasChanges(true);
}}
onKeyDown={e => {
if (e.key === 'Enter') handleSave();
}}
className="w-48"
/>
<span className="text-muted-foreground text-sm">%</span>
<Button onClick={handleSave} disabled={mutation.isPending || !hasChanges} size="sm">
{mutation.isPending ? 'Saving...' : 'Save'}
</Button>
Expand All @@ -114,7 +158,7 @@ export function RoutingContent() {
variant="outline"
size="sm"
>
Clear override
Clear overrides
</Button>
)}
</div>
Expand All @@ -136,17 +180,19 @@ export function RoutingContent() {

<div className="text-muted-foreground text-sm">
{isOverrideActive ? (
<p>
Override active:{' '}
<span className="text-foreground font-medium">{currentOverride}%</span> of traffic
goes to Vercel.
<div className="flex flex-col gap-1">
<p>
Active routing: chat {currentPercentages.chat ?? DEFAULT_VERCEL_PERCENTAGE}%,
embeddings {currentPercentages.embeddings ?? DEFAULT_VERCEL_PERCENTAGE}%,
transcription {currentPercentages.transcription ?? DEFAULT_VERCEL_PERCENTAGE}%.
</p>
{data?.updated_by_email && (
<span className="ml-1">
<p>
Set by {data.updated_by_email}
{data.updated_at && <> at {new Date(data.updated_at).toLocaleString()}</>}.
</span>
</p>
)}
</p>
</div>
) : (
<p>
No override set. Using default routing ({DEFAULT_VERCEL_PERCENTAGE}% to Vercel).
Expand Down
44 changes: 44 additions & 0 deletions apps/web/src/app/api/openrouter/audio/transcriptions/route.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import { getBalanceAndOrgSettings } from '@/lib/organizations/organization-usage
import type { User } from '@kilocode/db/schema';
import { emitApiMetricsForResponse } from '@/lib/ai-gateway/o11y/api-metrics.server';
import type { OrganizationSettings } from '@/lib/organizations/organization-types';
import { getTranscriptionProvider } from '@/lib/ai-gateway/providers/get-provider';
import PROVIDERS from '@/lib/ai-gateway/providers/provider-definitions';

jest.mock('next/server', () => {
return {
Expand All @@ -14,6 +16,7 @@ jest.mock('next/server', () => {

jest.mock('@/lib/user/server');
jest.mock('@/lib/organizations/organization-usage');
jest.mock('@/lib/ai-gateway/providers/get-provider');
jest.mock('@/lib/ai-gateway/o11y/api-metrics.server', () => ({
emitApiMetricsForResponse: jest.fn(),
}));
Expand All @@ -28,6 +31,7 @@ jest.mock('@/lib/ai-gateway/llm-proxy-helpers', () => {
const mockedGetUserFromAuth = jest.mocked(getUserFromAuth);
const mockedGetBalanceAndOrgSettings = jest.mocked(getBalanceAndOrgSettings);
const mockedEmitApiMetricsForResponse = jest.mocked(emitApiMetricsForResponse);
const mockedGetTranscriptionProvider = jest.mocked(getTranscriptionProvider);
const mockedFetch = jest.fn() as jest.MockedFunction<typeof globalThis.fetch>;
const originalFetch = globalThis.fetch;

Expand Down Expand Up @@ -71,6 +75,10 @@ describe('POST /api/gateway/v1/audio/transcriptions', () => {
beforeEach(() => {
jest.resetAllMocks();
globalThis.fetch = mockedFetch;
mockedGetTranscriptionProvider.mockResolvedValue({
provider: PROVIDERS.OPENROUTER,
userByok: null,
});
});

afterAll(() => {
Expand Down Expand Up @@ -149,6 +157,42 @@ describe('POST /api/gateway/v1/audio/transcriptions', () => {
expect(upstream.provider).toEqual({ only: ['openai'], data_collection: 'deny' });
});

it('adapts transcription requests to the Vercel REST API', async () => {
setUserAuth();
mockedGetTranscriptionProvider.mockResolvedValue({
provider: PROVIDERS.VERCEL_AI_GATEWAY,
userByok: null,
});
mockedFetch.mockResolvedValue(
makeUpstreamResponse({
text: 'hello world',
durationInSeconds: 1.5,
providerMetadata: { gateway: { cost: '0.00002' } },
})
);

const { POST } = await import('./route');
const response = await POST(
makeRequest({
model: 'openai/gpt-4o-mini-transcribe',
input_audio: { data: 'UklGRiQA', format: 'mp3' },
provider: { only: ['openai'] },
}) as never
);

expect(response.status).toBe(200);
const [url, init] = mockedFetch.mock.calls[0];
expect(url).toBe('https://ai-gateway.vercel.sh/v4/ai/transcription-model');
const headers = init?.headers as Headers;
expect(headers.get('ai-model-id')).toBe('openai/gpt-4o-mini-transcribe');
expect(headers.get('ai-transcription-model-specification-version')).toBe('4');
expect(JSON.parse(init?.body as string)).toMatchObject({
audio: 'UklGRiQA',
mediaType: 'audio/mpeg',
providerOptions: { gateway: { only: ['openai'] } },
});
});

it('rejects malformed transcription bodies before proxying', async () => {
setUserAuth();

Expand Down
34 changes: 30 additions & 4 deletions apps/web/src/app/api/openrouter/audio/transcriptions/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,12 @@ import { emitApiMetricsForResponse } from '@/lib/ai-gateway/o11y/api-metrics.ser
import { normalizeModelId } from '@/lib/ai-gateway/model-utils';
import {
buildUpstreamBody,
buildVercelTranscriptionBody,
extractTranscriptionPromptInfo,
TranscriptionRequestSchema,
} from '@/lib/ai-gateway/transcriptions/transcription-request';
import type { Provider } from '@/lib/ai-gateway/providers/types';
import { mapModelIdToVercel } from '@/lib/ai-gateway/providers/vercel/mapModelIdToVercel';

export const maxDuration = 300;

Expand All @@ -38,9 +40,10 @@ const PAID_MODEL_AUTH_REQUIRED = 'PAID_MODEL_AUTH_REQUIRED';
async function transcriptionProxyRequest(params: {
body: Record<string, unknown>;
provider: Provider;
model: string;
signal?: AbortSignal;
}) {
const { body, provider, signal } = params;
const { body, provider, model, signal } = params;
const headers = new Headers();
headers.set('Content-Type', 'application/json');
headers.set('Authorization', `Bearer ${provider.apiKey}`);
Expand All @@ -49,10 +52,20 @@ async function transcriptionProxyRequest(params: {
headers.set(key, value);
}

if (provider.id === 'vercel') {
headers.set('ai-model-id', mapModelIdToVercel(model, false));
headers.set('ai-transcription-model-specification-version', '4');
}

const timeout = AbortSignal.timeout(10 * 60 * 1000);
const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;

return await fetch(`${provider.apiUrl}/audio/transcriptions`, {
const targetUrl =
provider.id === 'vercel'
? `${provider.apiUrl.replace(/\/v1$/, '')}/v4/ai/transcription-model`
: `${provider.apiUrl}/audio/transcriptions`;

return await fetch(targetUrl, {
method: 'POST',
headers,
body: JSON.stringify(body),
Expand Down Expand Up @@ -133,7 +146,11 @@ export async function POST(request: NextRequest): Promise<NextResponseType<unkno
const user = maybeUser;

const { fraudHeaders, projectId } = extractFraudAndProjectHeaders(request);
const { provider, userByok } = await getTranscriptionProvider();
let { provider, userByok } = await getTranscriptionProvider(
requestedModelLowerCased,
body.provider,
user.id
);
const feature = validateFeatureHeader(request.headers.get(FEATURE_HEADER) || '');
const promptInfo = extractTranscriptionPromptInfo(body);

Expand Down Expand Up @@ -182,6 +199,13 @@ export async function POST(request: NextRequest): Promise<NextResponseType<unkno

if (providerConfig) {
body.provider = { ...body.provider, ...providerConfig };
({ provider, userByok } = await getTranscriptionProvider(
requestedModelLowerCased,
body.provider,
user.id
));
usageContext.provider = provider.id;
usageContext.user_byok = !!userByok;
}

sentryRootSpan()?.setAttribute(
Expand All @@ -192,11 +216,13 @@ export async function POST(request: NextRequest): Promise<NextResponseType<unkno
const span = startInactiveSpan({ name: 'transcription-request-start', op: 'http.client' });
body.safety_identifier = generateProviderSpecificHash(user.id, provider);
body.user = body.safety_identifier;
const upstreamBody = buildUpstreamBody(body);
const upstreamBody =
provider.id === 'vercel' ? buildVercelTranscriptionBody(body) : buildUpstreamBody(body);

const response = await transcriptionProxyRequest({
body: upstreamBody,
provider,
model: requestedModelLowerCased,
signal: request.signal,
});

Expand Down
Loading
Loading