feat: add isBot option to renderRouterToStream for streaming SSR bot detection

.changeset/ssr-isbot-option.md

@@ -0,0 +1,34 @@
+---
+'@tanstack/react-router': minor
+'@tanstack/solid-router': minor
+'@tanstack/vue-router': minor
+---
+
+feat: add `isBot` option to `renderRouterToStream` to configure streaming SSR bot detection
+
+Streaming SSR waits for the full document (React's `allReady`) for requests that `isbot` flags by their `User-Agent`, and streams the shell first for everyone else. This was hardcoded, so performance auditors (Lighthouse, PageSpeed Insights, WebPageTest, …) — which `isbot` classifies as bots — were measured on the buffered path instead of the streaming path real users get.
+
+`renderRouterToStream` now accepts an `isBot` option so the decision can be made in the server request handler (it lives in `ssr/server` and never ships to the client). The default is unchanged.
+
+- `undefined` (default): use the built-in `isbot` User-Agent check.
+- `boolean`: force the bot (`true`) or non-bot (`false`) path for the request.
+- `(request) => boolean`: provide a custom predicate.
+
+```tsx
+import {
+  createRequestHandler,
+  renderRouterToStream,
+  RouterServer,
+} from '@tanstack/react-router/ssr/server'
+
+createRequestHandler({ request, createRouter })(
+  ({ request, router, responseHeaders }) =>
+    renderRouterToStream({
+      request,
+      router,
+      responseHeaders,
+      children: <RouterServer router={router} />,
+      isBot: false,
+    }),
+)
+```

packages/react-router/src/ssr/renderRouterToStream.tsx

@@ -46,12 +46,24 @@ export const renderRouterToStream = async ({
   router,
   responseHeaders,
   children,
+  isBot,
 }: {
   request: Request
   router: AnyRouter
   responseHeaders: Headers
   children: ReactNode
+  /**
+   * Whether to treat the request as a bot/crawler. Bots wait for the full
+   * document (React's `allReady`) before responding instead of streaming the
+   * shell first. Defaults to the built-in `isbot` User-Agent check.
+   */
+  isBot?: boolean | ((request: Request) => boolean)
 }) => {
+  const isBotRequest =
+    typeof isBot === 'function'
+      ? isBot(request)
+      : (isBot ?? isbot(request.headers.get('User-Agent')))
+
   if (typeof ReactDOMServer.renderToReadableStream === 'function') {
     const stream = await ReactDOMServer.renderToReadableStream(children, {
       signal: request.signal,
@@ -64,7 +76,7 @@ export const renderRouterToStream = async ({
       },
     })
 
-    if (isbot(request.headers.get('User-Agent'))) {
+    if (isBotRequest) {
       await waitForReadyOrAbort(stream.allReady, request.signal)
     }
 
@@ -150,7 +162,7 @@ export const renderRouterToStream = async ({
       pipeable = ReactDOMServer.renderToPipeableStream(children, {
         nonce: router.options.ssr?.nonce,
         progressiveChunkSize: Number.POSITIVE_INFINITY,
-        ...(isbot(request.headers.get('User-Agent'))
+        ...(isBotRequest
           ? {
               onAllReady() {
                 pipeable!.pipe(reactAppPassthrough)

packages/react-router/tests/renderRouterToStream.test.tsx

@@ -232,3 +232,82 @@ describe('renderRouterToStream - pipeable sync errors', () => {
     }
   })
 })
+
+describe('renderRouterToStream - bot detection (isBot option)', () => {
+  const BOT_UA = 'Googlebot/2.1 (+http://www.google.com/bot.html)'
+  const HUMAN_UA =
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
+
+  function requestWith(headers: Record<string, string>) {
+    return new Request('http://localhost/', { headers })
+  }
+
+  // Captures whether renderToPipeableStream was given `onAllReady` (bot, wait
+  // for the full document) or `onShellReady` (stream the shell first).
+  async function getStreamMode(
+    request: Request,
+    isBot?: boolean | ((request: Request) => boolean),
+  ): Promise<'allReady' | 'shellReady' | undefined> {
+    const router = await buildRouter()
+    let mode: 'allReady' | 'shellReady' | undefined
+    reactDomServerMocks.renderToPipeableStream.mockImplementationOnce(
+      (_children, opts) => {
+        mode = opts.onAllReady ? 'allReady' : 'shellReady'
+        queueMicrotask(() => (opts.onAllReady ?? opts.onShellReady)())
+        return { abort: vi.fn(), pipe: vi.fn() }
+      },
+    )
+
+    const result = await renderRouterToStream({
+      request,
+      router,
+      responseHeaders: new Headers(),
+      children: null,
+      isBot,
+    })
+    try {
+      return mode
+    } finally {
+      await unwrapResponse(result)
+        .body?.cancel()
+        .catch(() => {})
+      router.serverSsr?.cleanup()
+    }
+  }
+
+  test('default: bot User-Agent waits for allReady', async () => {
+    expect(await getStreamMode(requestWith({ 'user-agent': BOT_UA }))).toBe(
+      'allReady',
+    )
+  })
+
+  test('default: human User-Agent streams the shell', async () => {
+    expect(await getStreamMode(requestWith({ 'user-agent': HUMAN_UA }))).toBe(
+      'shellReady',
+    )
+  })
+
+  test('isBot=false: bot User-Agent still streams the shell', async () => {
+    expect(
+      await getStreamMode(requestWith({ 'user-agent': BOT_UA }), false),
+    ).toBe('shellReady')
+  })
+
+  test('isBot=true: human User-Agent waits for allReady', async () => {
+    expect(
+      await getStreamMode(requestWith({ 'user-agent': HUMAN_UA }), true),
+    ).toBe('allReady')
+  })
+
+  test('isBot predicate receives the request and controls the mode', async () => {
+    const isBot = vi.fn(
+      (request: Request) => request.headers.get('x-prerender') === '1',
+    )
+    const request = requestWith({ 'user-agent': HUMAN_UA, 'x-prerender': '1' })
+
+    const mode = await getStreamMode(request, isBot)
+
+    expect(mode).toBe('allReady')
+    expect(isBot).toHaveBeenCalledWith(request)
+  })
+})

packages/solid-router/src/ssr/renderRouterToStream.tsx

@@ -38,12 +38,24 @@ export const renderRouterToStream = async ({
   router,
   responseHeaders,
   children,
+  isBot,
 }: {
   request: Request
   router: AnyRouter
   responseHeaders: Headers
   children: () => JSXElement
+  /**
+   * Whether to treat the request as a bot/crawler. Bots wait for the full
+   * server render before streaming. Defaults to the built-in `isbot`
+   * User-Agent check.
+   */
+  isBot?: boolean | ((request: Request) => boolean)
 }) => {
+  const isBotRequest =
+    typeof isBot === 'function'
+      ? isBot(request)
+      : (isBot ?? isbot(request.headers.get('User-Agent')))
+
   const { writable, readable } = new TransformStream()
 
   const docType = Solid.ssr('<!DOCTYPE html>')
@@ -121,7 +133,7 @@ export const renderRouterToStream = async ({
     })
   }
 
-  if (isbot(request.headers.get('User-Agent'))) {
+  if (isBotRequest) {
     await waitForReadyOrAbort(
       Promise.resolve(stream as unknown),
       request.signal,

packages/vue-router/src/ssr/renderRouterToStream.tsx

@@ -68,15 +68,27 @@ export const renderRouterToStream = async ({
   router,
   responseHeaders,
   App,
+  isBot,
 }: {
   request: Request
   router: AnyRouter
   responseHeaders: Headers
   App: Component
+  /**
+   * Whether to treat the request as a bot/crawler. Bots wait for the full
+   * document before streaming. Defaults to the built-in `isbot` User-Agent
+   * check.
+   */
+  isBot?: boolean | ((request: Request) => boolean)
 }) => {
   const app = Vue.createSSRApp(App, { router })
 
-  if (isbot(request.headers.get('User-Agent'))) {
+  const isBotRequest =
+    typeof isBot === 'function'
+      ? isBot(request)
+      : (isBot ?? isbot(request.headers.get('User-Agent')))
+
+  if (isBotRequest) {
     try {
       let cleanupAbortListener: (() => void) | undefined
       const abortPromise = new Promise<never>((_, reject) => {