Handle bill text size limit (#2169)

.prettierignore

@@ -17,3 +17,5 @@ llm
 playwright-report
 CLAUDE.md
 .cursor/
+.claude/
+.serena/

components/bill/Summary.tsx

@@ -4,7 +4,17 @@ import { useCallback, useEffect, useState } from "react"
 import type { ModalProps } from "react-bootstrap"
 import styled, { ThemeConsumer } from "styled-components"
 import { useMediaQuery } from "usehooks-ts"
-import { Button, Col, Container, Image, Modal, Row, Stack } from "../bootstrap"
+import {
+  Button,
+  Col,
+  Container,
+  Image,
+  Modal,
+  Row,
+  Spinner,
+  Stack
+} from "../bootstrap"
+import { getBillDocumentText } from "../db"
 import { firestore } from "../firebase"
 import * as links from "../links"
 import {
@@ -113,9 +123,30 @@ export const Summary = ({
 }: BillProps & { className?: string }) => {
   const [showBillDetails, setShowBillDetails] = useState(false)
   const [showFullSummary, setShowFullSummary] = useState(false)
-  const handleShowBillDetails = () => setShowBillDetails(true)
+  // Long bills store their text in a `contentBlocks` subcollection rather than
+  // inline; reassemble it lazily when the modal opens.
+  const [billText, setBillText] = useState<string | undefined>(
+    bill?.content?.DocumentText
+  )
+  const [loadingBillText, setLoadingBillText] = useState(false)
+  const hasBillText =
+    !!bill?.content?.DocumentText ||
+    (bill?.content?.DocumentTextBlockCount ?? 0) > 0
+
+  const handleShowBillDetails = useCallback(async () => {
+    setShowBillDetails(true)
+    if (billText === undefined && bill) {
+      setLoadingBillText(true)
+      try {
+        setBillText(
+          await getBillDocumentText(bill.court, bill.id, bill.content)
+        )
+      } finally {
+        setLoadingBillText(false)
+      }
+    }
+  }, [bill, billText])
   const handleHideBillDetails = () => setShowBillDetails(false)
-  const billText = bill?.content?.DocumentText
   const hearingIds = bill?.hearingIds
   const isBallotMeasure =
     bill?.currentCommittee?.id === currentBallotInitiativeCommittee
@@ -129,7 +160,7 @@ export const Summary = ({
         <TitleFormat>
           {bill.content.Title}
           <div className="d-flex justify-content-end">
-            {billText ? (
+            {hasBillText ? (
               <StyledButton
                 variant="link"
                 className="m-1"
@@ -156,9 +187,13 @@ export const Summary = ({
               <Modal.Title>{bill?.id}</Modal.Title>
             </Modal.Header>
             <Modal.Body className="bg-white">
-              <FormattedBillDetails>
-                {bill?.content?.DocumentText}
-              </FormattedBillDetails>
+              {loadingBillText ? (
+                <div className="d-flex justify-content-center">
+                  <Spinner animation="border" role="status" />
+                </div>
+              ) : (
+                <FormattedBillDetails>{billText}</FormattedBillDetails>
+              )}
             </Modal.Body>
           </Modal>
         </TitleFormat>

components/db/bills.ts

@@ -3,6 +3,7 @@ import {
   getDocs,
   limit,
   orderBy,
+  query,
   Timestamp,
   where
 } from "firebase/firestore"
@@ -34,6 +35,9 @@ export type BillContent = {
   LegislationTypeName: string
   Pinslip: string
   DocumentText?: string
+  /** Present only when DocumentText was too large to store inline and was
+   * chunked into the bill's `contentBlocks` subcollection. */
+  DocumentTextBlockCount?: number
 }
 
 export type BillTopic = {
@@ -87,6 +91,29 @@ export async function getBill(
   return bill as any
 }
 
+/**
+ * Resolve a bill's full document text. Returns the inline `DocumentText` when
+ * present; otherwise, for bills whose text was chunked past Firestore's 1 MiB
+ * limit, reassembles it from the ordered `contentBlocks` subcollection.
+ */
+export async function getBillDocumentText(
+  court: number,
+  id: string,
+  content: BillContent
+): Promise<string | undefined> {
+  if (content.DocumentText) return content.DocumentText
+  if (!content.DocumentTextBlockCount) return undefined
+
+  const blocksRef = collection(
+    firestore,
+    `/generalCourts/${court}/bills/${id}/contentBlocks`
+  )
+  const blocks = await getDocs(query(blocksRef, orderBy("index")))
+  if (blocks.empty) return undefined
+
+  return blocks.docs.map(d => (d.data().text as string) ?? "").join("")
+}
+
 export async function listBillsByHearingDate(
   limitCount: number
 ): Promise<Bill[]> {

functions/src/bills/bills.test.ts

@@ -1,11 +1,37 @@
+jest.mock("firebase-functions", () => ({
+  logger: { info: jest.fn(), warn: jest.fn() },
+  https: {}
+}))
 jest.mock("../malegislature", () => ({
   getDocument: jest.fn(),
-  getDocumentPdf: jest.fn()
+  getDocumentPdf: jest.fn(),
+  getBillHistory: jest.fn(),
+  getSimilarBills: jest.fn()
 }))
 jest.mock("./pdfText", () => ({
   extractBillTextFromPdf: jest.fn()
 }))
+// Avoid evaluating the real scraper (and Firebase init) when importing ./bills.
+jest.mock("../scraper", () => ({
+  createScraper: jest.fn(() => ({ fetchBatch: {}, startBatches: {} }))
+}))
+jest.mock("../firebase", () => ({
+  Timestamp: { fromMillis: jest.fn(() => "TS0") },
+  FieldValue: { delete: jest.fn(() => "__DELETE__") }
+}))
+jest.mock("./contentBlocks", () => ({
+  planDocumentTextStorage: jest.fn(),
+  writeDocumentTextBlocks: jest.fn(),
+  clearDocumentTextBlocks: jest.fn()
+}))
 
+import { FieldValue } from "../firebase"
+import { fetchBillResource } from "./bills"
+import {
+  clearDocumentTextBlocks,
+  planDocumentTextStorage,
+  writeDocumentTextBlocks
+} from "./contentBlocks"
 import { getDocumentWithPdfTextFallback } from "./documentTextFallback"
 import { extractBillTextFromPdf } from "./pdfText"
 
@@ -80,3 +106,83 @@ describe("getDocumentWithPdfTextFallback", () => {
     })
   })
 })
+
+describe("fetchBillResource document text storage", () => {
+  const mockedApi = jest.requireMock("../malegislature") as {
+    getDocument: jest.Mock
+    getBillHistory: jest.Mock
+    getSimilarBills: jest.Mock
+  }
+  const mockedPlan = planDocumentTextStorage as jest.MockedFunction<
+    typeof planDocumentTextStorage
+  >
+  const mockedWrite = writeDocumentTextBlocks as jest.MockedFunction<
+    typeof writeDocumentTextBlocks
+  >
+  const mockedClear = clearDocumentTextBlocks as jest.MockedFunction<
+    typeof clearDocumentTextBlocks
+  >
+
+  beforeEach(() => {
+    jest.resetAllMocks()
+    mockedApi.getBillHistory.mockResolvedValue([])
+    mockedApi.getSimilarBills.mockResolvedValue([])
+    mockedApi.getDocument.mockResolvedValue({
+      DocumentText: "bill text",
+      Cosponsors: []
+    })
+    ;(FieldValue.delete as jest.Mock).mockReturnValue("__DELETE__")
+    mockedWrite.mockResolvedValue(undefined)
+    mockedClear.mockResolvedValue(undefined)
+  })
+
+  it("keeps text inline when it fits", async () => {
+    mockedPlan.mockReturnValue({ inline: "bill text" })
+
+    const { content } = (await fetchBillResource(194, "H1")) as any
+
+    expect(mockedWrite).not.toHaveBeenCalled()
+    expect(mockedClear).not.toHaveBeenCalled()
+    expect(content.DocumentText).toBe("bill text")
+    expect(content).not.toHaveProperty("DocumentTextBlockCount")
+  })
+
+  it("chunks oversized text into blocks and drops the inline copy", async () => {
+    mockedPlan.mockReturnValue({ blocks: ["a", "b"] })
+
+    const { content } = (await fetchBillResource(194, "H5500")) as any
+
+    expect(mockedWrite).toHaveBeenCalledWith(194, "H5500", ["a", "b"])
+    expect(content.DocumentTextBlockCount).toBe(2)
+    expect(content.DocumentText).toBe("__DELETE__")
+  })
+
+  it("drops text without a count when block writing fails", async () => {
+    mockedPlan.mockReturnValue({ blocks: ["a"] })
+    mockedWrite.mockRejectedValue(new Error("write failed"))
+
+    const { content } = (await fetchBillResource(194, "H5500", {
+      content: { DocumentTextBlockCount: 3 }
+    })) as any
+
+    expect(mockedClear).toHaveBeenCalledWith(194, "H5500")
+    expect(content.DocumentText).toBe("__DELETE__")
+    expect(content.DocumentTextBlockCount).toBe("__DELETE__")
+  })
+
+  it("clears stale blocks when a previously chunked bill now fits inline", async () => {
+    mockedPlan.mockReturnValue({ inline: "small" })
+    mockedApi.getDocument.mockResolvedValue({
+      DocumentText: "small",
+      Cosponsors: []
+    })
+
+    const { content } = (await fetchBillResource(194, "H5500", {
+      content: { DocumentTextBlockCount: 2 }
+    })) as any
+
+    expect(mockedClear).toHaveBeenCalledWith(194, "H5500")
+    expect(content.DocumentText).toBe("small")
+    expect(content.DocumentTextBlockCount).toBe("__DELETE__")
+  })
+})