improvement(provenance): aggregate and attribute unrecorded durable reads (#7017)

Fail-open on unrecorded durable provenance rests on one compensating
control: the audit entry telling the people who own the secrets that a
read proceeded unvouched. An audit of all four surfaces found the
control incomplete in exactly the places this closes, and confirmed the
policy itself sound — so nothing here changes what any read or write
does, only what gets recorded about it.

Knowledge was the one surface with no audit trail at all: the
per-record import reports without a workspace, and the report skips the
audit row when it cannot name one, so fail-open knowledge reads emitted
one error log line per record and zero audit entries. Both importers
now count unrecorded records while the surface is open and report once
per read with the workspace, actor, and count — the shape memory and
tables already use. The search read reports once across chunks and
rendered metadata, and only when the registry did not latch, since a
latched read never reaches a model. Fault returns stay silent; those
reads fail closed.

Memory had the one silent local degrade: a record whose canonical hash
outgrows its bounds, or whose entries fail normalization, was stored
unknown with nothing logged anywhere — the table writer logs its
equivalent. The binding now logs the cause at error where it is
decided. An incoming unknown stays silent; its producer already
reported.

The memory list contract gains the page ceiling every other list
already has (max 1000, matching the table convention); no caller in
the repo passes a limit at all, and the route is internal-auth only.

Workspace-file audit rows now carry the acting user where the caller
already holds one — copilot vfs, the agent and mothership handlers,
and the provider attachment filter. Everywhere else, including
principals with no user to name, the actor stays null, which the
report type has always permitted.

Two comments catch up with the code: the file sidecar stores three
statuses since the absence/taint split, and the mounted-file scanner's
scan-overflow-to-taint is deliberate where the registry scan
over-approximates — that scan only narrows an already-sound candidate
set, while this one decides whether egress redaction would suffice for
bytes the same matcher just failed on.

Author: icecrasher321

apps/sim/app/api/knowledge/secret-provenance.ts

@@ -248,6 +248,8 @@ export async function finalizeKnowledgePersistedResponse(options: {
     registry,
     documents: options.documents,
     chunks: options.chunks,
+    ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}),
+    actorUserId: options.userId,
   })
   return finalizeKnowledgeRegistryResponse({
     request: options.request,

apps/sim/app/api/memory/secret-provenance.test.ts

@@ -18,6 +18,7 @@ vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({
   reportUnrecordedDurableProvenance: mockReport,
 }))
 
+import { memoryListQuerySchema } from '@/lib/api/contracts/memory'
 import { AuthType } from '@/lib/auth/hybrid'
 import {
   PRIVATE_SECRET_PROVENANCE_BUNDLE_V1,
@@ -341,3 +342,11 @@ describe('memory write secret provenance', () => {
     expect(mockReport).not.toHaveBeenCalled()
   })
 })
+
+describe('memory list query contract', () => {
+  it('rejects a limit past the page ceiling and keeps the default below it', () => {
+    expect(memoryListQuerySchema.safeParse({ limit: '2000' }).success).toBe(false)
+    expect(memoryListQuerySchema.parse({})).toMatchObject({ limit: 50 })
+    expect(memoryListQuerySchema.parse({ limit: '1000' })).toMatchObject({ limit: 1000 })
+  })
+})

apps/sim/executor/handlers/agent/agent-handler.ts

@@ -1680,6 +1680,7 @@ export class AgentBlockHandler implements BlockHandler {
               identity,
               registry: ctx.resolvedSecretTraceRegistry,
               view: 'opaque',
+              ...(ctx.userId ? { actorUserId: ctx.userId } : {}),
             })
             if (!safe) {
               unsafeGeneratedDocumentFiles.add(`${file.key}:${file.id}`)

apps/sim/executor/handlers/mothership/mothership-handler.ts

@@ -689,7 +689,7 @@ async function buildMothershipFileAttachments(
   )
   const modelSafe = await areModelSafeWorkspaceFileKeys(
     userFiles.map((file) => file.key).filter((key): key is string => Boolean(key)),
-    { workspaceId: ctx.workspaceId }
+    { workspaceId: ctx.workspaceId, ...(ctx.userId ? { actorUserId: ctx.userId } : {}) }
   )
   if (!modelSafe) throw new Error(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE)
 

apps/sim/lib/api/contracts/memory.ts

@@ -34,7 +34,13 @@ export const agentMemoryDataSchemaContract = agentMemoryDataSchema
 export const memoryListQuerySchema = z.object({
   workspaceId: z.string().optional(),
   query: z.string().nullable().optional(),
-  limit: z.coerce.number().int().min(1).optional().default(50),
+  limit: z.coerce
+    .number()
+    .int()
+    .min(1)
+    .max(1000, 'Cannot list more than 1000 memories per request')
+    .optional()
+    .default(50),
 })
 
 export const memoryMessageSchema = z

apps/sim/lib/copilot/tools/handlers/vfs.ts

@@ -126,6 +126,7 @@ async function canReturnWorkspaceFileValue(
         registry: context.resolvedSecretTraceRegistry,
         view: provenanceView,
         value,
+        actorUserId: context.userId,
       }))
     ) {
       return false

apps/sim/lib/execution/mounted-file-secret-provenance.ts

@@ -89,6 +89,14 @@ export async function createMountedFileSecretProvenanceScanner(
 
   return {
     hasSecrets,
+    /**
+     * A scan that cannot finish yields `unknown` — a taint — where the registry's per-value scan
+     * over-approximates instead. The asymmetry is deliberate: that scan only narrows a candidate
+     * set that is already a sound answer, while this one decides whether egress redaction of these
+     * entries would suffice for these bytes — a claim that cannot be made for content the same
+     * matcher just failed on. Reaching the event bound takes an eight-plus-character literal
+     * occurring ~a million times, so only degenerate content pays the refusal.
+     */
     scan(buffer) {
       const matched = new Map<string, WorkspaceFileSecretProvenanceEntry>()
       try {

apps/sim/lib/knowledge/application/search.ts

@@ -15,6 +15,10 @@ import { OrchestrationError } from '@/lib/core/orchestration/types'
 import { PlatformEvents } from '@/lib/core/telemetry'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { importDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance'
+import {
+  isDurableSecretProvenanceEnforced,
+  reportUnrecordedDurableProvenance,
+} from '@/lib/execution/durable-secret-provenance-enforcement'
 import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case'
 import {
   KnowledgeUsageLimitExceededError,
@@ -488,6 +492,8 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({
       }
     })
     if (registry && provenanceSnapshot) {
+      const knowledgeEnforced = isDurableSecretProvenanceEnforced('knowledge')
+      let unrecordedCount = provenanceSnapshot.unrecordedCount
       for (const [documentId, document] of Object.entries(provenanceSnapshot.documentMetadata)) {
         const renderedMetadata = results
           .filter((result) => result.documentId === documentId)
@@ -496,18 +502,34 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({
             sourceUrl: result.sourceUrl,
             metadata: result.metadata,
           }))
+        if (renderedMetadata.length === 0) continue
+        if (document.provenance.status === 'unknown' && !knowledgeEnforced) unrecordedCount += 1
         if (
-          renderedMetadata.length > 0 &&
           !(await importDurableSecretProvenance(
             registry,
             document.provenance,
             renderedMetadata,
-            'knowledge'
+            'knowledge',
+            { reportUnrecorded: false }
           ))
         ) {
           registry.markIncomplete('knowledge-result-provenance-unavailable')
         }
       }
+      /**
+       * One entry for the whole search — chunks and rendered metadata are one read. Skipped when
+       * the registry latched: a latched read never reaches a model, and this entry exists to say a
+       * fail-open read went ahead unvouched.
+       */
+      if (unrecordedCount > 0 && !registry.isPermanentlyIncomplete()) {
+        reportUnrecordedDurableProvenance({
+          surface: 'knowledge',
+          cause: 'durable-provenance-unknown',
+          affectedCount: unrecordedCount,
+          workspaceId: context.workspaceId,
+          actorUserId: userId,
+        })
+      }
     }
     const cost = baseCost
       ? {

apps/sim/lib/knowledge/secret-provenance.test.ts

@@ -1,24 +1,34 @@
 /**
  * @vitest-environment node
  */
-import { document } from '@sim/db/schema'
+import { document, embedding } from '@sim/db/schema'
 import { queueTableRows, resetDbChainMock } from '@sim/testing'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import { hashDurableSecretProvenanceValue } from '@/lib/execution/durable-secret-provenance'
 import {
   createKnowledgeDocumentSourceValue,
+  importKnowledgePersistedResponseSecretProvenance,
+  importKnowledgeSearchResultSecretProvenance,
   loadKnowledgeDocumentSecretRegistry,
   readBoundKnowledgeDocumentSecretProvenance,
 } from '@/lib/knowledge/secret-provenance'
+import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
-const { mockDecryptSecret } = vi.hoisted(() => ({
+const { mockDecryptSecret, mockIsEnforced, mockReport } = vi.hoisted(() => ({
   mockDecryptSecret: vi.fn(),
+  mockIsEnforced: vi.fn(() => false),
+  mockReport: vi.fn(),
 }))
 
 vi.mock('@/lib/core/security/encryption', () => ({
   decryptSecret: mockDecryptSecret,
 }))
 
+vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({
+  isDurableSecretProvenanceEnforced: mockIsEnforced,
+  reportUnrecordedDurableProvenance: mockReport,
+}))
+
 const DOCUMENT_SOURCE = createKnowledgeDocumentSourceValue({
   filename: 'source.pdf',
   fileUrl: '/api/files/serve/workspace%2Fworkspace-1%2Fsource.pdf?context=workspace',
@@ -39,6 +49,7 @@ describe('knowledge durable secret provenance', () => {
     resetDbChainMock()
     queueTableRows(document, [DOCUMENT_ROW])
     mockDecryptSecret.mockResolvedValue({ decrypted: 'tracked-secret' })
+    mockIsEnforced.mockReturnValue(false)
   })
 
   it('uses the same explicit source shape for joined rows and persisted writes', () => {
@@ -130,3 +141,108 @@ describe('knowledge durable secret provenance', () => {
     })
   })
 })
+
+describe('knowledge unrecorded-read reporting', () => {
+  const SCOPE = { userId: 'user-1', workspaceId: 'workspace-1' }
+  const UNRECORDED_DOCUMENT_ROW = {
+    id: 'doc-1',
+    ...DOCUMENT_SOURCE,
+    secretProvenanceVersion: 1,
+    provenanceSourceHash: null,
+    status: 'unknown',
+    entries: null,
+  }
+  const UNRECORDED_CHUNK_ROW = {
+    id: 'chunk-1',
+    documentId: 'doc-1',
+    content: 'chunk text',
+    chunkHash: 'stale',
+    secretProvenanceVersion: 1,
+    provenanceContentHash: null,
+    status: 'unknown',
+    entries: null,
+  }
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+    mockIsEnforced.mockReturnValue(false)
+  })
+
+  it('reports one aggregated entry per read, naming workspace, actor, and count', async () => {
+    queueTableRows(document, [UNRECORDED_DOCUMENT_ROW])
+    queueTableRows(embedding, [UNRECORDED_CHUNK_ROW])
+    const registry = new ResolvedSecretTraceRegistry([], SCOPE)
+
+    await expect(
+      importKnowledgePersistedResponseSecretProvenance({
+        registry,
+        documents: [{ id: 'doc-1', source: DOCUMENT_SOURCE, value: {} }],
+        chunks: [{ id: 'chunk-1', documentId: 'doc-1', content: 'chunk text', value: {} }],
+        workspaceId: 'workspace-1',
+        actorUserId: 'user-1',
+      })
+    ).resolves.toBe(true)
+
+    expect(registry.isPermanentlyIncomplete()).toBe(false)
+    expect(mockReport).toHaveBeenCalledTimes(1)
+    expect(mockReport).toHaveBeenCalledWith({
+      surface: 'knowledge',
+      cause: 'durable-provenance-unknown',
+      affectedCount: 2,
+      workspaceId: 'workspace-1',
+      actorUserId: 'user-1',
+    })
+  })
+
+  /** A fault return fails the read closed, so no unvouched record reached anything to report. */
+  it('reports nothing when the read fails closed on a missing row', async () => {
+    queueTableRows(document, [])
+    const registry = new ResolvedSecretTraceRegistry([], SCOPE)
+
+    await expect(
+      importKnowledgePersistedResponseSecretProvenance({
+        registry,
+        documents: [{ id: 'doc-1', source: DOCUMENT_SOURCE, value: {} }],
+        workspaceId: 'workspace-1',
+        actorUserId: 'user-1',
+      })
+    ).resolves.toBe(false)
+
+    expect(mockReport).not.toHaveBeenCalled()
+  })
+
+  it('latches without reporting once the surface is enforced', async () => {
+    mockIsEnforced.mockReturnValue(true)
+    queueTableRows(document, [UNRECORDED_DOCUMENT_ROW])
+    const registry = new ResolvedSecretTraceRegistry([], SCOPE)
+
+    await expect(
+      importKnowledgePersistedResponseSecretProvenance({
+        registry,
+        documents: [{ id: 'doc-1', source: DOCUMENT_SOURCE, value: {} }],
+        workspaceId: 'workspace-1',
+        actorUserId: 'user-1',
+      })
+    ).resolves.toBe(false)
+
+    expect(registry.isPermanentlyIncomplete()).toBe(true)
+    expect(mockReport).not.toHaveBeenCalled()
+  })
+
+  /** The search read spans chunks and rendered metadata, so its caller owns the one report. */
+  it('returns the unrecorded count from a search import instead of reporting it', async () => {
+    queueTableRows(embedding, [{ ...UNRECORDED_CHUNK_ROW, documentId: DOCUMENT_ROW.id }])
+    queueTableRows(document, [DOCUMENT_ROW])
+    const registry = new ResolvedSecretTraceRegistry([], SCOPE)
+
+    const snapshot = await importKnowledgeSearchResultSecretProvenance({
+      registry,
+      results: [{ id: 'chunk-1', documentId: DOCUMENT_ROW.id, content: 'chunk text' }],
+    })
+
+    expect(snapshot.imported).toBe(true)
+    expect(snapshot.unrecordedCount).toBe(1)
+    expect(mockReport).not.toHaveBeenCalled()
+  })
+})