diff --git a/.changeset/encrypted-search-idb.md b/.changeset/encrypted-search-idb.md new file mode 100644 index 000000000..2b8b14a87 --- /dev/null +++ b/.changeset/encrypted-search-idb.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +Add IndexedDB-backed persistent search index for all rooms via a MiniSearch web worker with multi-tab write safety and LRU eviction. Note: indexed message text is stored unencrypted in IndexedDB. diff --git a/config.json b/config.json index 2809e4f68..c18f788f4 100644 --- a/config.json +++ b/config.json @@ -44,5 +44,9 @@ "hashRouter": { "enabled": false, "basename": "/" + }, + + "features": { + "encryptedSearch": true } } diff --git a/package.json b/package.json index 2d605aebc..1db4173a0 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "marked": "^18.0.2", "matrix-js-sdk": "^38.4.0", "matrix-widget-api": "^1.16.1", + "minisearch": "^7.2.0", "pdfjs-dist": "^5.4.624", "react": "^18.3.1", "react-aria": "^3.46.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ee7836185..2d61b2251 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -153,6 +153,9 @@ importers: matrix-widget-api: specifier: ^1.16.1 version: 1.17.0 + minisearch: + specifier: ^7.2.0 + version: 7.2.0 pdfjs-dist: specifier: ^5.4.624 version: 5.5.207 @@ -4223,6 +4226,9 @@ packages: resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} engines: {node: '>=16 || 14 >=14.17'} + minisearch@7.2.0: + resolution: {integrity: sha512-dqT2XBYUOZOiC5t2HRnwADjhNS2cecp9u+TJRiJ1Qp/f5qjkeT5APcGPjHw+bz89Ms8Jp+cG4AlE+QZ/QnDglg==} + mlly@1.8.1: resolution: {integrity: sha512-SnL6sNutTwRWWR/vcmCYHSADjiEesp5TGQQ0pXyLhW5IoeibRlF/CbSLailbB3CNqJUk9cVJ9dUDnbD7GrcHBQ==} @@ -9462,6 +9468,8 @@ snapshots: minipass@7.1.3: {} + minisearch@7.2.0: {} + mlly@1.8.1: dependencies: acorn: 8.16.0 diff --git a/src/app/components/room-avatar/AvatarImage.tsx b/src/app/components/room-avatar/AvatarImage.tsx index f322bce0e..df29fd263 100644 --- a/src/app/components/room-avatar/AvatarImage.tsx +++ b/src/app/components/room-avatar/AvatarImage.tsx @@ -6,24 +6,56 @@ import { settingsAtom } from '$state/settings'; import { useSetting } from '$state/hooks/settings'; import * as css from './RoomAvatar.css'; -type AvatarImageProps = { - src: string; - alt?: string; - uniformIcons?: boolean; - onError: () => void; -}; +// Module-level cache: maps a Matrix media URL → processed blob URL so that +// SVG processing only runs once per unique image, even as virtual-list items +// unmount and remount. MXC URLs are content-addressed and never change, so +// the mapping is stable for the lifetime of the page. +const svgBlobCache = new Map(); -export function AvatarImage({ src, alt, uniformIcons, onError }: AvatarImageProps) { - const [uniformIconsSetting] = useSetting(settingsAtom, 'uniformIcons'); - const [image, setImage] = useState(undefined); - const [processedSrc, setProcessedSrc] = useState(src); +/** Number of SVG blob URLs currently held in the module-level cache. */ +export function getSvgCacheSize(): number { + return svgBlobCache.size; +} - const useUniformIcons = uniformIconsSetting && uniformIcons === true; - const normalizedBg = useUniformIcons && image ? bgColorImg(image) : undefined; +/** Revoke all cached SVG blob URLs and clear the cache to free memory. */ +export function clearSvgBlobCache(): void { + svgBlobCache.forEach((url) => URL.revokeObjectURL(url)); + svgBlobCache.clear(); +} + +/** + * Resolves an avatar HTTP URL through the SVG blob cache. + * - If `src` is already cached as a processed blob URL, returns it immediately. + * - If `src` is an SVG, fetches, sanitises animations, stores in cache, and + * returns the blob URL (falls back to raw `src` on error). + * - For non-SVG images, returns `src` unchanged (no extra processing needed). + * - If `src` is `undefined`, returns `undefined`. + * + * Sharing this hook between `AvatarImage` (room avatars) and `UserAvatar` + * (user avatars) means SVG avatars are processed and cached only once, + * regardless of which component first encounters them. + */ +export function useProcessedAvatarSrc(src: string | undefined): string | undefined { + const [processedSrc, setProcessedSrc] = useState(src); useEffect(() => { + if (!src) { + setProcessedSrc(undefined); + return; + } + let isMounted = true; - let objectUrl: string | null = null; + + // Reset to raw src while we check/process, so stale blob URLs never linger. + setProcessedSrc(src); + + const cachedBlobUrl = svgBlobCache.get(src); + if (cachedBlobUrl) { + setProcessedSrc(cachedBlobUrl); + return () => { + isMounted = false; + }; + } const processImage = async () => { try { @@ -46,8 +78,10 @@ export function AvatarImage({ src, alt, uniformIcons, onError }: AvatarImageProp const newSvgString = serializer.serializeToString(doc); const blob = new Blob([newSvgString], { type: 'image/svg+xml' }); - objectUrl = URL.createObjectURL(blob); - if (isMounted) setProcessedSrc(objectUrl); + const blobUrl = URL.createObjectURL(blob); + // Store in module cache so future remounts skip processing. + svgBlobCache.set(src, blobUrl); + if (isMounted) setProcessedSrc(blobUrl); } else if (isMounted) setProcessedSrc(src); } catch { if (isMounted) setProcessedSrc(src); @@ -58,12 +92,29 @@ export function AvatarImage({ src, alt, uniformIcons, onError }: AvatarImageProp return () => { isMounted = false; - if (objectUrl) { - URL.revokeObjectURL(objectUrl); - } + // Blob URLs are retained in svgBlobCache — do not revoke them here so + // that subsequent remounts can use the cached result without re-fetching. }; }, [src]); + return processedSrc; +} + +type AvatarImageProps = { + src: string; + alt?: string; + uniformIcons?: boolean; + onError: () => void; +}; + +export function AvatarImage({ src, alt, uniformIcons, onError }: AvatarImageProps) { + const [uniformIconsSetting] = useSetting(settingsAtom, 'uniformIcons'); + const [image, setImage] = useState(undefined); + const processedSrc = useProcessedAvatarSrc(src) ?? src; + + const useUniformIcons = uniformIconsSetting && uniformIcons === true; + const normalizedBg = useUniformIcons && image ? bgColorImg(image) : undefined; + const handleLoad: ReactEventHandler = (evt) => { evt.currentTarget.setAttribute('data-image-loaded', 'true'); setImage(evt.currentTarget); diff --git a/src/app/components/user-avatar/UserAvatar.tsx b/src/app/components/user-avatar/UserAvatar.tsx index 78288d393..2a0c9fd2d 100644 --- a/src/app/components/user-avatar/UserAvatar.tsx +++ b/src/app/components/user-avatar/UserAvatar.tsx @@ -3,6 +3,7 @@ import type { ReactEventHandler, ReactNode } from 'react'; import { useEffect, useState } from 'react'; import classNames from 'classnames'; import colorMXID from '$utils/colorMXID'; +import { useProcessedAvatarSrc } from '$components/room-avatar/AvatarImage'; import * as css from './UserAvatar.css'; type UserAvatarProps = { @@ -19,12 +20,13 @@ const handleImageLoad: ReactEventHandler = (evt) => { export function UserAvatar({ className, userId, src, alt, renderFallback }: UserAvatarProps) { const [error, setError] = useState(false); + const processedSrc = useProcessedAvatarSrc(src); useEffect(() => { setError(false); }, [src]); - if (!src || error) { + if (!processedSrc || error) { return ( setError(true)} onLoad={handleImageLoad} diff --git a/src/app/features/message-search/MessageSearch.tsx b/src/app/features/message-search/MessageSearch.tsx index 893ff00eb..434a2bb16 100644 --- a/src/app/features/message-search/MessageSearch.tsx +++ b/src/app/features/message-search/MessageSearch.tsx @@ -1,7 +1,6 @@ import type { RefObject } from 'react'; -import { useEffect, useMemo, useRef } from 'react'; +import { useCallback, useEffect, useMemo, useRef } from 'react'; import { Text, Box, Icon, Icons, config, Spinner, IconButton, Line, toRem } from 'folds'; -import { useAtomValue } from 'jotai'; import { useVirtualizer } from '@tanstack/react-virtual'; import { useInfiniteQuery } from '@tanstack/react-query'; import { useSearchParams } from 'react-router-dom'; @@ -16,12 +15,15 @@ import { useRoomNavigate } from '$hooks/useRoomNavigate'; import { ScrollTopContainer } from '$components/scroll-top-container'; import { ContainerColor } from '$styles/ContainerColor.css'; import { decodeSearchParamValueArray, encodeSearchParamValueArray } from '$pages/pathUtils'; -import { useRooms } from '$state/hooks/roomList'; +import { useSelectedRooms } from '$state/hooks/roomList'; import { allRoomsAtom } from '$state/room-list/roomList'; +import { isRoom } from '$utils/room'; +import { useAtomValue } from 'jotai'; import { mDirectAtom } from '$state/mDirectList'; import { VirtualTile } from '$components/virtualizer'; import type { MessageSearchParams } from './useMessageSearch'; import { useMessageSearch } from './useMessageSearch'; +import type { SearchHasType } from './useMessageSearch'; import { SearchResultGroup } from './SearchResultGroup'; import { SearchInput } from './SearchInput'; import { SearchFilters } from './SearchFilters'; @@ -34,6 +36,7 @@ const useSearchPathSearchParams = (searchParams: URLSearchParams): SearchPathSea order: searchParams.get('order') ?? undefined, rooms: searchParams.get('rooms') ?? undefined, senders: searchParams.get('senders') ?? undefined, + has: searchParams.get('has') ?? undefined, }), [searchParams] ); @@ -45,6 +48,9 @@ type MessageSearchProps = { senders?: string[]; scrollRef: RefObject; }; + +const VALID_HAS_TYPES = new Set(['image', 'file', 'audio', 'video', 'link']); + export function MessageSearch({ defaultRoomsFilterName, allowGlobal, @@ -54,7 +60,8 @@ export function MessageSearch({ }: Readonly) { const mx = useMatrixClient(); const mDirects = useAtomValue(mDirectAtom); - const allRooms = useRooms(mx, allRoomsAtom, mDirects); + const allRoomsSelector = useCallback((rId: string) => isRoom(mx.getRoom(rId)), [mx]); + const allRooms = useSelectedRooms(allRoomsAtom, allRoomsSelector); const [mediaAutoLoad] = useSetting(settingsAtom, 'mediaAutoLoad'); const [urlPreview] = useSetting(settingsAtom, 'urlPreview'); const [legacyUsernameColor] = useSetting(settingsAtom, 'legacyUsernameColor'); @@ -83,9 +90,18 @@ export function MessageSearch({ } return undefined; }, [searchPathSearchParams.senders]); + const searchParamHasTypes = useMemo(() => { + if (!searchPathSearchParams.has) return undefined; + const decoded = decodeSearchParamValueArray(searchPathSearchParams.has).filter( + (t): t is SearchHasType => VALID_HAS_TYPES.has(t as SearchHasType) + ); + return decoded.length > 0 ? decoded : undefined; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [searchPathSearchParams.has]); + + const isGlobal = searchPathSearchParams.global === 'true'; const msgSearchParams: MessageSearchParams = useMemo(() => { - const isGlobal = searchPathSearchParams.global === 'true'; const defaultRooms = isGlobal ? undefined : rooms; return { @@ -93,19 +109,30 @@ export function MessageSearch({ order: searchPathSearchParams.order ?? SearchOrderBy.Recent, rooms: searchParamRooms ?? defaultRooms, senders: searchParamsSenders ?? senders, + hasTypes: searchParamHasTypes, }; - }, [searchPathSearchParams, searchParamRooms, searchParamsSenders, rooms, senders]); + }, [ + isGlobal, + searchPathSearchParams, + searchParamRooms, + searchParamsSenders, + searchParamHasTypes, + rooms, + senders, + ]); const searchMessages = useMessageSearch(msgSearchParams); const { status, data, error, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteQuery({ - enabled: !!msgSearchParams.term, + enabled: + !!msgSearchParams.term || (!!msgSearchParams.hasTypes && msgSearchParams.hasTypes.length > 0), queryKey: [ 'search', msgSearchParams.term, msgSearchParams.order, msgSearchParams.rooms, msgSearchParams.senders, + msgSearchParams.hasTypes, ], queryFn: ({ pageParam }) => searchMessages(pageParam), initialPageParam: '', @@ -117,6 +144,8 @@ export function MessageSearch({ const mixed = data?.pages.flatMap((result) => result.highlights); return Array.from(new Set(mixed)); }, [data]); + // Only the first page carries in-memory results (no pagination for encrypted rooms) + const inMemoryRoomCount = data?.pages[0]?.inMemoryRoomCount ?? 0; const virtualizer = useVirtualizer({ count: groups.length, @@ -177,6 +206,28 @@ export function MessageSearch({ }); }; + const handleHasTypesChange = (hasTypes?: SearchHasType[]) => { + setSearchParams((prevParams) => { + const newParams = new URLSearchParams(prevParams); + newParams.delete('has'); + if (hasTypes && hasTypes.length > 0) { + newParams.append('has', encodeSearchParamValueArray(hasTypes)); + } + return newParams; + }); + }; + + const handleSendersChange = (newSenders?: string[]) => { + setSearchParams((prevParams) => { + const newParams = new URLSearchParams(prevParams); + newParams.delete('senders'); + if (newSenders && newSenders.length > 0) { + newParams.append('senders', encodeSearchParamValueArray(newSenders)); + } + return newParams; + }); + }; + const lastVItem = vItems.at(-1); const lastVItemIndex: number | undefined = lastVItem?.index; const lastGroupIndex = groups.length - 1; @@ -216,16 +267,35 @@ export function MessageSearch({ + {inMemoryRoomCount > 0 && status !== 'pending' && ( + + + + {`${inMemoryRoomCount} ${inMemoryRoomCount === 1 ? 'room' : 'rooms'} searched from local cache only.`} + + + )} + {!msgSearchParams.term && status === 'pending' && ( @@ -268,7 +338,13 @@ export function MessageSearch({ {vItems.length > 0 && ( - {`Results for "${msgSearchParams.term}"`} + + {msgSearchParams.term + ? `Results for "${msgSearchParams.term}"` + : msgSearchParams.hasTypes && msgSearchParams.hasTypes.length > 0 + ? `Results for ${msgSearchParams.hasTypes.join(', ')}` + : 'Results'} +
(null); const [menuAnchor, setMenuAnchor] = useState(); const [localSelected, setLocalSelected] = useState(selectedRooms); @@ -269,6 +281,13 @@ function SelectRoomButton({ roomList, selectedRooms, onChange }: SelectRoomButto src={getRoomIconSrc(Icons, room.getType(), room.getJoinRule())} /> } + after={ + encryptedSearchActive && mx.isRoomEncrypted(roomId) ? ( + + + + ) : null + } > {room.name} @@ -317,28 +336,281 @@ function SelectRoomButton({ roomList, selectedRooms, onChange }: SelectRoomButto ); } +const HAS_FILTER_OPTIONS: { type: SearchHasType; label: string; icon: IconSrc }[] = [ + { type: 'image', label: 'Image', icon: Icons.Photo }, + { type: 'file', label: 'File', icon: Icons.File }, + { type: 'audio', label: 'Audio', icon: Icons.VolumeHigh }, + { type: 'video', label: 'Video', icon: Icons.Play }, + { type: 'link', label: 'Link', icon: Icons.Link }, +]; + +type HasFilterChipsProps = { + hasTypes?: SearchHasType[]; + onChange: (hasTypes?: SearchHasType[]) => void; +}; +function HasFilterChips({ hasTypes, onChange }: HasFilterChipsProps) { + const toggle = (type: SearchHasType) => { + if (hasTypes?.includes(type)) { + const next = hasTypes.filter((t) => t !== type); + onChange(next.length > 0 ? next : undefined); + } else { + onChange([...(hasTypes ?? []), type]); + } + }; + + return ( + <> + {HAS_FILTER_OPTIONS.map(({ type, label, icon }) => { + const active = hasTypes?.includes(type); + return ( + : } + outlined + onClick={() => toggle(type)} + > + {label} + + ); + })} + + ); +} + +type SelectSenderButtonProps = { + roomList: string[]; + selectedSenders?: string[]; + onChange: (senders?: string[]) => void; +}; + +const SENDER_SEARCH_OPTS: UseAsyncSearchOptions = { limit: 50, matchOptions: { contain: true } }; +const SENDER_DEBOUNCE_OPTS: DebounceOptions = { wait: 200 }; +const getMemberStr: SearchItemStrGetter = (member, query) => { + const name = member.name ?? member.userId; + return query ? [name, member.userId] : name; +}; +const getMemberDisplayName = (member: RoomMember): string => member.name ?? member.userId; +function SelectSenderButton({ roomList, selectedSenders, onChange }: SelectSenderButtonProps) { + const mx = useMatrixClient(); + const useAuthentication = useMediaAuthentication(); + const [menuAnchor, setMenuAnchor] = useState(); + const scrollRef = useRef(null); + + const members = useMemo(() => { + const seen = new Set(); + const result: RoomMember[] = []; + const scope = roomList.length > 0 ? roomList : []; + for (const roomId of scope) { + const room = mx.getRoom(roomId); + if (!room) continue; + for (const m of room.getMembers()) { + if (!seen.has(m.userId)) { + seen.add(m.userId); + result.push(m); + } + } + } + return result.toSorted((a, b) => + getMemberDisplayName(a).localeCompare(getMemberDisplayName(b)) + ); + }, [mx, roomList]); + + const [searchState, searchMembersRaw, resetSearch] = useAsyncSearch( + members, + getMemberStr, + SENDER_SEARCH_OPTS + ); + const searchMembers = useDebounce(searchMembersRaw, SENDER_DEBOUNCE_OPTS); + const handleSearchChange: ChangeEventHandler = (evt) => { + const value = evt.currentTarget.value.trim(); + if (!value) { + resetSearch(); + return; + } + searchMembers(value); + }; + + const displayMembers = searchState?.items ?? members; + + const virtualizer = useVirtualizer({ + count: displayMembers.length, + getScrollElement: () => scrollRef.current, + estimateSize: () => 32 + 4, + overscan: 10, + }); + const vItems = virtualizer.getVirtualItems(); + + const handleOpenMenu: MouseEventHandler = (evt) => { + setMenuAnchor(evt.currentTarget.getBoundingClientRect()); + }; + + const handleMemberClick: MouseEventHandler = (evt) => { + const userId = evt.currentTarget.getAttribute('data-user-id'); + if (!userId) return; + if (selectedSenders?.includes(userId)) { + const next = selectedSenders.filter((s) => s !== userId); + onChange(next.length > 0 ? next : undefined); + } else { + onChange([...(selectedSenders ?? []), userId]); + } + }; + + return ( + setMenuAnchor(undefined), + clickOutsideDeactivates: true, + escapeDeactivates: stopPropagation, + }} + > + + + + From + 0 ? ( + + {searchState.items.length} + + ) : null + } + /> + + + + {displayMembers.length === 0 && ( + + No members found + + )} +
+ {vItems.map((vItem) => { + const member = displayMembers[vItem.index]!; + const selected = selectedSenders?.includes(member.userId); + return ( + + + } + /> + + } + > + + + {getMemberDisplayName(member)} + + + {member.userId} + + + + + ); + })} +
+ + + + + + } + > + } + > + Add Sender + + + ); +} + type SearchFiltersProps = { defaultRoomsFilterName: string; allowGlobal?: boolean; roomList: string[]; + defaultRooms: string[]; selectedRooms?: string[]; onSelectedRoomsChange: (selectedRooms?: string[]) => void; global?: boolean; onGlobalChange: (global?: boolean) => void; order?: string; onOrderChange: (order?: string) => void; + hasTypes?: SearchHasType[]; + onHasTypesChange: (hasTypes?: SearchHasType[]) => void; + senders?: string[]; + onSendersChange: (senders?: string[]) => void; }; export function SearchFilters({ defaultRoomsFilterName, allowGlobal, roomList, + defaultRooms, selectedRooms, onSelectedRoomsChange, global, order, onGlobalChange, onOrderChange, + hasTypes, + onHasTypesChange, + senders, + onSendersChange, }: SearchFiltersProps) { + const senderScope = selectedRooms && selectedRooms.length > 0 ? selectedRooms : defaultRooms; const mx = useMatrixClient(); return ( @@ -398,6 +670,41 @@ export function SearchFilters({ + + + Has: + + + + + From: + + {senders?.map((sender) => ( + { + const next = senders.filter((s) => s !== sender); + onSendersChange(next.length > 0 ? next : undefined); + }} + radii="Pill" + before={} + after={} + > + {mx.getUser(sender)?.displayName ?? sender} + + ))} + + ); } diff --git a/src/app/features/message-search/searchEncryptedRooms.test.ts b/src/app/features/message-search/searchEncryptedRooms.test.ts new file mode 100644 index 000000000..aac0f2f90 --- /dev/null +++ b/src/app/features/message-search/searchEncryptedRooms.test.ts @@ -0,0 +1,209 @@ +import { describe, it, expect } from 'vitest'; +import { EventType } from '$types/matrix-sdk'; +import type { IEventWithRoomId, MatrixClient, MatrixEvent } from '$types/matrix-sdk'; +import { + searchRoomTimeline, + searchEncryptedRoomsInMemory, + partitionRoomsByEncryption, + mergeSearchGroups, +} from './searchEncryptedRooms'; +import type { ResultGroup } from './useMessageSearch'; + +// Minimal MatrixEvent stub — only the methods used by searchRoomTimeline +function makeEvent(overrides: { + type?: string; + body?: string; + sender?: string; + ts?: number; + id?: string; + redacted?: boolean; +}): MatrixEvent { + return { + getType: () => overrides.type ?? EventType.RoomMessage, + getContent: () => ({ body: overrides.body ?? '', msgtype: 'm.text' }), + getSender: () => overrides.sender ?? '@alice:example.org', + getTs: () => overrides.ts ?? 1000, + getId: () => overrides.id ?? '$event1', + isRedacted: () => overrides.redacted ?? false, + getUnsigned: () => ({}), + event: {}, + } as unknown as MatrixEvent; +} + +function makeRoom(roomId: string, events: MatrixEvent[]) { + return { + roomId, + getLiveTimeline: () => ({ getEvents: () => events }), + }; +} + +describe('searchRoomTimeline', () => { + it('returns undefined when no events match', () => { + const room = makeRoom('!room:example.org', [makeEvent({ body: 'hello world' })]); + expect(searchRoomTimeline(room, 'goodbye')).toBeUndefined(); + }); + + it('matches a simple substring (case-insensitive)', () => { + const room = makeRoom('!room:example.org', [makeEvent({ body: 'Hello World', id: '$e1' })]); + const group = searchRoomTimeline(room, 'hello world'); + expect(group).toBeDefined(); + expect(group!.items).toHaveLength(1); + expect(group!.items[0]!.event.event_id).toBe('$e1'); + }); + + it('is case-insensitive', () => { + const room = makeRoom('!room:example.org', [makeEvent({ body: 'MATRIX ENCRYPTED' })]); + expect(searchRoomTimeline(room, 'matrix encrypted')).toBeDefined(); + }); + + it('skips non-message events', () => { + const room = makeRoom('!room:example.org', [ + makeEvent({ type: 'm.room.encrypted', body: 'search me' }), + makeEvent({ type: 'm.room.member', body: 'search me' }), + ]); + expect(searchRoomTimeline(room, 'search me')).toBeUndefined(); + }); + + it('skips redacted events', () => { + const room = makeRoom('!room:example.org', [makeEvent({ body: 'match', redacted: true })]); + expect(searchRoomTimeline(room, 'match')).toBeUndefined(); + }); + + it('filters by sender when senders list is provided', () => { + const room = makeRoom('!room:example.org', [ + makeEvent({ body: 'match', sender: '@alice:example.org', id: '$e1' }), + makeEvent({ body: 'match', sender: '@bob:example.org', id: '$e2' }), + ]); + const group = searchRoomTimeline(room, 'match', ['@alice:example.org']); + expect(group!.items).toHaveLength(1); + expect(group!.items[0]!.event.sender).toBe('@alice:example.org'); + }); + + it('sorts results most-recent-first', () => { + const room = makeRoom('!room:example.org', [ + makeEvent({ body: 'match', ts: 1000, id: '$old' }), + makeEvent({ body: 'match', ts: 3000, id: '$new' }), + makeEvent({ body: 'match', ts: 2000, id: '$mid' }), + ]); + const group = searchRoomTimeline(room, 'match'); + expect(group!.items.map((i) => i.event.event_id)).toEqual(['$new', '$mid', '$old']); + }); + + it('uses decrypted event type and content (getType/getContent)', () => { + // Simulates an e2ee event: underlying type is m.room.encrypted but + // getType()/getContent() return the decrypted values. + const room = makeRoom('!room:example.org', [ + makeEvent({ type: EventType.RoomMessage, body: 'secret message', id: '$enc' }), + ]); + const group = searchRoomTimeline(room, 'secret'); + expect(group!.items[0]!.event.type).toBe(EventType.RoomMessage); + expect(group!.items[0]!.event.content.body).toBe('secret message'); + }); +}); + +describe('searchEncryptedRoomsInMemory', () => { + it('searches across multiple rooms and returns matching groups', () => { + const mx = { + getRoom: (id: string) => { + const rooms = [ + makeRoom('!room1:example.org', [makeEvent({ body: 'hello', id: '$e1' })]), + makeRoom('!room2:example.org', [makeEvent({ body: 'goodbye', id: '$e2' })]), + ]; + return rooms.find((r) => r.roomId === id) ?? null; + }, + }; + const groups = searchEncryptedRoomsInMemory(mx as unknown as MatrixClient, 'hello', [ + '!room1:example.org', + '!room2:example.org', + ]); + expect(groups).toHaveLength(1); + expect(groups[0]!.roomId).toBe('!room1:example.org'); + }); + + it('returns empty array when no rooms match', () => { + const mx = { + getRoom: () => makeRoom('!room:example.org', [makeEvent({ body: 'unrelated content' })]), + }; + const groups = searchEncryptedRoomsInMemory(mx as unknown as MatrixClient, 'notfound', [ + '!room:example.org', + ]); + expect(groups).toHaveLength(0); + }); + + it('skips rooms not found in the client', () => { + const mx = { getRoom: () => null }; + const groups = searchEncryptedRoomsInMemory(mx as unknown as MatrixClient, 'match', [ + '!ghost:example.org', + ]); + expect(groups).toHaveLength(0); + }); +}); + +describe('partitionRoomsByEncryption', () => { + const mx = { + getRooms: () => [{ roomId: '!enc:example.org' }, { roomId: '!plain:example.org' }], + isRoomEncrypted: (id: string) => id === '!enc:example.org', + }; + + it('returns all encrypted rooms and undefined serverRooms for global search', () => { + const result = partitionRoomsByEncryption(mx as unknown as MatrixClient, undefined); + expect(result.encryptedRoomIds).toEqual(['!enc:example.org']); + expect(result.serverRooms).toBeUndefined(); + expect(result.skipServerSearch).toBe(false); + }); + + it('splits a mixed room list correctly', () => { + const result = partitionRoomsByEncryption(mx as unknown as MatrixClient, [ + '!enc:example.org', + '!plain:example.org', + ]); + expect(result.encryptedRoomIds).toEqual(['!enc:example.org']); + expect(result.serverRooms).toEqual(['!plain:example.org']); + expect(result.skipServerSearch).toBe(false); + }); + + it('sets skipServerSearch when all specified rooms are encrypted', () => { + const result = partitionRoomsByEncryption(mx as unknown as MatrixClient, ['!enc:example.org']); + expect(result.skipServerSearch).toBe(true); + expect(result.serverRooms).toBeUndefined(); + }); +}); + +const makeGroup = (roomId: string, ts: number): ResultGroup => ({ + roomId, + items: [ + { + rank: 1, + event: { room_id: roomId, origin_server_ts: ts } as unknown as IEventWithRoomId, + context: { events_before: [], events_after: [], profile_info: {} }, + }, + ], +}); + +describe('mergeSearchGroups', () => { + it('returns server groups unchanged when there are no in-memory groups', () => { + const server = [makeGroup('!a:x', 2000)]; + expect(mergeSearchGroups(server, [])).toBe(server); + }); + + it('returns in-memory groups unchanged when there are no server groups', () => { + const mem = [makeGroup('!b:x', 1000)]; + expect(mergeSearchGroups([], mem)).toBe(mem); + }); + + it('sorts by timestamp for recent order', () => { + const server = [makeGroup('!a:x', 1000)]; + const mem = [makeGroup('!b:x', 3000)]; + const merged = mergeSearchGroups(server, mem, 'recent'); + expect(merged[0]!.roomId).toBe('!b:x'); + expect(merged[1]!.roomId).toBe('!a:x'); + }); + + it('puts server results first for rank order', () => { + const server = [makeGroup('!a:x', 1000)]; + const mem = [makeGroup('!b:x', 3000)]; + const merged = mergeSearchGroups(server, mem, 'rank'); + expect(merged[0]!.roomId).toBe('!a:x'); + expect(merged[1]!.roomId).toBe('!b:x'); + }); +}); diff --git a/src/app/features/message-search/searchEncryptedRooms.ts b/src/app/features/message-search/searchEncryptedRooms.ts new file mode 100644 index 000000000..c73b3e4af --- /dev/null +++ b/src/app/features/message-search/searchEncryptedRooms.ts @@ -0,0 +1,188 @@ +import { EventType } from '$types/matrix-sdk'; +import type { + IEventWithRoomId, + IResultContext, + MatrixClient, + MatrixEvent, +} from '$types/matrix-sdk'; +import type { ResultGroup, ResultItem } from './useMessageSearch'; + +/** Media / content type filters — mirrors Discord's `has:` filter. */ +export type SearchHasType = 'image' | 'file' | 'audio' | 'video' | 'link'; + +const HAS_TYPE_TO_MSGTYPE: Partial> = { + image: 'm.image', + file: 'm.file', + audio: 'm.audio', + video: 'm.video', +}; + +function mEventMatchesHasTypes(mEvent: MatrixEvent, hasTypes: SearchHasType[]): boolean { + const content = mEvent.getContent() as { msgtype?: string; body?: string }; + for (const type of hasTypes) { + const msgtype = HAS_TYPE_TO_MSGTYPE[type]; + if (msgtype && content.msgtype === msgtype) return true; + if (type === 'link' && /https?:\/\//i.test(content.body ?? '')) return true; + } + return false; +} + +// Shared empty context — in-memory results have no surrounding-event context. +export const EMPTY_CONTEXT: IResultContext = { + events_before: [], + events_after: [], + profile_info: {}, +}; + +/** + * Builds an IEventWithRoomId from a live MatrixEvent, using the decrypted + * content and event type. This is what makes encrypted-room search work: + * getContent() returns plaintext even for e2ee events that have been decrypted. + */ +export function toSearchEvent(mEvent: MatrixEvent, roomId: string): IEventWithRoomId { + return { + event_id: mEvent.getId() ?? '', + room_id: roomId, + sender: mEvent.getSender() ?? '', + origin_server_ts: mEvent.getTs(), + content: mEvent.getContent(), // decrypted content for e2ee events + type: mEvent.getType(), // decrypted event type (e.g. m.room.message, not m.room.encrypted) + unsigned: mEvent.getUnsigned(), + } as IEventWithRoomId; +} + +/** + * Searches a single room's live timeline for message events that contain + * `lowerTerm` in their body. Returns a ResultGroup or undefined if no matches. + */ +export function searchRoomTimeline( + room: { roomId: string; getLiveTimeline: () => { getEvents: () => MatrixEvent[] } }, + lowerTerm: string, + senders?: string[], + hasTypes?: SearchHasType[] +): ResultGroup | undefined { + const events = room.getLiveTimeline().getEvents(); + const items: ResultItem[] = []; + + for (const mEvent of events) { + // Skip non-message events and still-encrypted events (decryption failed or not yet decrypted) + if ((mEvent.getType() as EventType) !== EventType.RoomMessage) continue; + if (mEvent.isRedacted()) continue; + + const sender = mEvent.getSender(); + if (!sender) continue; + if (senders && !senders.includes(sender)) continue; + + if (hasTypes && hasTypes.length > 0 && !mEventMatchesHasTypes(mEvent, hasTypes)) continue; + + if (lowerTerm !== '') { + const body: string = mEvent.getContent().body ?? ''; + if (!body || !body.toLowerCase().includes(lowerTerm)) continue; + } + + items.push({ + rank: 1, + event: toSearchEvent(mEvent, room.roomId), + context: EMPTY_CONTEXT, + }); + } + + if (items.length === 0) return undefined; + + // Most recent first, consistent with server "recent" ordering + items.sort((a, b) => b.event.origin_server_ts - a.event.origin_server_ts); + + return { roomId: room.roomId, items }; +} + +/** + * Searches the in-memory live timeline of each listed encrypted room. + * Returns one ResultGroup per room that has at least one match. + */ +export function searchEncryptedRoomsInMemory( + mx: Pick, + term: string, + encryptedRoomIds: string[], + senders?: string[], + hasTypes?: SearchHasType[] +): ResultGroup[] { + const lowerTerm = term.toLowerCase(); + const groups: ResultGroup[] = []; + + for (const roomId of encryptedRoomIds) { + const room = mx.getRoom(roomId); + if (!room) continue; + + const group = searchRoomTimeline(room, lowerTerm, senders, hasTypes); + if (group) groups.push(group); + } + + return groups; +} + +/** + * Splits the user's room filter into encrypted (in-memory) and plaintext (server) buckets. + * + * - When `rooms` is undefined (global search), the server handles plaintext rooms and + * we additionally scan all joined encrypted rooms in memory. + * - When `rooms` is defined, each room is routed to the appropriate search path. + */ +export function partitionRoomsByEncryption( + mx: Pick, + rooms?: string[] +): { encryptedRoomIds: string[]; serverRooms: string[] | undefined; skipServerSearch: boolean } { + if (rooms === undefined) { + // Global: server handles everything it can; we supplement with all encrypted rooms + const encryptedRoomIds = mx + .getRooms() + .filter((r) => mx.isRoomEncrypted(r.roomId)) + .map((r) => r.roomId); + return { encryptedRoomIds, serverRooms: undefined, skipServerSearch: false }; + } + + const encryptedRoomIds: string[] = []; + const serverRooms: string[] = []; + + for (const roomId of rooms) { + if (mx.isRoomEncrypted(roomId)) { + encryptedRoomIds.push(roomId); + } else { + serverRooms.push(roomId); + } + } + + return { + encryptedRoomIds, + serverRooms: serverRooms.length > 0 ? serverRooms : undefined, + // All specified rooms are encrypted — skip the server call entirely + skipServerSearch: rooms.length > 0 && serverRooms.length === 0, + }; +} + +/** + * Merges server-side and in-memory ResultGroups. + * For "recent" order: interleaved by each group's most recent event timestamp. + * For "rank" order: server results first (real relevance scores), then in-memory. + */ +export function mergeSearchGroups( + serverGroups: ResultGroup[], + inMemoryGroups: ResultGroup[], + order?: string +): ResultGroup[] { + if (inMemoryGroups.length === 0) return serverGroups; + if (serverGroups.length === 0) return inMemoryGroups; + + const all = [...serverGroups, ...inMemoryGroups]; + + if (order === 'rank') { + // Keep server results first — they have real rank scores + return all; + } + + // Recent order: sort groups by the most recent event in each + return all.toSorted((a, b) => { + const aTs = a.items[0]?.event.origin_server_ts ?? 0; + const bTs = b.items[0]?.event.origin_server_ts ?? 0; + return bTs - aTs; + }); +} diff --git a/src/app/features/message-search/useMessageSearch.ts b/src/app/features/message-search/useMessageSearch.ts index dfe205f69..357a2d65c 100644 --- a/src/app/features/message-search/useMessageSearch.ts +++ b/src/app/features/message-search/useMessageSearch.ts @@ -8,6 +8,96 @@ import type { } from '$types/matrix-sdk'; import { useCallback } from 'react'; import { useMatrixClient } from '$hooks/useMatrixClient'; +import { useClientConfig } from '$hooks/useClientConfig'; +import { useAtomValue } from 'jotai'; +import { settingsAtom } from '$state/settings'; +import { + searchEncryptedRoomsInMemory, + partitionRoomsByEncryption, + mergeSearchGroups, + EMPTY_CONTEXT, + toSearchEvent, +} from './searchEncryptedRooms'; +import type { SearchHasType } from './searchEncryptedRooms'; +import type { MatrixClient } from '$types/matrix-sdk'; +import type { IndexableEvent } from '$plugins/search-worker/types'; +import { useSearchIndex } from '$hooks/useSearchIndex'; + +export type { SearchHasType }; + +/** + * Convert IDB-indexed events back to the ResultGroup format used by the search UI. + * + * Prefers the live MatrixEvent from the room cache so that media messages + * (m.image, m.file, m.audio, m.video) render with their full content + * (url, file, info, …). Falls back to a plain-text synthetic event showing + * the stored filename/body when the event is no longer in memory. + * + * For "recent" order, items within each group and the groups themselves are + * sorted newest-first so that mergeSearchGroups' timestamp-based interleaving + * (and its single-source early-return fast paths) both produce correct order. + */ +function idbEventsToGroups( + mx: Pick, + events: IndexableEvent[], + order?: string +): ResultGroup[] { + const byRoom = new Map(); + for (const ev of events) { + const liveEvent = mx.getRoom(ev.roomId)?.findEventById(ev.eventId); + const eventData: IEventWithRoomId = liveEvent + ? toSearchEvent(liveEvent, ev.roomId) + : ({ + event_id: ev.eventId, + room_id: ev.roomId, + sender: ev.sender, + origin_server_ts: ev.ts, + // Reconstruct full content from IDB-stored fields so media events + // (m.image, m.file, m.audio, m.video) render correctly even when + // the event is no longer in the live timeline window. + // Fall back to m.text only for pre-v3 index entries that lack media fields. + content: { + msgtype: ev.url !== undefined || ev.file !== undefined ? ev.msgtype : 'm.text', + body: ev.body, + ...(ev.url !== undefined && { url: ev.url }), + ...(ev.file !== undefined && { file: ev.file }), + ...(ev.info !== undefined && { info: ev.info }), + ...(ev.filename !== undefined && { filename: ev.filename }), + }, + type: 'm.room.message', + unsigned: {}, + } as IEventWithRoomId); + const item: ResultItem = { + rank: 1, + event: eventData, + context: EMPTY_CONTEXT, + }; + const arr = byRoom.get(ev.roomId) ?? []; + arr.push(item); + byRoom.set(ev.roomId, arr); + } + + const groups = Array.from(byRoom.entries()).map(([roomId, items]) => ({ + roomId, + // Sort items newest-first so items[0] is always the most recent — required + // for mergeSearchGroups' timestamp comparisons to be correct. + items: + order !== 'rank' + ? items.toSorted( + (a, b) => (b.event.origin_server_ts ?? 0) - (a.event.origin_server_ts ?? 0) + ) + : items, + })); + + // Sort groups newest-first so single-source fast-paths in mergeSearchGroups + // (which return the array unchanged) still produce correct recent order. + return order !== 'rank' + ? groups.toSorted( + (a, b) => + (b.items[0]?.event.origin_server_ts ?? 0) - (a.items[0]?.event.origin_server_ts ?? 0) + ) + : groups; +} export type ResultItem = { rank: number; @@ -24,6 +114,8 @@ export type SearchResult = { nextToken?: string; highlights: string[]; groups: ResultGroup[]; + /** Number of encrypted rooms whose in-memory timeline was searched. */ + inMemoryRoomCount?: number; }; const groupSearchResult = (results: ISearchResult[]): ResultGroup[] => { @@ -68,20 +160,139 @@ export type MessageSearchParams = { order?: string; rooms?: string[]; senders?: string[]; + hasTypes?: SearchHasType[]; }; export const useMessageSearch = (params: MessageSearchParams) => { const mx = useMatrixClient(); - const { term, order, rooms, senders } = params; + const { features } = useClientConfig(); + const settings = useAtomValue(settingsAtom); + const searchIndex = useSearchIndex(); + const { term, order, rooms, senders, hasTypes } = params; + + const filterGroupsByHasType = useCallback( + (grps: ResultGroup[]): ResultGroup[] => { + if (!hasTypes || hasTypes.length === 0) return grps; + const withMsgtype = hasTypes.filter((t) => t !== 'link'); + return grps + .map((g) => ({ + ...g, + items: g.items.filter((item) => { + const content = item.event.content as { msgtype?: string; body?: string }; + if (withMsgtype.length > 0) { + const msgtypeMap: Record = { + image: 'm.image', + file: 'm.file', + audio: 'm.audio', + video: 'm.video', + }; + if (withMsgtype.some((t) => content.msgtype === msgtypeMap[t])) return true; + } + if (hasTypes.includes('link') && /https?:\/\//i.test(content.body ?? '')) return true; + return false; + }), + })) + .filter((g) => g.items.length > 0); + }, + [hasTypes] + ); const searchMessages = useCallback( async (nextBatch?: string) => { - if (!term) + const hasHasTypes = hasTypes && hasTypes.length > 0; + if (!term && !hasHasTypes) return { highlights: [], groups: [], }; const limit = 20; + // Operator kill switch takes priority; user toggle controls the rest. + const encryptedSearchEnabled = + features?.encryptedSearch !== false && settings.encryptedSearch; + // Use IDB index when the user has enabled it and the index is ready. + const useIdbSearch = settings.idbSearchIndex && searchIndex?.isReady === true; + const isFirstPage = !nextBatch || nextBatch === ''; + + const { encryptedRoomIds, serverRooms, skipServerSearch } = encryptedSearchEnabled + ? partitionRoomsByEncryption(mx, rooms) + : { encryptedRoomIds: [], serverRooms: rooms, skipServerSearch: false }; + + // For IDB search: only run on first page (IDB has no pagination cursor here). + // Prefer IDB when available; fall back to in-memory live timeline. + let inMemoryGroups: ResultGroup[] = []; + let usedIdb = false; + if (encryptedSearchEnabled && isFirstPage && encryptedRoomIds.length > 0) { + if (useIdbSearch && (term || hasHasTypes)) { + const idbEvents = await searchIndex.query(term ?? '', { + roomIds: encryptedRoomIds, + senders, + hasTypes: hasHasTypes ? hasTypes : undefined, + }); + inMemoryGroups = idbEventsToGroups(mx, idbEvents, order); + usedIdb = true; + } else { + inMemoryGroups = searchEncryptedRoomsInMemory( + mx, + term ?? '', + encryptedRoomIds, + senders, + hasTypes + ); + } + } + + // When there's no text term, skip server search (server requires search_term). + // For has: filters, scan all rooms' timelines (encrypted + unencrypted). + if (skipServerSearch || !term) { + let unencryptedMemoryGroups: ResultGroup[] = []; + let unencryptedRoomCount = 0; + let usedIdbForUnencrypted = false; + if (hasHasTypes && isFirstPage) { + // When scoped (rooms defined), use only unencrypted rooms within scope (may be empty + // when all scoped rooms are encrypted). When global (rooms undefined), fall back to + // all non-encrypted joined rooms. + const unencryptedRooms = + rooms !== undefined + ? (serverRooms ?? []) + : mx + .getRooms() + .filter((r) => !mx.isRoomEncrypted(r.roomId)) + .map((r) => r.roomId); + unencryptedRoomCount = unencryptedRooms.length; + if (unencryptedRooms.length > 0) { + if (useIdbSearch) { + const idbEvents = await searchIndex.query('', { + roomIds: unencryptedRooms, + senders, + hasTypes, + }); + unencryptedMemoryGroups = idbEventsToGroups(mx, idbEvents, order); + usedIdbForUnencrypted = true; + } else { + unencryptedMemoryGroups = searchEncryptedRoomsInMemory( + mx, + '', + unencryptedRooms, + senders, + hasTypes + ); + } + } + } + return { + highlights: [], + groups: mergeSearchGroups( + filterGroupsByHasType(inMemoryGroups), + unencryptedMemoryGroups, + order + ), + // Only report local-cache count for rooms that were actually searched in-memory. + inMemoryRoomCount: + (usedIdb ? 0 : encryptedRoomIds.length) + + (usedIdbForUnencrypted ? 0 : unencryptedRoomCount) || undefined, + }; + } + const requestBody: ISearchRequestBody = { search_categories: { room_events: { @@ -92,8 +303,9 @@ export const useMessageSearch = (params: MessageSearchParams) => { }, filter: { limit, - rooms, + rooms: serverRooms, senders, + ...(hasTypes?.includes('link') && { contains_url: true }), }, include_state: false, order_by: order as SearchOrderBy.Recent, @@ -106,9 +318,41 @@ export const useMessageSearch = (params: MessageSearchParams) => { body: requestBody, next_batch: nextBatch === '' ? undefined : nextBatch, }); - return parseSearchResult(r); + const serverResult = parseSearchResult(r); + const filteredServerResult = { + ...serverResult, + groups: filterGroupsByHasType(serverResult.groups), + }; + + if (inMemoryGroups.length === 0) { + return filteredServerResult; + } + + const termWords = term.split(/\s+/).filter(Boolean); + return { + ...filteredServerResult, + groups: mergeSearchGroups( + filteredServerResult.groups, + filterGroupsByHasType(inMemoryGroups), + order + ), + highlights: Array.from(new Set([...filteredServerResult.highlights, ...termWords])), + inMemoryRoomCount: usedIdb ? undefined : encryptedRoomIds.length, + }; }, - [mx, term, order, rooms, senders] + [ + mx, + features, + settings.encryptedSearch, + settings.idbSearchIndex, + searchIndex, + term, + order, + rooms, + senders, + hasTypes, + filterGroupsByHasType, + ] ); return searchMessages; diff --git a/src/app/features/room/RoomViewHeader.tsx b/src/app/features/room/RoomViewHeader.tsx index f45e6172f..4aee4f277 100644 --- a/src/app/features/room/RoomViewHeader.tsx +++ b/src/app/features/room/RoomViewHeader.tsx @@ -45,8 +45,14 @@ import { useMatrixClient } from '$hooks/useMatrixClient'; import { useIsDirectRoom, useRoom } from '$hooks/useRoom'; import { useSetting } from '$state/hooks/settings'; import { settingsAtom } from '$state/settings'; +import { useClientConfig } from '$hooks/useClientConfig'; import { useSpaceOptionally } from '$hooks/useSpace'; -import { getHomeSearchPath, getSpaceSearchPath, withSearchParam } from '$pages/pathUtils'; +import { + getDirectSearchPath, + getHomeSearchPath, + getSpaceSearchPath, + withSearchParam, +} from '$pages/pathUtils'; import { createLogger } from '$utils/debug'; import { getCanonicalAliasOrRoomId, @@ -371,6 +377,10 @@ export function RoomViewHeader({ callView }: Readonly<{ callView?: boolean }>) { const encryptionEvent = useStateEvent(room, EventType.RoomEncryption); const encryptedRoom = !!encryptionEvent; + + const { features } = useClientConfig(); + const settings = useAtomValue(settingsAtom); + const encryptedSearchEnabled = features?.encryptedSearch !== false && settings.encryptedSearch; const avatarMxc = useRoomAvatar(room, direct && !customDMCards); const name = useRoomName(room); const topic = useRoomTopic(room); @@ -556,7 +566,9 @@ export function RoomViewHeader({ callView }: Readonly<{ callView?: boolean }>) { }; const path = space ? getSpaceSearchPath(getCanonicalAliasOrRoomId(mx, space.roomId)) - : getHomeSearchPath(); + : direct + ? getDirectSearchPath() + : getHomeSearchPath(); navigate(withSearchParam(path, searchParams)); }; @@ -670,13 +682,13 @@ export function RoomViewHeader({ callView }: Readonly<{ callView?: boolean }>) { {(!room.isCallRoom() || chat) && ( <> - {!encryptedRoom && ( + {(!encryptedRoom || encryptedSearchEnabled) && ( - Search + {encryptedRoom ? 'Search (local cache)' : 'Search'} } > diff --git a/src/app/features/search/Search.tsx b/src/app/features/search/Search.tsx index 0c928aa38..3e514d836 100644 --- a/src/app/features/search/Search.tsx +++ b/src/app/features/search/Search.tsx @@ -49,8 +49,10 @@ import { useKeyDown } from '$hooks/useKeyDown'; import { useMediaAuthentication } from '$hooks/useMediaAuthentication'; import { KeySymbol } from '$utils/key-symbol'; import { isMacOS } from '$utils/user-agent'; +import { useNavigate, useLocation } from 'react-router-dom'; import { useSelectedSpace } from '$hooks/router/useSelectedSpace'; import { getMxIdServer } from '$utils/mxIdHelper'; +import { getHomeSearchPath, getDirectSearchPath, getSpaceSearchPath } from '$pages/pathUtils'; enum SearchRoomType { Rooms = '#', @@ -203,6 +205,32 @@ export function RoomSearchModal({ requestClose, pickRoom }: RoomSearchModalProps const listFocus = useListFocusIndex(roomsToRender.length, 0); + const navigate = useNavigate(); + const { pathname } = useLocation(); + const [messageSearchQuery, setMessageSearchQuery] = useState(null); + + const handleNavigateMessageSearch = useCallback( + (query: string) => { + const sp = new URLSearchParams(); + if (query) sp.set('term', query); + const qs = sp.toString() ? `?${sp.toString()}` : ''; + + let basePath: string; + if (pathname.startsWith('/direct/')) { + basePath = getDirectSearchPath(); + } else if (!pathname.startsWith('/home/')) { + const spaceIdOrAlias = decodeURIComponent(pathname.split('/').find(Boolean) ?? ''); + basePath = spaceIdOrAlias ? getSpaceSearchPath(spaceIdOrAlias) : getHomeSearchPath(); + } else { + basePath = getHomeSearchPath(); + } + + navigate(`${basePath}${qs}`); + requestClose(); + }, + [pathname, navigate, requestClose] + ); + const queryHighlighRegex = result?.query ? makeHighlightRegex(result.query.split(' ')) : undefined; @@ -223,6 +251,15 @@ export function RoomSearchModal({ requestClose, pickRoom }: RoomSearchModalProps const target = evt.currentTarget; let value = target.value.trim(); + + if (!pickRoom && value.startsWith('>')) { + setMessageSearchQuery(value.slice(1).trimStart()); + setSearchRoomType(undefined); + resetSearch(); + return; + } + + setMessageSearchQuery(null); const prefix = value.match(/^[#@*]/)?.[0]; const searchType = typeof prefix === 'string' && getSearchPrefixToRoomType(prefix); if (searchType) { @@ -240,6 +277,10 @@ export function RoomSearchModal({ requestClose, pickRoom }: RoomSearchModalProps }; const handleInputKeyDown: KeyboardEventHandler = (evt) => { + if (isKeyHotkey('enter', evt) && messageSearchQuery !== null) { + handleNavigateMessageSearch(messageSearchQuery); + return; + } const roomId = roomsToRender[listFocus.index]; if (isKeyHotkey('enter', evt) && roomId) { handleActivateRoom(roomId, spaces.includes(roomId)); @@ -340,40 +381,7 @@ export function RoomSearchModal({ requestClose, pickRoom }: RoomSearchModalProps /> - {roomsToRender.length === 0 && ( - - - {pickRoom - ? result - ? 'No Match Found' - : pickRoom.eligibleRoomIds.length === 0 - ? 'No rooms to forward to' - : 'No rooms match this filter' - : result - ? 'No Match Found' - : 'No Rooms'} - - - {pickRoom - ? result - ? `No match found for "${result.query}".` - : pickRoom.eligibleRoomIds.length === 0 - ? 'You cannot send messages in any joined room yet.' - : 'Try another search, or use # for group rooms and @ for direct messages.' - : result - ? `No match found for "${result.query}".` - : 'You do not have any Rooms to display yet.'} - - - )} - {roomsToRender.length > 0 && ( + {messageSearchQuery !== null && !pickRoom ? (
- {roomsToRender.map((roomId, index) => { - const room = getRoom(roomId); - if (!room) return null; - - const dm = mDirects.has(roomId); - const dmUserId = dm && getDmUserId(roomId, getRoom, mx.getSafeUserId()); - const dmUsername = dmUserId && getMxIdLocalPart(dmUserId); - const dmUserServer = dmUserId && getMxIdServer(dmUserId); - - const allParents = getAllParents(roomToParents, roomId); - const orphanParents = - allParents && orphanSpaces.filter((o) => allParents.has(o)); - const perfectOrphanParent = - orphanParents && guessPerfectParent(mx, roomId, orphanParents); - - const exactParents = roomToParents.get(roomId); - const perfectParent = - exactParents && guessPerfectParent(mx, roomId, Array.from(exactParents)); - - const unread = roomToUnread.get(roomId); - - return ( - - {dmUserServer && ( - - {dmUserServer} - - )} - {!dm && perfectOrphanParent && ( - - {getRoom(perfectOrphanParent)?.name ?? perfectOrphanParent} - - )} - {unread && ( - - 0} - count={unread.highlight > 0 ? unread.highlight : unread.total} - /> - - )} - - } - before={ - - {dm || room.isSpaceRoom() ? ( - ( - - {nameInitials(room.name)} - - )} - /> - ) : ( - - )} - - } - > - - - {queryHighlighRegex - ? highlightText(queryHighlighRegex, [room.name]) - : room.name} - - {dmUsername && ( - - @ - {queryHighlighRegex - ? highlightText(queryHighlighRegex, [dmUsername]) - : dmUsername} - - )} - {!dm && perfectParent && perfectParent !== perfectOrphanParent && ( - - — {getRoom(perfectParent)?.name ?? perfectParent} - - )} - - - ); - })} + handleNavigateMessageSearch(messageSearchQuery)} + before={ + + + + } + > + + + {messageSearchQuery + ? `Search messages: "${messageSearchQuery}"` + : 'Search messages'} + + +
+ ) : ( + <> + {roomsToRender.length === 0 && ( + + + {pickRoom + ? result + ? 'No Match Found' + : pickRoom.eligibleRoomIds.length === 0 + ? 'No rooms to forward to' + : 'No rooms match this filter' + : result + ? 'No Match Found' + : 'No Rooms'} + + + {pickRoom + ? result + ? `No match found for "${result.query}".` + : pickRoom.eligibleRoomIds.length === 0 + ? 'You cannot send messages in any joined room yet.' + : 'Try another search, or use # for group rooms and @ for direct messages.' + : result + ? `No match found for "${result.query}".` + : 'You do not have any Rooms to display yet.'} + + + )} + {roomsToRender.length > 0 && ( + +
+ {roomsToRender.map((roomId, index) => { + const room = getRoom(roomId); + if (!room) return null; + + const dm = mDirects.has(roomId); + const dmUserId = dm && getDmUserId(roomId, getRoom, mx.getSafeUserId()); + const dmUsername = dmUserId && getMxIdLocalPart(dmUserId); + const dmUserServer = dmUserId && getMxIdServer(dmUserId); + + const allParents = getAllParents(roomToParents, roomId); + const orphanParents = + allParents && orphanSpaces.filter((o) => allParents.has(o)); + const perfectOrphanParent = + orphanParents && guessPerfectParent(mx, roomId, orphanParents); + + const exactParents = roomToParents.get(roomId); + const perfectParent = + exactParents && + guessPerfectParent(mx, roomId, Array.from(exactParents)); + + const unread = roomToUnread.get(roomId); + + return ( + + {dmUserServer && ( + + {dmUserServer} + + )} + {!dm && perfectOrphanParent && ( + + + {getRoom(perfectOrphanParent)?.name ?? perfectOrphanParent} + + + )} + {unread && ( + + 0} + count={ + unread.highlight > 0 ? unread.highlight : unread.total + } + /> + + )} + + } + before={ + + {dm || room.isSpaceRoom() ? ( + ( + + {nameInitials(room.name)} + + )} + /> + ) : ( + + )} + + } + > + + + {queryHighlighRegex + ? highlightText(queryHighlighRegex, [room.name]) + : room.name} + + {dmUsername && ( + + @ + {queryHighlighRegex + ? highlightText(queryHighlighRegex, [dmUsername]) + : dmUsername} + + )} + {!dm && perfectParent && perfectParent !== perfectOrphanParent && ( + + — {getRoom(perfectParent)?.name ?? perfectParent} + + )} + + + ); + })} +
+ + )} + )} @@ -500,8 +579,8 @@ export function RoomSearchModal({ requestClose, pickRoom }: RoomSearchModalProps ) : ( <> - Type # for rooms, @ for DMs and * for spaces. Hotkey:{' '} - {isMacOS() ? KeySymbol.Command : 'Ctrl'} + k + Type # for rooms, @ for DMs, * for spaces and {'>'}{' '} + for messages. Hotkey: {isMacOS() ? KeySymbol.Command : 'Ctrl'} + k {' / '} {isMacOS() ? KeySymbol.Command : 'Ctrl'} + f diff --git a/src/app/features/settings/developer-tools/DevelopTools.tsx b/src/app/features/settings/developer-tools/DevelopTools.tsx index 100119726..969d9fe1c 100644 --- a/src/app/features/settings/developer-tools/DevelopTools.tsx +++ b/src/app/features/settings/developer-tools/DevelopTools.tsx @@ -15,6 +15,7 @@ import { AccountData } from './AccountData'; import { SyncDiagnostics } from './SyncDiagnostics'; import { DebugLogViewer } from './DebugLogViewer'; import { SentrySettings } from './SentrySettings'; +import { SearchIndexCache } from './SearchIndexCache'; type DeveloperToolsProps = { requestBack?: () => void; @@ -127,6 +128,7 @@ export function DeveloperTools({ requestBack, requestClose }: DeveloperToolsProp )} + {developerTools && } diff --git a/src/app/features/settings/developer-tools/SearchIndexCache.tsx b/src/app/features/settings/developer-tools/SearchIndexCache.tsx new file mode 100644 index 000000000..c464a16c5 --- /dev/null +++ b/src/app/features/settings/developer-tools/SearchIndexCache.tsx @@ -0,0 +1,122 @@ +import { useCallback, useEffect, useState } from 'react'; +import { Box, Button, Text } from 'folds'; +import { SequenceCard } from '$components/sequence-card'; +import { SettingTile } from '$components/setting-tile'; +import { useSetting } from '$state/hooks/settings'; +import { settingsAtom } from '$state/settings'; +import { useSearchIndex } from '$hooks/useSearchIndex'; +import type { SearchIndexStats } from '$hooks/useSearchIndex'; +import { SequenceCardStyle } from '$features/settings/styles.css'; + +const LIMIT_OPTIONS: Array<{ label: string; value: number }> = [ + { label: '500 messages', value: 500 }, + { label: '1,000 messages', value: 1000 }, + { label: '2,000 messages (default)', value: 2000 }, + { label: '5,000 messages', value: 5000 }, + { label: 'Unlimited', value: Number.MAX_SAFE_INTEGER }, +]; + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +export function SearchIndexCache() { + const [idbSearchIndex] = useSetting(settingsAtom, 'idbSearchIndex'); + const [searchIndexMessageLimit, setSearchIndexMessageLimit] = useSetting( + settingsAtom, + 'searchIndexMessageLimit' + ); + const searchIndex = useSearchIndex(); + + const [stats, setStats] = useState(null); + const [clearing, setClearing] = useState(false); + + const refreshStats = useCallback(async () => { + if (!searchIndex?.isReady) return; + const s = await searchIndex.getStats(); + setStats(s); + }, [searchIndex]); + + useEffect(() => { + void refreshStats(); + const id = window.setInterval(() => void refreshStats(), 5000); + return () => window.clearInterval(id); + }, [refreshStats]); + + const handleClear = useCallback(async () => { + if (!searchIndex) return; + setClearing(true); + await searchIndex.clearIndex(); + setStats(null); + setClearing(false); + }, [searchIndex]); + + if (!idbSearchIndex) return null; + + return ( + + Message Search Index + + + {searchIndex?.isBackfilling && ( + + )} + setSearchIndexMessageLimit(Number(e.target.value))} + style={{ padding: '4px 8px', borderRadius: '4px' }} + > + {LIMIT_OPTIONS.map((opt) => ( + + ))} + + } + /> + void handleClear()} + variant="Secondary" + fill="Soft" + size="300" + radii="300" + outlined + disabled={clearing || !searchIndex?.isReady} + > + {clearing ? 'Clearing…' : 'Clear'} + + } + /> + + + ); +} diff --git a/src/app/features/settings/experimental/EncryptedSearch.tsx b/src/app/features/settings/experimental/EncryptedSearch.tsx new file mode 100644 index 000000000..1c3ad3ddb --- /dev/null +++ b/src/app/features/settings/experimental/EncryptedSearch.tsx @@ -0,0 +1,38 @@ +import { SequenceCard } from '$components/sequence-card'; +import { SettingTile } from '$components/setting-tile'; +import { useSetting } from '$state/hooks/settings'; +import { settingsAtom } from '$state/settings'; +import { useClientConfig } from '$hooks/useClientConfig'; +import { Box, Switch, Text } from 'folds'; +import { SequenceCardStyle } from '../styles.css'; + +export function EncryptedSearch() { + const { features } = useClientConfig(); + const [encryptedSearch, setEncryptedSearch] = useSetting(settingsAtom, 'encryptedSearch'); + + // If the operator has explicitly disabled this in config.json, hide the toggle. + if (features?.encryptedSearch === false) return null; + + return ( + + Encrypted Room Search + + + } + /> + + + ); +} diff --git a/src/app/features/settings/experimental/Experimental.tsx b/src/app/features/settings/experimental/Experimental.tsx index 330412185..b993235cf 100644 --- a/src/app/features/settings/experimental/Experimental.tsx +++ b/src/app/features/settings/experimental/Experimental.tsx @@ -10,6 +10,8 @@ import { Sync } from '../general'; import { SettingsSectionPage } from '../SettingsSectionPage'; import { BandwidthSavingEmojis } from './BandwithSavingEmojis'; import { MSC4268HistoryShare } from './MSC4268HistoryShare'; +import { EncryptedSearch } from './EncryptedSearch'; +import { SearchIndexToggle } from './SearchIndexToggle'; function PersonaToggle() { const [showPersonaSetting, setShowPersonaSetting] = useSetting( @@ -59,6 +61,8 @@ export function Experimental({ requestBack, requestClose }: Readonly + + diff --git a/src/app/features/settings/experimental/SearchIndexToggle.tsx b/src/app/features/settings/experimental/SearchIndexToggle.tsx new file mode 100644 index 000000000..b11b3caeb --- /dev/null +++ b/src/app/features/settings/experimental/SearchIndexToggle.tsx @@ -0,0 +1,33 @@ +import { SequenceCard } from '$components/sequence-card'; +import { SettingTile } from '$components/setting-tile'; +import { useSetting } from '$state/hooks/settings'; +import { settingsAtom } from '$state/settings'; +import { Box, Switch, Text } from 'folds'; +import { SequenceCardStyle } from '../styles.css'; + +export function SearchIndexToggle() { + const [idbSearchIndex, setIdbSearchIndex] = useSetting(settingsAtom, 'idbSearchIndex'); + + return ( + + Message Search Index + + + } + /> + + + ); +} diff --git a/src/app/features/settings/settingsLink.ts b/src/app/features/settings/settingsLink.ts index 6cca050a5..f10c7d2ff 100644 --- a/src/app/features/settings/settingsLink.ts +++ b/src/app/features/settings/settingsLink.ts @@ -181,7 +181,13 @@ const settingsLinkFocusIdsBySection: Record { const directMatch = useMatch({ @@ -20,3 +20,13 @@ export const useDirectCreateSelected = (): boolean => { return !!match; }; + +export const useDirectSearchSelected = (): boolean => { + const match = useMatch({ + path: getDirectSearchPath(), + caseSensitive: true, + end: false, + }); + + return !!match; +}; diff --git a/src/app/hooks/useClientConfig.ts b/src/app/hooks/useClientConfig.ts index 6cb2a9ad3..94a6473bb 100644 --- a/src/app/hooks/useClientConfig.ts +++ b/src/app/hooks/useClientConfig.ts @@ -50,6 +50,12 @@ export type ClientConfig = { themeCatalogApprovedHostPrefixes?: string[]; settingsDefaults?: Partial; + + features?: { + polls?: boolean; + /** Enable in-memory search for encrypted rooms (default: true). */ + encryptedSearch?: boolean; + }; }; const ClientConfigContext = createContext(null); diff --git a/src/app/hooks/useSearchIndex.tsx b/src/app/hooks/useSearchIndex.tsx new file mode 100644 index 000000000..65ac74c8e --- /dev/null +++ b/src/app/hooks/useSearchIndex.tsx @@ -0,0 +1,610 @@ +/** + * useSearchIndex — manages the search worker lifecycle, live indexing, and headless backfill. + * + * Mount once via SearchIndexProvider in the client tree. Consume via useSearchIndex(). + */ + +import { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useRef, + useState, + type ReactNode, +} from 'react'; +import { + ClientEvent, + Direction, + EventTimelineSet, + EventType, + MatrixEventEvent, + RoomEvent, + SyncState, + type MatrixEvent, + type Room, +} from '$types/matrix-sdk'; +import { useMatrixClient } from '$hooks/useMatrixClient'; +import { useSetting } from '$state/hooks/settings'; +import { settingsAtom } from '$state/settings'; +import type { + BackfillState, + IndexableEvent, + WorkerInMessage, + WorkerOutMessage, +} from '$plugins/search-worker/types'; + +// ── Types ─────────────────────────────────────────────────────────────────── + +export type SearchIndexStats = { + indexedEventCount: number; + roomCount: number; + /** Estimated IDB size in bytes */ + estimatedBytes: number; + /** Number of encrypted rooms still being backfilled */ + backfillingRoomCount: number; +}; + +type SearchIndexCtx = { + /** Query the IDB-backed index. Resolves to an empty array when the index is unavailable. */ + query: ( + term: string, + opts?: { roomIds?: string[]; senders?: string[]; hasTypes?: string[] } + ) => Promise; + /** Request current stats from the worker. */ + getStats: () => Promise; + /** Wipe the index + IDB. */ + clearIndex: () => Promise; + /** True once the worker has hydrated from IDB and is ready to accept queries. */ + isReady: boolean; + /** True while background backfill is actively running. */ + isBackfilling: boolean; +}; + +// ── Context ────────────────────────────────────────────────────────────────── + +const SearchIndexContext = createContext(null); + +export function useSearchIndex(): SearchIndexCtx | null { + return useContext(SearchIndexContext); +} + +// ── Idle scheduler ─────────────────────────────────────────────────────────── + +/** + * On systems with requestIdleCallback (desktop, Android Chrome) the browser's + * own idle scheduler provides natural backpressure, so we allow unlimited + * concurrent backfills — this restores the fast pre-bf4d8d6 behaviour. + * + * On iOS Safari (no requestIdleCallback) we cap concurrency to prevent the + * HTTP connection pool from being saturated and starving the /sync long-poll. + */ +const HAS_IDLE_CALLBACK = typeof requestIdleCallback === 'function'; +const MAX_CONCURRENT_BACKFILLS = HAS_IDLE_CALLBACK ? Infinity : 4; + +function scheduleIdle(cb: () => void): () => void { + if (HAS_IDLE_CALLBACK) { + const id = requestIdleCallback(cb, { timeout: 5000 }); + return () => cancelIdleCallback(id); + } + // iOS Safari: no requestIdleCallback — use a short delay; the concurrency + // cap (MAX_CONCURRENT_BACKFILLS) prevents HTTP connection pool saturation. + const id = setTimeout(cb, 150); + return () => clearTimeout(id); +} + +// ── Event conversion ────────────────────────────────────────────────────────── + +const MEDIA_MSGTYPES = new Set(['m.image', 'm.file', 'm.audio', 'm.video']); + +function toIndexableEvent(mEvent: MatrixEvent, roomId: string): IndexableEvent | null { + const eventId = mEvent.getId(); + if (!eventId) return null; + // Skip still-encrypted and redacted events + if (mEvent.getType() === 'm.room.encrypted') return null; + if (mEvent.getType() !== (EventType.RoomMessage as string)) return null; + if (mEvent.isRedacted()) return null; + const content = mEvent.getContent<{ + body?: string; + msgtype?: string; + url?: string; + file?: Record; + info?: Record; + filename?: string; + }>(); + const body: string = content.body ?? ''; + if (!body.trim()) return null; + const sender = mEvent.getSender(); + if (!sender) return null; + const msgtype = content.msgtype ?? 'm.text'; + const base: IndexableEvent = { eventId, roomId, sender, msgtype, body, ts: mEvent.getTs() }; + if (MEDIA_MSGTYPES.has(msgtype)) { + if (content.url !== undefined) base.url = content.url; + if (content.file !== undefined) base.file = content.file; + if (content.info !== undefined) base.info = content.info; + if (content.filename !== undefined) base.filename = content.filename; + } + return base; +} + +// ── Provider ───────────────────────────────────────────────────────────────── + +const BACKFILL_PAGE_SIZE = 50; + +type PendingQuery = { + resolve: (events: IndexableEvent[]) => void; + reject: (err: unknown) => void; +}; + +type PendingStats = { + backfillingRoomCount: number; + resolve: (stats: SearchIndexStats) => void; +}; + +export function SearchIndexProvider({ children }: { children: ReactNode }) { + const mx = useMatrixClient(); + const [idbSearchIndex] = useSetting(settingsAtom, 'idbSearchIndex'); + const [searchIndexMessageLimit] = useSetting(settingsAtom, 'searchIndexMessageLimit'); + + const [isReady, setIsReady] = useState(false); + const [isBackfilling, setIsBackfilling] = useState(false); + + const workerRef = useRef(null); + const pendingQueriesRef = useRef>(new Map()); + const pendingStatsRef = useRef(null); + // Rooms whose backfill is actively scheduled (to avoid double-scheduling) + const backfillingRoomsRef = useRef>(new Set()); + // Store headless timeline sets per room for pagination continuity + const headlessSetsRef = useRef>(new Map()); + // cancellable idle callbacks for backfill + const cancelIdlesRef = useRef void>>([]); + // Queue of rooms waiting to start backfill (for the concurrency limiter) + const backfillQueueRef = useRef>([]); + // Current Matrix sync state — used to pause backfill when sync is struggling + const syncStateRef = useRef(null); + // Persisted backfill states received from the worker — used by the ClientEvent.Room + // listener to correctly handle rooms that are added after the initial startBackfill call + // (e.g. rooms loaded by sliding sync after the initial window of 100). + const backfillStatesRef = useRef>({}); + + const postToWorker = useCallback((msg: WorkerInMessage) => { + // oxlint-disable-next-line require-post-message-target-origin -- Worker.postMessage has no targetOrigin + workerRef.current?.postMessage(msg); + }, []); + + // ── Live indexing ────────────────────────────────────────────────────────── + + const indexEvent = useCallback( + (mEvent: MatrixEvent, room: Room) => { + const handleDecrypted = () => { + const ev = toIndexableEvent(mEvent, room.roomId); + if (ev) postToWorker({ type: 'INDEX_EVENTS', events: [ev] }); + }; + + if (mEvent.getType() === 'm.room.encrypted') { + // Still encrypted — wait for decryption + mEvent.once(MatrixEventEvent.Decrypted, handleDecrypted); + } else { + handleDecrypted(); + } + }, + [postToWorker] + ); + + // ── Headless backfill ────────────────────────────────────────────────────── + + const backfillRoom = useCallback( + async (room: Room, state: BackfillState): Promise => { + if (state.done) return; + + // Get or create headless timeline set for this room + let headlessSet = headlessSetsRef.current.get(room.roomId); + if (!headlessSet) { + headlessSet = new EventTimelineSet(room, {}); + headlessSetsRef.current.set(room.roomId, headlessSet); + } + + const headlessTimeline = headlessSet.getLiveTimeline(); + + // Seed the backward token: from IDB state, or from the room's live timeline + const seedToken = + state.token ?? room.getLiveTimeline().getPaginationToken(Direction.Backward); + if (!seedToken) { + // Room has no history to paginate — mark done + postToWorker({ + type: 'SET_BACKFILL_STATE', + roomId: room.roomId, + state: { ...state, done: true }, + }); + backfillingRoomsRef.current.delete(room.roomId); + return; + } + + headlessTimeline.setPaginationToken(seedToken, Direction.Backward); + + // Snapshot event count before pagination so we can slice only new events + const prevEventCount = headlessTimeline.getEvents().length; + + let hasMore = false; + try { + hasMore = await mx.paginateEventTimeline(headlessTimeline, { + backwards: true, + limit: BACKFILL_PAGE_SIZE, + }); + } catch { + // Pagination error — stop this room for now + backfillingRoomsRef.current.delete(room.roomId); + return; + } + + // Only process events added by this pagination pass. The headless timeline + // accumulates all paginated events, so slicing from prevEventCount avoids + // re-indexing already-seen events (O(n²) → O(n) per page). + const allEvents = headlessTimeline.getEvents(); + const newEvents = allEvents.slice(0, allEvents.length - prevEventCount); + + const events: IndexableEvent[] = []; + for (const ev of newEvents) { + if (ev.getType() === 'm.room.encrypted') { + // Still encrypted — re-use the live-indexing path which registers a + // Decrypted listener so the event is indexed once keys arrive. + indexEvent(ev, room); + } else { + const indexable = toIndexableEvent(ev, room.roomId); + if (indexable) events.push(indexable); + } + } + + if (events.length > 0) { + postToWorker({ type: 'INDEX_EVENTS', events }); + } + + const nextToken = headlessTimeline.getPaginationToken(Direction.Backward); + const done = !hasMore || !nextToken; + + postToWorker({ + type: 'SET_BACKFILL_STATE', + roomId: room.roomId, + state: { + token: nextToken, + done, + indexedCount: state.indexedCount + events.length, + }, + }); + + if (!done) { + // Schedule next page — but yield to the main sync if it's struggling. + // resumeBackfill will restart this room once sync recovers. + const nextState: BackfillState = { + token: nextToken, + done: false, + indexedCount: state.indexedCount + events.length, + }; + const cancel = scheduleIdle(() => { + const s = syncStateRef.current; + if (s !== SyncState.Syncing && s !== SyncState.Prepared && s !== SyncState.Catchup) { + backfillingRoomsRef.current.delete(room.roomId); + backfillQueueRef.current.unshift({ room, state: nextState }); + return; + } + void backfillRoom(room, nextState); + }); + cancelIdlesRef.current.push(cancel); + } else { + backfillingRoomsRef.current.delete(room.roomId); + // Dequeue the next room from the concurrency queue while under the limit + while ( + backfillingRoomsRef.current.size < MAX_CONCURRENT_BACKFILLS && + backfillQueueRef.current.length > 0 + ) { + const next = backfillQueueRef.current.shift()!; + backfillingRoomsRef.current.add(next.room.roomId); + const cancel = scheduleIdle(() => void backfillRoom(next.room, next.state)); + cancelIdlesRef.current.push(cancel); + } + if (backfillingRoomsRef.current.size === 0 && backfillQueueRef.current.length === 0) { + setIsBackfilling(false); + } + } + }, + [mx, indexEvent, postToWorker] + ); + + /** + * Dequeue rooms from the backfill queue up to the concurrency limit. + * Skips when the Matrix sync is not healthy so the /sync connection is + * never starved by background pagination requests. + */ + const resumeBackfill = useCallback(() => { + const s = syncStateRef.current; + if (s !== SyncState.Syncing && s !== SyncState.Prepared && s !== SyncState.Catchup) return; + + while ( + backfillingRoomsRef.current.size < MAX_CONCURRENT_BACKFILLS && + backfillQueueRef.current.length > 0 + ) { + const next = backfillQueueRef.current.shift()!; + backfillingRoomsRef.current.add(next.room.roomId); + const cancel = scheduleIdle(() => void backfillRoom(next.room, next.state)); + cancelIdlesRef.current.push(cancel); + } + }, [backfillRoom]); + + const startBackfill = useCallback( + (backfillStates: Record) => { + backfillStatesRef.current = backfillStates; + const rooms = mx.getRooms().filter((r) => !r.isSpaceRoom()); + + // Enqueue all unfinished rooms that are not already active + for (const room of rooms) { + const state = backfillStates[room.roomId] ?? { + token: null, + done: false, + indexedCount: 0, + }; + if (state.done) continue; + if (backfillingRoomsRef.current.has(room.roomId)) continue; + if (backfillQueueRef.current.some((e) => e.room.roomId === room.roomId)) continue; + + backfillQueueRef.current.push({ room, state }); + } + + if (backfillQueueRef.current.length > 0 || backfillingRoomsRef.current.size > 0) { + setIsBackfilling(true); + } + resumeBackfill(); + }, + [mx, resumeBackfill] + ); + + // ── Worker message handler ───────────────────────────────────────────────── + + const handleWorkerMessage = useCallback( + (event: MessageEvent) => { + const msg = event.data; + + switch (msg.type) { + case 'READY': + setIsReady(true); + // Request backfill states, then start background fill + postToWorker({ type: 'GET_BACKFILL_STATES' }); + break; + + case 'BACKFILL_STATES': + startBackfill(msg.states); + break; + + case 'QUERY_RESULT': { + const pending = pendingQueriesRef.current.get(msg.id); + if (pending) { + pendingQueriesRef.current.delete(msg.id); + pending.resolve(msg.events); + } + break; + } + + case 'STATS': { + const pending = pendingStatsRef.current; + if (pending) { + pendingStatsRef.current = null; + pending.resolve({ + indexedEventCount: msg.indexedEventCount, + roomCount: msg.roomCount, + estimatedBytes: msg.estimatedBytes, + backfillingRoomCount: pending.backfillingRoomCount, + }); + } + break; + } + + case 'ERROR': + // eslint-disable-next-line no-console + console.error('[SearchIndex worker error]', msg.message); + break; + + default: + break; + } + }, + [postToWorker, startBackfill] + ); + + // ── Worker lifecycle ─────────────────────────────────────────────────────── + + useEffect(() => { + if (!idbSearchIndex) { + setIsReady(false); + return () => {}; + } + + const userId = mx.getUserId(); + if (!userId) return () => {}; + + const worker = new Worker( + new URL('../plugins/search-worker/searchWorker.ts', import.meta.url), + { type: 'module' } + ); + workerRef.current = worker; + worker.addEventListener('message', handleWorkerMessage); + + postToWorker({ + type: 'INIT', + userId, + maxMessagesPerRoom: searchIndexMessageLimit, + } satisfies WorkerInMessage); + + // Seed sync state so backfill is correctly paused if the worker becomes + // ready before the first PREPARED/SYNCING event fires. + syncStateRef.current = mx.getSyncState(); + + // When sync recovers, restart any rooms that were paused mid-backfill. + const handleSync = (state: SyncState) => { + syncStateRef.current = state; + if ( + state === SyncState.Syncing || + state === SyncState.Prepared || + state === SyncState.Catchup + ) { + resumeBackfill(); + } + }; + mx.on(ClientEvent.Sync, handleSync as unknown as (...args: unknown[]) => void); + + // Live indexing listener + const handleTimeline = (mEvent: MatrixEvent, room: Room | undefined) => { + if (!room) return; + indexEvent(mEvent, room); + }; + mx.on(RoomEvent.Timeline, handleTimeline as unknown as (...args: unknown[]) => void); + + // Enqueue rooms added by sliding sync after the initial startBackfill call. + // Sliding sync starts with an initial window of 100 rooms; additional rooms + // are received progressively as the list expands, firing ClientEvent.Room. + const handleRoomAdded = (room: Room) => { + if (room.isSpaceRoom()) return; + if (backfillingRoomsRef.current.has(room.roomId)) return; + if (backfillQueueRef.current.some((e) => e.room.roomId === room.roomId)) return; + const state = backfillStatesRef.current[room.roomId] ?? { + token: null, + done: false, + indexedCount: 0, + }; + if (state.done) return; + backfillQueueRef.current.push({ room, state }); + setIsBackfilling(true); + resumeBackfill(); + }; + mx.on(ClientEvent.Room, handleRoomAdded as unknown as (...args: unknown[]) => void); + + return () => { + // Ask the worker to flush before terminating. We wait up to 2 s then + // force-terminate regardless so the cleanup never hangs. + worker.removeEventListener('message', handleWorkerMessage); + postToWorker({ type: 'FLUSH' }); + const terminateTimeout = setTimeout(() => { + worker.terminate(); + workerRef.current = null; + }, 2000); + worker.addEventListener('message', (ev: MessageEvent) => { + if (ev.data.type === 'FLUSH_DONE') { + clearTimeout(terminateTimeout); + worker.terminate(); + workerRef.current = null; + } + }, { once: true }); + setIsReady(false); + setIsBackfilling(false); + mx.removeListener(ClientEvent.Sync, handleSync as unknown as (...args: unknown[]) => void); + mx.removeListener( + RoomEvent.Timeline, + handleTimeline as unknown as (...args: unknown[]) => void + ); + mx.removeListener( + ClientEvent.Room, + handleRoomAdded as unknown as (...args: unknown[]) => void + ); + + // Cancel all pending idle callbacks + // eslint-disable-next-line react-hooks/exhaustive-deps -- mutable non-DOM refs, current is intentional at cleanup time + const cancels = cancelIdlesRef.current; + // eslint-disable-next-line react-hooks/exhaustive-deps -- mutable non-DOM refs, current is intentional at cleanup time + const backfillingRooms = backfillingRoomsRef.current; + // eslint-disable-next-line react-hooks/exhaustive-deps -- mutable non-DOM refs, current is intentional at cleanup time + const headlessSets = headlessSetsRef.current; + for (const cancel of cancels) cancel(); + cancelIdlesRef.current = []; + backfillingRooms.clear(); + headlessSets.clear(); + backfillQueueRef.current = []; + + // Reject any in-flight search/stats promises so callers don't hang forever + // eslint-disable-next-line react-hooks/exhaustive-deps -- mutable non-DOM refs, current is intentional at cleanup time + const pendingQueries = pendingQueriesRef.current; + // eslint-disable-next-line react-hooks/exhaustive-deps -- mutable non-DOM refs, current is intentional at cleanup time + const pendingStats = pendingStatsRef.current; + for (const { reject } of pendingQueries.values()) { + reject(new Error('Search index unmounted')); + } + pendingQueries.clear(); + if (pendingStats) { + pendingStats.resolve({ + indexedEventCount: 0, + roomCount: 0, + estimatedBytes: 0, + backfillingRoomCount: 0, + }); + pendingStatsRef.current = null; + } + }; + }, [ + idbSearchIndex, + mx, + searchIndexMessageLimit, + handleWorkerMessage, + indexEvent, + resumeBackfill, + postToWorker, + ]); + + // ── Public API ───────────────────────────────────────────────────────────── + + const query = useCallback( + ( + term: string, + opts?: { roomIds?: string[]; senders?: string[]; hasTypes?: string[] } + ): Promise => { + if (!workerRef.current || !isReady) return Promise.resolve([]); + const id = crypto.randomUUID(); + return new Promise((resolve, reject) => { + pendingQueriesRef.current.set(id, { resolve, reject }); + postToWorker({ + type: 'QUERY', + id, + term, + roomIds: opts?.roomIds, + senders: opts?.senders, + hasTypes: opts?.hasTypes, + }); + }); + }, + [isReady, postToWorker] + ); + + const getStats = useCallback((): Promise => { + if (!workerRef.current || !isReady) { + return Promise.resolve({ + indexedEventCount: 0, + roomCount: 0, + estimatedBytes: 0, + backfillingRoomCount: 0, + }); + } + return new Promise((resolve) => { + pendingStatsRef.current = { + backfillingRoomCount: backfillingRoomsRef.current.size, + resolve, + }; + postToWorker({ type: 'GET_STATS' }); + }); + }, [isReady, postToWorker]); + + const clearIndex = useCallback((): Promise => { + if (!workerRef.current) return Promise.resolve(); + postToWorker({ type: 'CLEAR_INDEX' }); + // Reset local state + headlessSetsRef.current.clear(); + for (const cancel of cancelIdlesRef.current) cancel(); + cancelIdlesRef.current = []; + backfillingRoomsRef.current.clear(); + setIsBackfilling(false); + return Promise.resolve(); + }, [postToWorker]); + + const ctx = useMemo( + () => ({ query, getStats, clearIndex, isReady, isBackfilling }), + [query, getStats, clearIndex, isReady, isBackfilling] + ); + + return {children}; +} diff --git a/src/app/pages/Router.tsx b/src/app/pages/Router.tsx index aec62cc86..84d995558 100644 --- a/src/app/pages/Router.tsx +++ b/src/app/pages/Router.tsx @@ -66,7 +66,7 @@ import { import { ClientBindAtoms, ClientLayout, ClientRoot, ClientRouteOutlet } from './client'; import { HandleNotificationClick, ClientNonUIFeatures } from './client/ClientNonUIFeatures'; import { Home, HomeRouteRoomProvider, HomeSearch } from './client/home'; -import { Direct, DirectCreate, DirectRouteRoomProvider } from './client/direct'; +import { Direct, DirectCreate, DirectRouteRoomProvider, DirectSearch } from './client/direct'; import { RouteSpaceProvider, Space, SpaceRouteRoomProvider, SpaceSearch } from './client/space'; import { Explore, FeaturedRooms, PublicRooms } from './client/explore'; import { Notifications, Inbox, Invites } from './client/inbox'; @@ -268,6 +268,7 @@ export const createRouter = (clientConfig: ClientConfig, screenSize: ScreenSize) > {mobile ? null : } />} } /> + } /> + @@ -884,6 +885,6 @@ export function ClientNonUIFeatures({ children }: ClientNonUIFeaturesProps) { {children} - + ); } diff --git a/src/app/pages/client/direct/Direct.tsx b/src/app/pages/client/direct/Direct.tsx index 2a81b849d..58e11ea26 100644 --- a/src/app/pages/client/direct/Direct.tsx +++ b/src/app/pages/client/direct/Direct.tsx @@ -30,8 +30,9 @@ import { NavEmptyLayout, NavItem, NavItemContent, + NavLink, } from '$components/nav'; -import { getDirectCreatePath, getDirectRoomPath } from '$pages/pathUtils'; +import { getDirectCreatePath, getDirectRoomPath, getDirectSearchPath } from '$pages/pathUtils'; import { getCanonicalAliasOrRoomId } from '$utils/matrix'; import { useSelectedRoom } from '$hooks/router/useSelectedRoom'; import { VirtualTile } from '$components/virtualizer'; @@ -51,7 +52,7 @@ import { getRoomNotificationMode, useRoomsNotificationPreferencesContext, } from '$hooks/useRoomsNotificationPreferences'; -import { useDirectCreateSelected } from '$hooks/router/useDirectSelected'; +import { useDirectCreateSelected, useDirectSearchSelected } from '$hooks/router/useDirectSelected'; import { useDirectRooms } from './useDirectRooms'; import { SidebarResizer } from '$pages/client/sidebar/SidebarResizer'; import { useScreenSizeContext, ScreenSize } from '$hooks/useScreenSize'; @@ -198,6 +199,7 @@ export function Direct() { const [joinCallOnSingleClick] = useSetting(settingsAtom, 'joinCallOnSingleClick'); const createDirectSelected = useDirectCreateSelected(); + const searchSelected = useDirectSearchSelected(); const selectedRoomId = useSelectedRoom(); const noRoomToDisplay = directs.length === 0; @@ -281,10 +283,14 @@ export function Direct() { as="span" grow="Yes" alignItems="Center" + justifyContent="Start" gap="200" - justifyContent="Center" > - + {!hideText && ( @@ -298,6 +304,34 @@ export function Direct() { + + + + + + + + {!hideText && ( + + + Message Search + + + )} + + + + diff --git a/src/app/pages/client/direct/Search.tsx b/src/app/pages/client/direct/Search.tsx new file mode 100644 index 000000000..bc6eb4620 --- /dev/null +++ b/src/app/pages/client/direct/Search.tsx @@ -0,0 +1,54 @@ +import { useRef } from 'react'; +import { Box, Icon, Icons, Text, Scroll, IconButton } from 'folds'; +import { Page, PageContent, PageContentCenter, PageHeader } from '$components/page'; +import { MessageSearch } from '$features/message-search'; +import { ScreenSize, useScreenSizeContext } from '$hooks/useScreenSize'; +import { BackRouteHandler } from '$components/BackRouteHandler'; +import { useDirectRooms } from './useDirectRooms'; + +export function DirectSearch() { + const scrollRef = useRef(null); + const rooms = useDirectRooms(); + const screenSize = useScreenSizeContext(); + + return ( + + + + + {screenSize === ScreenSize.Mobile && ( + + {(onBack) => ( + + + + )} + + )} + + + {screenSize !== ScreenSize.Mobile && } + + Message Search + + + + + + + + + + + + + + + + ); +} diff --git a/src/app/pages/client/direct/index.ts b/src/app/pages/client/direct/index.ts index d247bbc03..770aa28f2 100644 --- a/src/app/pages/client/direct/index.ts +++ b/src/app/pages/client/direct/index.ts @@ -1,3 +1,4 @@ export * from './Direct'; export * from './RoomProvider'; export * from './DirectCreate'; +export * from './Search'; diff --git a/src/app/pages/pathUtils.ts b/src/app/pages/pathUtils.ts index 4a95f47fc..9d2ef00b9 100644 --- a/src/app/pages/pathUtils.ts +++ b/src/app/pages/pathUtils.ts @@ -7,6 +7,7 @@ import { DIRECT_CREATE_PATH, DIRECT_PATH, DIRECT_ROOM_PATH, + DIRECT_SEARCH_PATH, EXPLORE_FEATURED_PATH, EXPLORE_PATH, EXPLORE_SERVER_PATH, @@ -102,6 +103,7 @@ export const getHomeRoomPath = (roomIdOrAlias: string, eventId?: string): string export const getDirectPath = (): string => DIRECT_PATH; export const getDirectCreatePath = (): string => DIRECT_CREATE_PATH; +export const getDirectSearchPath = (): string => DIRECT_SEARCH_PATH; export const getDirectRoomPath = (roomIdOrAlias: string, eventId?: string): string => { const params = { roomIdOrAlias: encodeURIComponent(roomIdOrAlias), diff --git a/src/app/pages/paths.ts b/src/app/pages/paths.ts index 1ac57b756..046f7f0eb 100644 --- a/src/app/pages/paths.ts +++ b/src/app/pages/paths.ts @@ -37,6 +37,7 @@ export type SearchPathSearchParams = { order?: string; rooms?: string; senders?: string; + has?: string; }; export const SEARCH_PATH_SEGMENT = 'search/'; @@ -57,6 +58,7 @@ export type DirectCreateSearchParams = { userId?: string; }; export const DIRECT_CREATE_PATH = `/direct/${CREATE_PATH_SEGMENT}`; +export const DIRECT_SEARCH_PATH = `/direct/${SEARCH_PATH_SEGMENT}`; export const DIRECT_ROOM_PATH = `/direct/${ROOM_PATH_SEGMENT}`; export const SPACE_PATH = '/:spaceIdOrAlias/'; diff --git a/src/app/plugins/search-worker/searchWorker.ts b/src/app/plugins/search-worker/searchWorker.ts new file mode 100644 index 000000000..c766b2af6 --- /dev/null +++ b/src/app/plugins/search-worker/searchWorker.ts @@ -0,0 +1,429 @@ +/** + * searchWorker.ts — Web Worker that owns the MiniSearch index and IndexedDB persistence. + * + * Vite instantiation (main thread): + * new Worker(new URL('./searchWorker.ts', import.meta.url), { type: 'module' }) + */ + +import MiniSearch from 'minisearch'; +import type { IndexableEvent, BackfillState, WorkerInMessage, WorkerOutMessage } from './types'; + +// ── IDB helpers ───────────────────────────────────────────────────────────── + +function openDb(dbName: string): Promise { + return new Promise((resolve, reject) => { + const req = indexedDB.open(dbName, 3); + req.onupgradeneeded = (event) => { + const db = req.result; + const oldVersion = event.oldVersion; + if (oldVersion < 1) { + db.createObjectStore('index'); + db.createObjectStore('backfill'); + } + // v2: added msgtype to stored fields + if (oldVersion >= 1 && oldVersion < 2) { + db.deleteObjectStore('index'); + db.deleteObjectStore('backfill'); + db.createObjectStore('index'); + db.createObjectStore('backfill'); + } + // v3: added url/file/info/filename to stored fields for media events + if (oldVersion >= 2 && oldVersion < 3) { + db.deleteObjectStore('index'); + db.deleteObjectStore('backfill'); + db.createObjectStore('index'); + db.createObjectStore('backfill'); + } + }; + req.addEventListener('success', () => resolve(req.result)); + req.addEventListener('error', () => reject(req.error)); + }); +} + +function idbGet(db: IDBDatabase, store: string, key: string): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(store, 'readonly'); + const req = tx.objectStore(store).get(key); + req.addEventListener('success', () => resolve(req.result as T | undefined)); + req.addEventListener('error', () => reject(req.error)); + }); +} + +function idbPut(db: IDBDatabase, store: string, key: string, value: unknown): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(store, 'readwrite'); + const req = tx.objectStore(store).put(value, key); + req.addEventListener('success', () => resolve()); + req.addEventListener('error', () => reject(req.error)); + }); +} + +function idbGetAll(db: IDBDatabase, store: string): Promise<{ key: string; value: unknown }[]> { + return new Promise((resolve, reject) => { + const tx = db.transaction(store, 'readonly'); + const results: { key: string; value: unknown }[] = []; + const keysReq = tx.objectStore(store).openCursor(); + keysReq.addEventListener('success', () => { + const cursor = keysReq.result; + if (cursor) { + results.push({ key: cursor.key as string, value: cursor.value as unknown }); + cursor.continue(); + } else { + resolve(results); + } + }); + keysReq.addEventListener('error', () => reject(keysReq.error)); + }); +} + +function idbClear(db: IDBDatabase, store: string): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(store, 'readwrite'); + const req = tx.objectStore(store).clear(); + req.addEventListener('success', () => resolve()); + req.addEventListener('error', () => reject(req.error)); + }); +} + +// ── Worker state ───────────────────────────────────────────────────────────── + +let db: IDBDatabase | null = null; +let index: MiniSearch | null = null; +let maxMessagesPerRoom = 2000; + +/** + * Per-room queue of [eventId, ts] sorted ascending by ts. + * Used for LRU eviction when the per-room limit is exceeded. + */ +const roomQueues = new Map>(); + +/** + * Parallel store of all indexed documents, keyed by eventId. + * Maintained alongside the MiniSearch index to avoid relying on + * private MiniSearch internals (_storedFields) for full-scan queries. + */ +const storedDocs = new Map(); + +/** Dirty flag — index changed since last flush */ +let dirty = false; + +const FLUSH_DEBOUNCE_MS = 5000; +let flushTimer: ReturnType | null = null; + +function makeIndex(): MiniSearch { + return new MiniSearch({ + idField: 'eventId', + fields: ['body', 'sender'], + storeFields: [ + 'eventId', + 'roomId', + 'sender', + 'msgtype', + 'ts', + 'body', + 'url', + 'file', + 'info', + 'filename', + ], + searchOptions: { + boost: { body: 2 }, + fuzzy: 0.2, + prefix: true, + combineWith: 'AND', + }, + }); +} + +function post(msg: WorkerOutMessage): void { + // oxlint-disable-next-line require-post-message-target-origin -- Worker.postMessage has no targetOrigin + self.postMessage(msg); +} + +function scheduleFlush(): void { + if (flushTimer !== null) clearTimeout(flushTimer); + flushTimer = setTimeout(() => { + flushTimer = null; + void flushIndex(); + }, FLUSH_DEBOUNCE_MS); +} + +async function flushIndex(): Promise { + if (!db || !index || !dirty) return; + try { + const doFlush = async () => { + if (!db || !index) return; + // Persist serialized MiniSearch + await idbPut(db, 'index', 'v1', JSON.stringify(index)); + // Persist room queues for reconstruction on next load + const roomQueuesData: Record> = {}; + for (const [roomId, queue] of roomQueues.entries()) { + roomQueuesData[roomId] = queue; + } + await idbPut(db, 'index', 'rooms', roomQueuesData); + dirty = false; + }; + if ('locks' in navigator) { + await navigator.locks.request('sable-search-index-writer', { mode: 'exclusive' }, doFlush); + } else { + await doFlush(); + } + } catch { + // Non-fatal: will retry on next flush + } +} + +// ── Eviction (oldest-first per room) ────────────────────────────────────────── + +function evictOldestForRoom(roomId: string): void { + if (!index) return; + const queue = roomQueues.get(roomId); + if (!queue) return; + const excess = queue.length - maxMessagesPerRoom; + if (excess <= 0) return; + + const toRemove = queue.splice(0, excess); + for (const [eventId] of toRemove) { + index.discard(eventId); + storedDocs.delete(eventId); + } +} + +// ── Message handler ──────────────────────────────────────────────────────── + +/** Iterate every document stored in the index using the parallel storedDocs map. */ +function iterateStoredDocs(): IterableIterator { + return storedDocs.values(); +} + +/** Matrix msgtype for each SearchHasType chip. */ +const HAS_TYPE_TO_MSGTYPE: Record = { + image: 'm.image', + file: 'm.file', + audio: 'm.audio', + video: 'm.video', +}; + +function makeTypeFilter(hasTypes: string[] | undefined): ((ev: IndexableEvent) => boolean) | null { + if (!hasTypes || hasTypes.length === 0) return null; + const allowedMsgtypes = new Set(hasTypes.map((t) => HAS_TYPE_TO_MSGTYPE[t]).filter(Boolean)); + const needsLink = hasTypes.includes('link'); + return (ev: IndexableEvent) => { + if (allowedMsgtypes.has(ev.msgtype)) return true; + if (needsLink && /https?:\/\//i.test(ev.body)) return true; + return false; + }; +} + +async function handleInit(userId: string, maxPerRoom: number): Promise { + maxMessagesPerRoom = maxPerRoom; + const dbName = `sable-search-${userId}`; + + db = await openDb(dbName); + + const serialized = await idbGet(db, 'index', 'v1'); + if (serialized) { + try { + index = MiniSearch.loadJSON(serialized, { + idField: 'eventId', + fields: ['body', 'sender'], + storeFields: [ + 'eventId', + 'roomId', + 'sender', + 'msgtype', + 'ts', + 'body', + 'url', + 'file', + 'info', + 'filename', + ], + searchOptions: { + boost: { body: 2 }, + fuzzy: 0.2, + prefix: true, + combineWith: 'AND', + }, + }); + // Rebuild room queues from persisted data + const savedQueues = await idbGet>>( + db, + 'index', + 'rooms' + ); + if (savedQueues) { + for (const [roomId, queue] of Object.entries(savedQueues)) { + roomQueues.set(roomId, queue); + } + } + } catch { + index = makeIndex(); + } + } else { + index = makeIndex(); + } + + post({ + type: 'READY', + indexedEventCount: index.documentCount, + roomCount: roomQueues.size, + }); +} + +function handleIndexEvents(events: IndexableEvent[]): void { + if (!index) return; + + for (const ev of events) { + if (!ev.eventId || !ev.body.trim()) continue; + + // Skip duplicates already in the index + if (index.has(ev.eventId)) continue; + + index.add(ev); + storedDocs.set(ev.eventId, ev); + let queue = roomQueues.get(ev.roomId); + if (!queue) { + queue = []; + roomQueues.set(ev.roomId, queue); + } + // Insert in ts-ascending order (most backfill arrives in order, so push is common) + const lastEntry = queue[queue.length - 1]; + if (queue.length === 0 || (lastEntry !== undefined && lastEntry[1] <= ev.ts)) { + queue.push([ev.eventId, ev.ts]); + } else { + // Binary search insertion for out-of-order events + let lo = 0; + let hi = queue.length; + while (lo < hi) { + const mid = (lo + hi) >>> 1; + const midEntry = queue[mid]; + if (midEntry !== undefined && midEntry[1] <= ev.ts) lo = mid + 1; + else hi = mid; + } + queue.splice(lo, 0, [ev.eventId, ev.ts]); + } + + // Enforce per-room limit; amortise by only evicting when 10% over + if (queue.length > maxMessagesPerRoom * 1.1) { + evictOldestForRoom(ev.roomId); + } + } + + dirty = true; + scheduleFlush(); +} + +function handleQuery( + id: string, + term: string, + roomIds?: string[], + senders?: string[], + hasTypes?: string[] +): void { + if (!index) { + post({ type: 'QUERY_RESULT', id, events: [] }); + return; + } + + const typeFilter = makeTypeFilter(hasTypes); + + function matchesFilters(ev: IndexableEvent): boolean { + if (roomIds && roomIds.length > 0 && !roomIds.includes(ev.roomId)) return false; + if (senders && senders.length > 0 && !senders.includes(ev.sender)) return false; + if (typeFilter && !typeFilter(ev)) return false; + return true; + } + + if (!term) { + // Chip-only query: scan all stored documents — MiniSearch can't search an empty term. + const results: IndexableEvent[] = []; + for (const ev of iterateStoredDocs()) { + if (matchesFilters(ev)) results.push(ev); + } + post({ type: 'QUERY_RESULT', id, events: results }); + return; + } + + const rawResults = index.search(term, { + filter: (r) => matchesFilters(r as unknown as IndexableEvent), + }) as unknown as IndexableEvent[]; + + post({ type: 'QUERY_RESULT', id, events: rawResults }); +} + +async function handleSetBackfillState(roomId: string, state: BackfillState): Promise { + if (!db) return; + await idbPut(db, 'backfill', roomId, state); +} + +async function handleGetBackfillStates(): Promise { + if (!db) { + post({ type: 'BACKFILL_STATES', states: {} }); + return; + } + const rows = await idbGetAll(db, 'backfill'); + const states: Record = {}; + for (const { key, value } of rows) { + states[key] = value as BackfillState; + } + post({ type: 'BACKFILL_STATES', states }); +} + +function handleGetStats(): void { + if (!index) { + post({ type: 'STATS', indexedEventCount: 0, roomCount: 0, estimatedBytes: 0 }); + return; + } + const serialized = JSON.stringify(index); + post({ + type: 'STATS', + indexedEventCount: index.documentCount, + roomCount: roomQueues.size, + estimatedBytes: serialized.length * 2, // UTF-16 approximation + }); +} + +async function handleClearIndex(): Promise { + if (!db) return; + index = makeIndex(); + roomQueues.clear(); + storedDocs.clear(); + dirty = false; + await idbClear(db, 'index'); + await idbClear(db, 'backfill'); +} + +self.addEventListener('message', (event: MessageEvent) => { + const msg = event.data; + switch (msg.type) { + case 'INIT': + void handleInit(msg.userId, msg.maxMessagesPerRoom); + break; + case 'INDEX_EVENTS': + handleIndexEvents(msg.events); + break; + case 'QUERY': + handleQuery(msg.id, msg.term, msg.roomIds, msg.senders, msg.hasTypes); + break; + case 'SET_BACKFILL_STATE': + void handleSetBackfillState(msg.roomId, msg.state); + break; + case 'GET_BACKFILL_STATES': + void handleGetBackfillStates(); + break; + case 'GET_STATS': + handleGetStats(); + break; + case 'CLEAR_INDEX': + void handleClearIndex(); + break; + case 'FLUSH': + void flushIndex().then(() => { + self.postMessage({ type: 'FLUSH_DONE' }); + }); + break; + default: + break; + } +}); diff --git a/src/app/plugins/search-worker/types.ts b/src/app/plugins/search-worker/types.ts new file mode 100644 index 000000000..f1cba3d66 --- /dev/null +++ b/src/app/plugins/search-worker/types.ts @@ -0,0 +1,98 @@ +/** A plain, serializable event suitable for passing into/out of the search worker. */ +export type IndexableEvent = { + eventId: string; + roomId: string; + sender: string; + /** Matrix msgtype, e.g. 'm.text', 'm.image', 'm.file', 'm.audio', 'm.video'. */ + msgtype: string; + body: string; + ts: number; + // ── Media fields (present for m.image / m.file / m.audio / m.video) ── + /** mxc:// URL for unencrypted media. */ + url?: string; + /** EncryptedFile descriptor for encrypted media (contains url + key material). */ + file?: Record; + /** Dimensions, mimetype, size, thumbnail info, etc. */ + info?: Record; + /** Original filename (m.file). */ + filename?: string; +}; + +export type BackfillState = { + /** Pagination token to resume backward pagination, or null when at the beginning. */ + token: string | null; + /** True once we've reached the beginning of the room history. */ + done: boolean; + /** How many events for this room are currently in the index. */ + indexedCount: number; +}; + +// ── Main → Worker ────────────────────────────────────────────────────────── + +export type WorkerInMessage = + | { + type: 'INIT'; + userId: string; + maxMessagesPerRoom: number; + } + | { + type: 'INDEX_EVENTS'; + events: IndexableEvent[]; + } + | { + type: 'QUERY'; + id: string; + term: string; + roomIds?: string[]; + senders?: string[]; + /** SearchHasType values to filter by, e.g. ['image', 'link']. */ + hasTypes?: string[]; + } + | { + type: 'SET_BACKFILL_STATE'; + roomId: string; + state: BackfillState; + } + | { + type: 'GET_BACKFILL_STATES'; + } + | { + type: 'GET_STATS'; + } + | { + type: 'CLEAR_INDEX'; + } + | { + type: 'FLUSH'; + }; + +// ── Worker → Main ────────────────────────────────────────────────────────── + +export type WorkerOutMessage = + | { + type: 'READY'; + indexedEventCount: number; + roomCount: number; + } + | { + type: 'QUERY_RESULT'; + id: string; + events: IndexableEvent[]; + } + | { + type: 'BACKFILL_STATES'; + states: Record; + } + | { + type: 'STATS'; + indexedEventCount: number; + roomCount: number; + estimatedBytes: number; + } + | { + type: 'ERROR'; + message: string; + } + | { + type: 'FLUSH_DONE'; + }; diff --git a/src/app/state/settings.ts b/src/app/state/settings.ts index 5efe57552..d348e92a4 100644 --- a/src/app/state/settings.ts +++ b/src/app/state/settings.ts @@ -128,6 +128,9 @@ export interface Settings { developerTools: boolean; enableMSC4268CMD: boolean; settingsSyncEnabled: boolean; + encryptedSearch: boolean; + idbSearchIndex: boolean; + searchIndexMessageLimit: number; // Cosmetics! jumboEmojiSize: JumboEmojiSize; @@ -264,6 +267,9 @@ export const defaultSettings: Settings = { developerTools: false, settingsSyncEnabled: false, + encryptedSearch: false, + idbSearchIndex: false, + searchIndexMessageLimit: 2000, // Cosmetics! jumboEmojiSize: 'normal', diff --git a/src/app/utils/settingsSync.test.ts b/src/app/utils/settingsSync.test.ts index 608a94343..9c91b3cdc 100644 --- a/src/app/utils/settingsSync.test.ts +++ b/src/app/utils/settingsSync.test.ts @@ -33,6 +33,7 @@ describe('NON_SYNCABLE_KEYS', () => { 'memberSortFilterIndex', 'developerTools', 'settingsSyncEnabled', + 'searchIndexMessageLimit', ] as const; expected.forEach((key) => { diff --git a/src/app/utils/settingsSync.ts b/src/app/utils/settingsSync.ts index 83c8ff11f..8be66853c 100644 --- a/src/app/utils/settingsSync.ts +++ b/src/app/utils/settingsSync.ts @@ -18,6 +18,9 @@ export const NON_SYNCABLE_KEYS = new Set([ 'developerTools', // Sync toggle itself must never be uploaded (it's device-local) 'settingsSyncEnabled', + // Search index capacity varies by device (mobile is capped at 50 MB, desktop at 300 MB), + // so each device should keep its own limit rather than inheriting a desktop value. + 'searchIndexMessageLimit', ]); export const SETTINGS_SYNC_VERSION = 1;