From 68f3c45864c21c92d1aa77efbec4c69eef7d1a69 Mon Sep 17 00:00:00 2001 From: Tommy Carlsson <1904092+TommyC81@users.noreply.github.com> Date: Tue, 18 Aug 2026 10:32:09 +0400 Subject: [PATCH] feat: make max file size configurable via SEMBLE_MAX_FILE_BYTES Files larger than 1 MB are skipped during indexing without any indication, silently leaving gaps in search results (#250). Resolve the limit per call from the SEMBLE_MAX_FILE_BYTES environment variable (following SEMBLE_CACHE_LOCATION / SEMBLE_CLONE_TIMEOUT / SEMBLE_MODEL_NAME), falling back to the unchanged 1 MB default; malformed or nonpositive values warn and fall back instead of crashing indexing. Warn at index time naming files skipped for size, with the CLI surfacing warnings on stderr via an idempotent, CLI-owned handler. --- README.md | 2 ++ src/semble/cli.py | 18 +++++++++++++++ src/semble/index/create.py | 23 +++++++++++++++++++ src/semble/index/files.py | 27 ++++++++++++++++++++-- tests/index/test_index.py | 47 +++++++++++++++++++++++++++++++++----- 5 files changed, 109 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a66069ec1..1ea196502 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,8 @@ Savings are calculated as follows: for each call, semble records the total chara By default, your Semble savings statistics and any saved indexes are stored in the OS cache folder (`~/Library/Caches/semble/` on macOS, `~/.cache/semble/` on Linux, `%LOCALAPPDATA%\semble\Cache\` on Windows). To override this location you can supply an environment variable `SEMBLE_CACHE_LOCATION` which should be the full path to the target cache location e.g. `~/my-folder/my-caches/semble`. +Files larger than 1 MB are skipped during indexing to keep index builds lean. Skipped files are reported as a warning at index time. If you work with large generated or ingested documents, you can raise (or lower) this limit with the `SEMBLE_MAX_FILE_BYTES` environment variable (in bytes). + On first use, Semble also downloads the embedding model from Hugging Face and caches it in the standard Hugging Face cache (`~/.cache/huggingface/` by default, or `$HF_HOME` if set); this only happens once and requires network access. Use `semble clear` to remove cached data: `semble clear index` (saved indexes), `semble clear savings` (usage stats), `semble clear orphans` (indexes for repos no longer present on disk), or `semble clear all` (everything). diff --git a/src/semble/cli.py b/src/semble/cli.py index 466b42ef7..1ba51b298 100644 --- a/src/semble/cli.py +++ b/src/semble/cli.py @@ -2,6 +2,7 @@ import asyncio import io import json +import logging import re import sys import warnings @@ -207,7 +208,24 @@ def _run_clear(clear_type: _CLEAR_CHOICE) -> None: _clear_orphans(cache_folder) +class _CliLogHandler(logging.StreamHandler): + """stderr handler owned by the CLI; setup is idempotent on this type, not on foreign handlers.""" + + +def _configure_cli_logging() -> None: + """Surface semble warnings (e.g. skipped oversized files) on stderr without touching the root logger.""" + package_logger = logging.getLogger("semble") + if any(isinstance(handler, _CliLogHandler) for handler in package_logger.handlers): + return + handler = _CliLogHandler(sys.stderr) + handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) + package_logger.addHandler(handler) + if package_logger.level == logging.NOTSET: + package_logger.setLevel(logging.WARNING) + + def _cli_main() -> None: + _configure_cli_logging() parser = argparse.ArgumentParser(prog="semble") parser.add_argument("-V", "--version", action="version", version=__version__) sub = parser.add_subparsers(dest="command") diff --git a/src/semble/index/create.py b/src/semble/index/create.py index 8354ea57e..c2c8a80d5 100644 --- a/src/semble/index/create.py +++ b/src/semble/index/create.py @@ -1,4 +1,5 @@ import contextlib +import logging from collections.abc import Sequence from pathlib import Path @@ -15,6 +16,7 @@ detect_language, get_extensions, get_file_status, + get_max_file_bytes, read_file_text, ) from semble.index.sparse import enrich_for_bm25 @@ -22,6 +24,21 @@ from semble.tokens import tokenize from semble.types import Chunk, ContentType, EmbeddingMatrix +logger = logging.getLogger(__name__) + + +def _warn_skipped_large(skipped_large: list[str]) -> None: + """Warn about files skipped for exceeding the maximum indexable file size.""" + if skipped_large: + logger.warning( + "Skipped %d file(s) exceeding the maximum file size of %d bytes " + "(raise SEMBLE_MAX_FILE_BYTES to include them): %s%s", + len(skipped_large), + get_max_file_bytes(), + ", ".join(skipped_large[:5]), + " ..." if len(skipped_large) > 5 else "", + ) + def _reindex_file( bm25_index: BM25, @@ -79,10 +96,14 @@ def create_index_from_path( manifest: dict[str, FileManifestEntry] = {} embedding_parts: list[tuple[int, int, int]] = [] + skipped_large: list[str] = [] + for file_path in walk_files(path, resolved_extensions): language = detect_language(file_path) with contextlib.suppress(OSError): file_status = get_file_status(file_path, None) + if file_status is FileStatus.TOO_LARGE: + skipped_large.append(str(file_path)) if file_status != FileStatus.VALID: continue @@ -109,6 +130,8 @@ def create_index_from_path( for indexed_path in previous_manifest.keys() - manifest.keys(): _reindex_file(bm25_index, indexed_path, [], previous_manifest[indexed_path]) + _warn_skipped_large(skipped_large) + if not chunks: raise ValueError(f"No supported files found under {path}.") diff --git a/src/semble/index/files.py b/src/semble/index/files.py index 7aa0702bf..e3f4f9419 100644 --- a/src/semble/index/files.py +++ b/src/semble/index/files.py @@ -1,3 +1,5 @@ +import logging +import os from collections import defaultdict from collections.abc import Sequence from enum import Enum @@ -5,8 +7,10 @@ from semble.types import ContentType -_MAX_FILE_BYTES = 1_000_000 # 1 MB max file size to read and index +_DEFAULT_MAX_FILE_BYTES = 1_000_000 # Default 1 MB max file size to read and index _EMPTY_FILE_BYTES = 128 + +logger = logging.getLogger(__name__) _EXTENSION_TO_LANGUAGE = { ".4th": "forth", ".ada": "ada", @@ -488,6 +492,25 @@ def read_file_text(file_path: Path) -> str: return file_path.read_text(encoding="utf-8", errors="replace") +def get_max_file_bytes() -> int: + """Resolve the maximum file size to index from SEMBLE_MAX_FILE_BYTES, falling back to the default. + + Malformed or nonpositive values warn and fall back to the default rather than crash indexing. + """ + raw = os.environ.get("SEMBLE_MAX_FILE_BYTES") + if raw is None: + return _DEFAULT_MAX_FILE_BYTES + try: + value = int(raw) + except ValueError: + logger.warning("Invalid SEMBLE_MAX_FILE_BYTES %r, using the default of %d bytes", raw, _DEFAULT_MAX_FILE_BYTES) + return _DEFAULT_MAX_FILE_BYTES + if value <= 0: + logger.warning("SEMBLE_MAX_FILE_BYTES must be positive, using the default of %d bytes", _DEFAULT_MAX_FILE_BYTES) + return _DEFAULT_MAX_FILE_BYTES + return value + + def get_file_status(file_path: Path, write_time: float | None) -> FileStatus: """Checks if a file should be indexed based on its size and modification time.""" stat = file_path.stat() @@ -495,7 +518,7 @@ def get_file_status(file_path: Path, write_time: float | None) -> FileStatus: # Index invalid, file invalid return FileStatus.NEWER size = stat.st_size - if size > _MAX_FILE_BYTES: + if size > get_max_file_bytes(): # index valid, file invalid return FileStatus.TOO_LARGE if size < _EMPTY_FILE_BYTES and not read_file_text(file_path).strip(): diff --git a/tests/index/test_index.py b/tests/index/test_index.py index 0f20c6631..e485c89be 100644 --- a/tests/index/test_index.py +++ b/tests/index/test_index.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from typing import Any from unittest.mock import MagicMock, patch @@ -8,7 +9,7 @@ from semble import SembleIndex from semble.index.create import create_index_from_path -from semble.index.files import _MAX_FILE_BYTES, FileStatus, get_file_status +from semble.index.files import _DEFAULT_MAX_FILE_BYTES, FileStatus, get_file_status, get_max_file_bytes from semble.types import ContentType from tests.conftest import make_chunk @@ -69,11 +70,45 @@ def test_index_empty_returns_zero_chunks(mock_model: StaticModel, tmp_path: Path create_index_from_path(tmp_path, mock_model) -def test_oversized_file_is_skipped(mock_model: StaticModel, tmp_path: Path) -> None: - """Files exceeding _MAX_FILE_BYTES are silently skipped during indexing.""" - (tmp_path / "big.py").write_bytes(b"x" * (_MAX_FILE_BYTES + 1)) - with pytest.raises(ValueError): # no indexable content remains - create_index_from_path(tmp_path, mock_model) +def test_max_file_bytes_resolution(monkeypatch: pytest.MonkeyPatch) -> None: + """The limit resolves from SEMBLE_MAX_FILE_BYTES, falling back to the 1 MB default.""" + monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False) + assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES + monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", str(_DEFAULT_MAX_FILE_BYTES + 5)) + assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES + 5 + + +def test_max_file_bytes_invalid_values_fall_back(monkeypatch: pytest.MonkeyPatch) -> None: + """Malformed or nonpositive SEMBLE_MAX_FILE_BYTES values fall back to the default.""" + monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False) + assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES + for bad in ("not-a-number", "0", "-5"): + monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", bad) + assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES + + +def test_oversized_file_is_skipped_with_warning( + mock_model: StaticModel, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Files exceeding the limit are skipped during indexing, with a warning naming them.""" + monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False) + (tmp_path / "big.py").write_bytes(b"x" * (_DEFAULT_MAX_FILE_BYTES + 1)) + with caplog.at_level(logging.WARNING, logger="semble.index.create"): + with pytest.raises(ValueError): # no indexable content remains + create_index_from_path(tmp_path, mock_model) + assert "big.py" in caplog.text + assert str(_DEFAULT_MAX_FILE_BYTES) in caplog.text + assert "SEMBLE_MAX_FILE_BYTES" in caplog.text + + +def test_oversized_file_indexed_when_limit_raised( + mock_model: StaticModel, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Raising SEMBLE_MAX_FILE_BYTES lets oversized files into the index.""" + monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", str(_DEFAULT_MAX_FILE_BYTES + 1024)) + (tmp_path / "big.py").write_bytes(b"x = 1\n" + b"#" * _DEFAULT_MAX_FILE_BYTES) + _, _, chunks, _ = create_index_from_path(tmp_path, mock_model) + assert any(chunk.file_path.endswith("big.py") for chunk in chunks) def test_tiny_invalid_utf8_file_status_does_not_crash(tmp_path: Path) -> None: