diff --git a/README.md b/README.md index a66069ec..1ea19650 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,8 @@ Savings are calculated as follows: for each call, semble records the total chara By default, your Semble savings statistics and any saved indexes are stored in the OS cache folder (`~/Library/Caches/semble/` on macOS, `~/.cache/semble/` on Linux, `%LOCALAPPDATA%\semble\Cache\` on Windows). To override this location you can supply an environment variable `SEMBLE_CACHE_LOCATION` which should be the full path to the target cache location e.g. `~/my-folder/my-caches/semble`. +Files larger than 1 MB are skipped during indexing to keep index builds lean. Skipped files are reported as a warning at index time. If you work with large generated or ingested documents, you can raise (or lower) this limit with the `SEMBLE_MAX_FILE_BYTES` environment variable (in bytes). + On first use, Semble also downloads the embedding model from Hugging Face and caches it in the standard Hugging Face cache (`~/.cache/huggingface/` by default, or `$HF_HOME` if set); this only happens once and requires network access. Use `semble clear` to remove cached data: `semble clear index` (saved indexes), `semble clear savings` (usage stats), `semble clear orphans` (indexes for repos no longer present on disk), or `semble clear all` (everything). diff --git a/src/semble/cli.py b/src/semble/cli.py index 466b42ef..1ba51b29 100644 --- a/src/semble/cli.py +++ b/src/semble/cli.py @@ -2,6 +2,7 @@ import asyncio import io import json +import logging import re import sys import warnings @@ -207,7 +208,24 @@ def _run_clear(clear_type: _CLEAR_CHOICE) -> None: _clear_orphans(cache_folder) +class _CliLogHandler(logging.StreamHandler): + """stderr handler owned by the CLI; setup is idempotent on this type, not on foreign handlers.""" + + +def _configure_cli_logging() -> None: + """Surface semble warnings (e.g. skipped oversized files) on stderr without touching the root logger.""" + package_logger = logging.getLogger("semble") + if any(isinstance(handler, _CliLogHandler) for handler in package_logger.handlers): + return + handler = _CliLogHandler(sys.stderr) + handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) + package_logger.addHandler(handler) + if package_logger.level == logging.NOTSET: + package_logger.setLevel(logging.WARNING) + + def _cli_main() -> None: + _configure_cli_logging() parser = argparse.ArgumentParser(prog="semble") parser.add_argument("-V", "--version", action="version", version=__version__) sub = parser.add_subparsers(dest="command") diff --git a/src/semble/index/create.py b/src/semble/index/create.py index 8354ea57..c2c8a80d 100644 --- a/src/semble/index/create.py +++ b/src/semble/index/create.py @@ -1,4 +1,5 @@ import contextlib +import logging from collections.abc import Sequence from pathlib import Path @@ -15,6 +16,7 @@ detect_language, get_extensions, get_file_status, + get_max_file_bytes, read_file_text, ) from semble.index.sparse import enrich_for_bm25 @@ -22,6 +24,21 @@ from semble.tokens import tokenize from semble.types import Chunk, ContentType, EmbeddingMatrix +logger = logging.getLogger(__name__) + + +def _warn_skipped_large(skipped_large: list[str]) -> None: + """Warn about files skipped for exceeding the maximum indexable file size.""" + if skipped_large: + logger.warning( + "Skipped %d file(s) exceeding the maximum file size of %d bytes " + "(raise SEMBLE_MAX_FILE_BYTES to include them): %s%s", + len(skipped_large), + get_max_file_bytes(), + ", ".join(skipped_large[:5]), + " ..." if len(skipped_large) > 5 else "", + ) + def _reindex_file( bm25_index: BM25, @@ -79,10 +96,14 @@ def create_index_from_path( manifest: dict[str, FileManifestEntry] = {} embedding_parts: list[tuple[int, int, int]] = [] + skipped_large: list[str] = [] + for file_path in walk_files(path, resolved_extensions): language = detect_language(file_path) with contextlib.suppress(OSError): file_status = get_file_status(file_path, None) + if file_status is FileStatus.TOO_LARGE: + skipped_large.append(str(file_path)) if file_status != FileStatus.VALID: continue @@ -109,6 +130,8 @@ def create_index_from_path( for indexed_path in previous_manifest.keys() - manifest.keys(): _reindex_file(bm25_index, indexed_path, [], previous_manifest[indexed_path]) + _warn_skipped_large(skipped_large) + if not chunks: raise ValueError(f"No supported files found under {path}.") diff --git a/src/semble/index/files.py b/src/semble/index/files.py index 7aa0702b..e3f4f941 100644 --- a/src/semble/index/files.py +++ b/src/semble/index/files.py @@ -1,3 +1,5 @@ +import logging +import os from collections import defaultdict from collections.abc import Sequence from enum import Enum @@ -5,8 +7,10 @@ from semble.types import ContentType -_MAX_FILE_BYTES = 1_000_000 # 1 MB max file size to read and index +_DEFAULT_MAX_FILE_BYTES = 1_000_000 # Default 1 MB max file size to read and index _EMPTY_FILE_BYTES = 128 + +logger = logging.getLogger(__name__) _EXTENSION_TO_LANGUAGE = { ".4th": "forth", ".ada": "ada", @@ -488,6 +492,25 @@ def read_file_text(file_path: Path) -> str: return file_path.read_text(encoding="utf-8", errors="replace") +def get_max_file_bytes() -> int: + """Resolve the maximum file size to index from SEMBLE_MAX_FILE_BYTES, falling back to the default. + + Malformed or nonpositive values warn and fall back to the default rather than crash indexing. + """ + raw = os.environ.get("SEMBLE_MAX_FILE_BYTES") + if raw is None: + return _DEFAULT_MAX_FILE_BYTES + try: + value = int(raw) + except ValueError: + logger.warning("Invalid SEMBLE_MAX_FILE_BYTES %r, using the default of %d bytes", raw, _DEFAULT_MAX_FILE_BYTES) + return _DEFAULT_MAX_FILE_BYTES + if value <= 0: + logger.warning("SEMBLE_MAX_FILE_BYTES must be positive, using the default of %d bytes", _DEFAULT_MAX_FILE_BYTES) + return _DEFAULT_MAX_FILE_BYTES + return value + + def get_file_status(file_path: Path, write_time: float | None) -> FileStatus: """Checks if a file should be indexed based on its size and modification time.""" stat = file_path.stat() @@ -495,7 +518,7 @@ def get_file_status(file_path: Path, write_time: float | None) -> FileStatus: # Index invalid, file invalid return FileStatus.NEWER size = stat.st_size - if size > _MAX_FILE_BYTES: + if size > get_max_file_bytes(): # index valid, file invalid return FileStatus.TOO_LARGE if size < _EMPTY_FILE_BYTES and not read_file_text(file_path).strip(): diff --git a/tests/index/test_index.py b/tests/index/test_index.py index 0f20c663..e485c89b 100644 --- a/tests/index/test_index.py +++ b/tests/index/test_index.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from typing import Any from unittest.mock import MagicMock, patch @@ -8,7 +9,7 @@ from semble import SembleIndex from semble.index.create import create_index_from_path -from semble.index.files import _MAX_FILE_BYTES, FileStatus, get_file_status +from semble.index.files import _DEFAULT_MAX_FILE_BYTES, FileStatus, get_file_status, get_max_file_bytes from semble.types import ContentType from tests.conftest import make_chunk @@ -69,11 +70,45 @@ def test_index_empty_returns_zero_chunks(mock_model: StaticModel, tmp_path: Path create_index_from_path(tmp_path, mock_model) -def test_oversized_file_is_skipped(mock_model: StaticModel, tmp_path: Path) -> None: - """Files exceeding _MAX_FILE_BYTES are silently skipped during indexing.""" - (tmp_path / "big.py").write_bytes(b"x" * (_MAX_FILE_BYTES + 1)) - with pytest.raises(ValueError): # no indexable content remains - create_index_from_path(tmp_path, mock_model) +def test_max_file_bytes_resolution(monkeypatch: pytest.MonkeyPatch) -> None: + """The limit resolves from SEMBLE_MAX_FILE_BYTES, falling back to the 1 MB default.""" + monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False) + assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES + monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", str(_DEFAULT_MAX_FILE_BYTES + 5)) + assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES + 5 + + +def test_max_file_bytes_invalid_values_fall_back(monkeypatch: pytest.MonkeyPatch) -> None: + """Malformed or nonpositive SEMBLE_MAX_FILE_BYTES values fall back to the default.""" + monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False) + assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES + for bad in ("not-a-number", "0", "-5"): + monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", bad) + assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES + + +def test_oversized_file_is_skipped_with_warning( + mock_model: StaticModel, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Files exceeding the limit are skipped during indexing, with a warning naming them.""" + monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False) + (tmp_path / "big.py").write_bytes(b"x" * (_DEFAULT_MAX_FILE_BYTES + 1)) + with caplog.at_level(logging.WARNING, logger="semble.index.create"): + with pytest.raises(ValueError): # no indexable content remains + create_index_from_path(tmp_path, mock_model) + assert "big.py" in caplog.text + assert str(_DEFAULT_MAX_FILE_BYTES) in caplog.text + assert "SEMBLE_MAX_FILE_BYTES" in caplog.text + + +def test_oversized_file_indexed_when_limit_raised( + mock_model: StaticModel, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Raising SEMBLE_MAX_FILE_BYTES lets oversized files into the index.""" + monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", str(_DEFAULT_MAX_FILE_BYTES + 1024)) + (tmp_path / "big.py").write_bytes(b"x = 1\n" + b"#" * _DEFAULT_MAX_FILE_BYTES) + _, _, chunks, _ = create_index_from_path(tmp_path, mock_model) + assert any(chunk.file_path.endswith("big.py") for chunk in chunks) def test_tiny_invalid_utf8_file_status_does_not_crash(tmp_path: Path) -> None: