Make max file size configurable via SEMBLE_MAX_FILE_BYTES and warn on skipped files

README.md

@@ -174,6 +174,8 @@ Savings are calculated as follows: for each call, semble records the total chara
 
 By default, your Semble savings statistics and any saved indexes are stored in the OS cache folder (`~/Library/Caches/semble/` on macOS, `~/.cache/semble/` on Linux, `%LOCALAPPDATA%\semble\Cache\` on Windows). To override this location you can supply an environment variable `SEMBLE_CACHE_LOCATION` which should be the full path to the target cache location e.g. `~/my-folder/my-caches/semble`.
 
+Files larger than 1 MB are skipped during indexing to keep index builds lean. Skipped files are reported as a warning at index time. If you work with large generated or ingested documents, you can raise (or lower) this limit with the `SEMBLE_MAX_FILE_BYTES` environment variable (in bytes).
+
 On first use, Semble also downloads the embedding model from Hugging Face and caches it in the standard Hugging Face cache (`~/.cache/huggingface/` by default, or `$HF_HOME` if set); this only happens once and requires network access.
 
 Use `semble clear` to remove cached data: `semble clear index` (saved indexes), `semble clear savings` (usage stats), `semble clear orphans` (indexes for repos no longer present on disk), or `semble clear all` (everything).

src/semble/cli.py

@@ -2,6 +2,7 @@
 import asyncio
 import io
 import json
+import logging
 import re
 import sys
 import warnings
@@ -207,7 +208,24 @@ def _run_clear(clear_type: _CLEAR_CHOICE) -> None:
         _clear_orphans(cache_folder)
 
 
+class _CliLogHandler(logging.StreamHandler):
+    """stderr handler owned by the CLI; setup is idempotent on this type, not on foreign handlers."""
+
+
+def _configure_cli_logging() -> None:
+    """Surface semble warnings (e.g. skipped oversized files) on stderr without touching the root logger."""
+    package_logger = logging.getLogger("semble")
+    if any(isinstance(handler, _CliLogHandler) for handler in package_logger.handlers):
+        return
+    handler = _CliLogHandler(sys.stderr)
+    handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
+    package_logger.addHandler(handler)
+    if package_logger.level == logging.NOTSET:
+        package_logger.setLevel(logging.WARNING)
+
+
 def _cli_main() -> None:
+    _configure_cli_logging()
     parser = argparse.ArgumentParser(prog="semble")
     parser.add_argument("-V", "--version", action="version", version=__version__)
     sub = parser.add_subparsers(dest="command")

src/semble/index/create.py

@@ -1,4 +1,5 @@
 import contextlib
+import logging
 from collections.abc import Sequence
 from pathlib import Path
 
@@ -15,13 +16,29 @@
     detect_language,
     get_extensions,
     get_file_status,
+    get_max_file_bytes,
     read_file_text,
 )
 from semble.index.sparse import enrich_for_bm25
 from semble.index.types import FileManifestEntry, PreviousIndex, make_chunk_id
 from semble.tokens import tokenize
 from semble.types import Chunk, ContentType, EmbeddingMatrix
 
+logger = logging.getLogger(__name__)
+
+
+def _warn_skipped_large(skipped_large: list[str]) -> None:
+    """Warn about files skipped for exceeding the maximum indexable file size."""
+    if skipped_large:
+        logger.warning(
+            "Skipped %d file(s) exceeding the maximum file size of %d bytes "
+            "(raise SEMBLE_MAX_FILE_BYTES to include them): %s%s",
+            len(skipped_large),
+            get_max_file_bytes(),
+            ", ".join(skipped_large[:5]),
+            " ..." if len(skipped_large) > 5 else "",
+        )
+
 
 def _reindex_file(
     bm25_index: BM25,
@@ -79,10 +96,14 @@ def create_index_from_path(
     manifest: dict[str, FileManifestEntry] = {}
     embedding_parts: list[tuple[int, int, int]] = []
 
+    skipped_large: list[str] = []
+
     for file_path in walk_files(path, resolved_extensions):
         language = detect_language(file_path)
         with contextlib.suppress(OSError):
             file_status = get_file_status(file_path, None)
+            if file_status is FileStatus.TOO_LARGE:
+                skipped_large.append(str(file_path))
             if file_status != FileStatus.VALID:
                 continue
 
@@ -109,6 +130,8 @@ def create_index_from_path(
     for indexed_path in previous_manifest.keys() - manifest.keys():
         _reindex_file(bm25_index, indexed_path, [], previous_manifest[indexed_path])
 
+    _warn_skipped_large(skipped_large)
+
     if not chunks:
         raise ValueError(f"No supported files found under {path}.")
 

src/semble/index/files.py

@@ -1,12 +1,16 @@
+import logging
+import os
 from collections import defaultdict
 from collections.abc import Sequence
 from enum import Enum
 from pathlib import Path
 
 from semble.types import ContentType
 
-_MAX_FILE_BYTES = 1_000_000  # 1 MB max file size to read and index
+_DEFAULT_MAX_FILE_BYTES = 1_000_000  # Default 1 MB max file size to read and index
 _EMPTY_FILE_BYTES = 128
+
+logger = logging.getLogger(__name__)
 _EXTENSION_TO_LANGUAGE = {
     ".4th": "forth",
     ".ada": "ada",
@@ -488,14 +492,33 @@ def read_file_text(file_path: Path) -> str:
     return file_path.read_text(encoding="utf-8", errors="replace")
 
 
+def get_max_file_bytes() -> int:
+    """Resolve the maximum file size to index from SEMBLE_MAX_FILE_BYTES, falling back to the default.
+
+    Malformed or nonpositive values warn and fall back to the default rather than crash indexing.
+    """
+    raw = os.environ.get("SEMBLE_MAX_FILE_BYTES")
+    if raw is None:
+        return _DEFAULT_MAX_FILE_BYTES
+    try:
+        value = int(raw)
+    except ValueError:
+        logger.warning("Invalid SEMBLE_MAX_FILE_BYTES %r, using the default of %d bytes", raw, _DEFAULT_MAX_FILE_BYTES)
+        return _DEFAULT_MAX_FILE_BYTES
+    if value <= 0:
+        logger.warning("SEMBLE_MAX_FILE_BYTES must be positive, using the default of %d bytes", _DEFAULT_MAX_FILE_BYTES)
+        return _DEFAULT_MAX_FILE_BYTES
+    return value
+
+
 def get_file_status(file_path: Path, write_time: float | None) -> FileStatus:
     """Checks if a file should be indexed based on its size and modification time."""
     stat = file_path.stat()
     if write_time is not None and stat.st_mtime > write_time:
         # Index invalid, file invalid
         return FileStatus.NEWER
     size = stat.st_size
-    if size > _MAX_FILE_BYTES:
+    if size > get_max_file_bytes():
         # index valid, file invalid
         return FileStatus.TOO_LARGE
     if size < _EMPTY_FILE_BYTES and not read_file_text(file_path).strip():

tests/index/test_index.py

@@ -1,3 +1,4 @@
+import logging
 from pathlib import Path
 from typing import Any
 from unittest.mock import MagicMock, patch
@@ -8,7 +9,7 @@
 
 from semble import SembleIndex
 from semble.index.create import create_index_from_path
-from semble.index.files import _MAX_FILE_BYTES, FileStatus, get_file_status
+from semble.index.files import _DEFAULT_MAX_FILE_BYTES, FileStatus, get_file_status, get_max_file_bytes
 from semble.types import ContentType
 from tests.conftest import make_chunk
 
@@ -69,11 +70,45 @@ def test_index_empty_returns_zero_chunks(mock_model: StaticModel, tmp_path: Path
         create_index_from_path(tmp_path, mock_model)
 
 
-def test_oversized_file_is_skipped(mock_model: StaticModel, tmp_path: Path) -> None:
-    """Files exceeding _MAX_FILE_BYTES are silently skipped during indexing."""
-    (tmp_path / "big.py").write_bytes(b"x" * (_MAX_FILE_BYTES + 1))
-    with pytest.raises(ValueError):  # no indexable content remains
-        create_index_from_path(tmp_path, mock_model)
+def test_max_file_bytes_resolution(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The limit resolves from SEMBLE_MAX_FILE_BYTES, falling back to the 1 MB default."""
+    monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False)
+    assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES
+    monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", str(_DEFAULT_MAX_FILE_BYTES + 5))
+    assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES + 5
+
+
+def test_max_file_bytes_invalid_values_fall_back(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Malformed or nonpositive SEMBLE_MAX_FILE_BYTES values fall back to the default."""
+    monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False)
+    assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES
+    for bad in ("not-a-number", "0", "-5"):
+        monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", bad)
+        assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES
+
+
+def test_oversized_file_is_skipped_with_warning(
+    mock_model: StaticModel, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    """Files exceeding the limit are skipped during indexing, with a warning naming them."""
+    monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False)
+    (tmp_path / "big.py").write_bytes(b"x" * (_DEFAULT_MAX_FILE_BYTES + 1))
+    with caplog.at_level(logging.WARNING, logger="semble.index.create"):
+        with pytest.raises(ValueError):  # no indexable content remains
+            create_index_from_path(tmp_path, mock_model)
+    assert "big.py" in caplog.text
+    assert str(_DEFAULT_MAX_FILE_BYTES) in caplog.text
+    assert "SEMBLE_MAX_FILE_BYTES" in caplog.text
+
+
+def test_oversized_file_indexed_when_limit_raised(
+    mock_model: StaticModel, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Raising SEMBLE_MAX_FILE_BYTES lets oversized files into the index."""
+    monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", str(_DEFAULT_MAX_FILE_BYTES + 1024))
+    (tmp_path / "big.py").write_bytes(b"x = 1\n" + b"#" * _DEFAULT_MAX_FILE_BYTES)
+    _, _, chunks, _ = create_index_from_path(tmp_path, mock_model)
+    assert any(chunk.file_path.endswith("big.py") for chunk in chunks)
 
 
 def test_tiny_invalid_utf8_file_status_does_not_crash(tmp_path: Path) -> None: