diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index 5e6b7f70..21f90460 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -64,6 +64,9 @@ def __init__( self.source = source self.config = config self.schema = schema + # Set to True when no token limit is known for the configured model and + # the 8192 fallback is used; see _create_llm. + self.model_tokens_defaulted = False self.llm_model = self._create_llm(config["llm"]) self.verbose = False if config is None else config.get("verbose", False) self.headless = True if self.config is None else config.get("headless", True) @@ -218,6 +221,12 @@ def _create_llm(self, llm_config: dict) -> object: llm_params["model"], ) self.model_token = 8192 + # A silent 8192 window truncates long pages and changes the + # answer without failing. The log line alone is not reachable + # from the returned object, so callers that batch runs (or + # capture stdout only) have no way to tell a real limit from + # the fallback. Record it so it can be asserted on. + self.model_tokens_defaulted = True else: self.model_token = llm_params["model_tokens"] diff --git a/tests/graphs/abstract_graph_test.py b/tests/graphs/abstract_graph_test.py index adf4fc04..6b57666a 100644 --- a/tests/graphs/abstract_graph_test.py +++ b/tests/graphs/abstract_graph_test.py @@ -32,6 +32,61 @@ def test_llm_missing_tokens(monkeypatch, capsys): assert "Max input tokens for model" in captured +def test_llm_missing_tokens_sets_defaulted_flag(monkeypatch): + """An unknown model must record that the 8192 window is a fallback. + + The warning is only a log line, so a caller reading ``model_token`` sees + 8192 and cannot tell whether that is the model's real limit or the + default. A silent 8192 window truncates long pages and changes the answer + without failing, which is the failure mode reported in #1121. + """ + from scrapegraphai.graphs import abstract_graph + + monkeypatch.setattr( + abstract_graph, "models_tokens", {"openai": {"gpt-3.5-turbo": 4096}} + ) + llm_config = {"model": "openai/not-known-model", "openai_api_key": "test"} + with patch.object(TestGraph, "_create_graph", return_value=Mock(nodes=[])): + graph = TestGraph("Test prompt", {"llm": llm_config}) + + assert graph.model_token == 8192 + assert graph.model_tokens_defaulted is True + + +def test_known_model_does_not_set_defaulted_flag(monkeypatch): + """A model with a known limit must not be flagged as defaulted.""" + from scrapegraphai.graphs import abstract_graph + + monkeypatch.setattr( + abstract_graph, "models_tokens", {"openai": {"gpt-3.5-turbo": 4096}} + ) + llm_config = {"model": "openai/gpt-3.5-turbo", "openai_api_key": "test"} + with patch.object(TestGraph, "_create_graph", return_value=Mock(nodes=[])): + graph = TestGraph("Test prompt", {"llm": llm_config}) + + assert graph.model_token == 4096 + assert graph.model_tokens_defaulted is False + + +def test_explicit_model_tokens_does_not_set_defaulted_flag(monkeypatch): + """An explicit model_tokens is authoritative, not a fallback.""" + from scrapegraphai.graphs import abstract_graph + + monkeypatch.setattr( + abstract_graph, "models_tokens", {"openai": {"gpt-3.5-turbo": 4096}} + ) + llm_config = { + "model": "openai/not-known-model", + "openai_api_key": "test", + "model_tokens": 1000000, + } + with patch.object(TestGraph, "_create_graph", return_value=Mock(nodes=[])): + graph = TestGraph("Test prompt", {"llm": llm_config}) + + assert graph.model_token == 1000000 + assert graph.model_tokens_defaulted is False + + def test_burr_kwargs(): """Test that burr_kwargs configuration correctly sets use_burr and burr_config on the graph.""" dummy_graph = Mock()