diff --git a/CHANGELOG.md b/CHANGELOG.md index 629b645a5..c1249d9fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@e3546c794 + ## [0.3.34] - feat: update llama.cpp to ggml-org/llama.cpp@e3546c794 diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index b5bffd46b..845650a09 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -114,6 +114,9 @@ def __init__( # KV cache quantization type_k: Optional[int] = None, type_v: Optional[int] = None, + # Progress callback + progress_callback: Optional[Callable[[float, Any], bool]] = None, + progress_callback_user_data: Optional[Any] = None, # Misc spm_infill: bool = False, verbose: bool = True, @@ -190,6 +193,8 @@ def __init__( verbose: Print verbose output to stderr. type_k: KV cache data type for K (default: f16) type_v: KV cache data type for V (default: f16) + progress_callback: Optional callback invoked during model loading. Signature: callback(progress: float, user_data: Any) -> bool, progress is a value between 0.0 and 1.0. Returning False aborts model loading + progress_callback_user_data: Optional context pointer passed to progress_callback. spm_infill: Use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this. Raises: @@ -247,6 +252,17 @@ def __init__( self.model_params.use_mmap = use_mmap if lora_path is None else False self.model_params.use_mlock = use_mlock + # Progress callback + if progress_callback is not None: + self._progress_callback = llama_cpp.llama_progress_callback( + progress_callback + ) + self.model_params.progress_callback = self._progress_callback + if progress_callback_user_data is not None: + self.model_params.progress_callback_user_data = ( + progress_callback_user_data + ) + # kv_overrides is the original python dict self.kv_overrides = kv_overrides if kv_overrides is not None: