diff --git a/.env.example b/.env.example index 31093441..624d64f3 100644 --- a/.env.example +++ b/.env.example @@ -237,14 +237,25 @@ EVA_MODEL_LIST='[ #v EVA_MODEL__TURN_START_STRATEGY_PARAMS='{}' #i Turn stop strategy: when to consider the user has finished speaking. +#i krisp_viva_turn uses Krisp's VIVA SDK turn detection v3 (streaming, frame-by-frame). +#i It needs the proprietary krisp_audio SDK installed plus KRISP_VIVA_TURN_MODEL_PATH +#i (path to a .kef model file) and KRISP_VIVA_API_KEY set below. #d enum -#e turn_analyzer,speech_timeout,external +#e turn_analyzer,speech_timeout,krisp_viva_turn,external #v EVA_MODEL__TURN_STOP_STRATEGY=turn_analyzer #i Turn stop strategy parameters. For speech_timeout: {"user_speech_timeout": 0.8}. +#i For krisp_viva_turn: {"threshold": 0.5, "frame_duration_ms": 20} (model_path/api_key +#i can also be passed here to override the env vars). #d json_object #v EVA_MODEL__TURN_STOP_STRATEGY_PARAMS='{}' +#i Krisp VIVA SDK credentials (only used when TURN_STOP_STRATEGY=krisp_viva_turn). +#i The turn model ships in the Krisp VIVA "Turn-Taking models" download (krisp-viva-tt-models); +#i point at the krisp-viva-tp-v3.kef file. VAD is supplied by Silero, so no Krisp VAD model needed. +#v KRISP_VIVA_TURN_MODEL_PATH=vendor/krisp/krisp-viva-tp-v3.kef +#v KRISP_VIVA_API_KEY=your_krisp_license_key + #i VAD (Voice Activity Detection) analyzer. #d enum #e silero,none diff --git a/.gitignore b/.gitignore index e9ef7669..bf3094b3 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,7 @@ scripts/extract_and_generate_leaderboard.py # Website data generation scripts (local only) website/scripts/ + +# Krisp VIVA SDK — licensed binaries, provisioned locally before Docker build (not public) +vendor/krisp/*.whl +vendor/krisp/*.kef diff --git a/Dockerfile b/Dockerfile index 8fe229a2..2f6349f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,15 +62,23 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ && rm -rf /var/lib/apt/lists/* +# Copy uv to allow for `uv pip install --python /opt/venv/bin/python` (bare `pip install` would miss the venv). +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + # Copy deps venv separately so it stays cached when only source changes COPY --from=deps /opt/venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" # Overlay only the eva package files that changed (tiny, ~seconds to copy) COPY --from=builder /opt/venv/lib/python3.11/site-packages/eva /opt/venv/lib/python3.11/site-packages/eva COPY --from=builder /opt/venv/lib/python3.11/site-packages/eva-*.dist-info /opt/venv/lib/python3.11/site-packages/ COPY --from=builder /opt/venv/bin/eva /opt/venv/bin/eva +# Copy `scripts/docker_eva_wrapper.sh` as `/usr/local/bin/eva`, which precedes the real `/opt/venv/bin/eva` in `PATH`. +# That wrapper tries to install Krisp on every invocation of `eva`. +COPY scripts/docker_eva_wrapper.sh /usr/local/bin/eva +RUN chmod +x /usr/local/bin/eva +ENV PATH="/usr/local/bin:/opt/venv/bin:$PATH" + # Copy application code COPY src/ ./src/ COPY scripts/ ./scripts/ @@ -85,6 +93,9 @@ RUN groupadd --gid 1000 eva && \ # Create directory for output with correct ownership RUN mkdir -p /app/output && chown eva:eva /app/output +# /opt/venv must be writable by the runtime user to install the Krisp SDK into it at container start. +RUN chown -R eva:eva /opt/venv + # Python runtime settings ENV PYTHONPATH="/app/src" ENV PYTHONUNBUFFERED=1 diff --git a/compose.yaml b/compose.yaml index 56227564..988a0729 100644 --- a/compose.yaml +++ b/compose.yaml @@ -24,6 +24,8 @@ services: - ./output:/app/output # Mount configs for easy customization (includes prompts) - ./configs:/app/configs:ro + # Krisp SDK and models + - ./vendor/krisp:/app/vendor/krisp:ro # Resource limits (adjust based on your system) deploy: diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh new file mode 100755 index 00000000..5fe8bf11 --- /dev/null +++ b/scripts/docker_build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +IMAGE="${1:?Missing image tag. Usage: $0 [--push]}" +PUSH="${2:-}" +shift 2>/dev/null || true + +GIT_COMMIT_SHA=$(git rev-parse HEAD) +GIT_BRANCH=$(git branch --show-current) +GIT_DIRTY=$([[ -n $(git status --porcelain) ]] && echo true || echo false) +GIT_DIFF_HASH=$(git diff | shasum -a 256 | cut -c1-12) + +docker build \ + --platform linux/amd64 \ + --build-arg GIT_COMMIT_SHA="$GIT_COMMIT_SHA" \ + --build-arg GIT_BRANCH="$GIT_BRANCH" \ + --build-arg GIT_DIRTY="$GIT_DIRTY" \ + --build-arg GIT_DIFF_HASH="$GIT_DIFF_HASH" \ + -t "$IMAGE" \ + "$@" . + +if [[ "$PUSH" == "--push" ]]; then + docker push "$IMAGE" +fi diff --git a/scripts/docker_eva_wrapper.sh b/scripts/docker_eva_wrapper.sh new file mode 100755 index 00000000..413cfcc8 --- /dev/null +++ b/scripts/docker_eva_wrapper.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Installed as /usr/local/bin/eva, which comes first on PATH, shadowing the real +# entry point at /opt/venv/bin/eva. Runs before every `eva` invocation, regardless of +# how the container is launched — Docker ENTRYPOINT, `sh -c 'eva ... && eva ...'`, or +# a Toolkit job's custom `command` (which replaces the image's ENTRYPOINT outright, so +# hooking this at the container level instead of here would silently miss those). +# +# Krisp VIVA SDK is proprietary and can't be baked into the image at build time +# (see vendor/krisp/README.md), so it's installed here at container start instead, +# from whatever is bind-mounted at /app/vendor/krisp. It's optional: with no wheel +# present, eva still runs fine, just without the krisp_viva_turn strategy. +set -e + +whl_count=$(ls /app/vendor/krisp/*.whl 2>/dev/null | wc -l) +if [ "$whl_count" -gt 1 ]; then + echo "Expected at most one Krisp wheel in vendor/krisp/, found $whl_count" >&2 + exit 1 +elif [ "$whl_count" -eq 1 ]; then + echo "Installing Krisp VIVA SDK..." + uv pip install --python /opt/venv/bin/python --no-cache /app/vendor/krisp/*.whl +else + echo "No Krisp wheel in vendor/krisp/ — krisp_viva_turn will be unavailable" +fi + +# Full path, not bare "eva": avoids re-resolving PATH back to this wrapper, and keeps +# argparse's default program name in --help/usage/error output as "eva". +exec /opt/venv/bin/eva "$@" diff --git a/src/eva/assistant/pipeline/turn_config.py b/src/eva/assistant/pipeline/turn_config.py index e4ec3083..7208f638 100644 --- a/src/eva/assistant/pipeline/turn_config.py +++ b/src/eva/assistant/pipeline/turn_config.py @@ -94,7 +94,7 @@ def create_turn_stop_strategy( """Create a user turn stop strategy from configuration. Args: - strategy_type: Strategy type ('speech_timeout', 'turn_analyzer', 'external') + strategy_type: Strategy type ('speech_timeout', 'turn_analyzer', 'krisp_viva_turn', 'external') strategy_params: Strategy-specific parameters smart_turn_stop_secs: stop_secs for SmartTurnParams (used with turn_analyzer strategy) @@ -117,11 +117,26 @@ def create_turn_stop_strategy( smart_params = SmartTurnParams(stop_secs=stop_secs) if stop_secs is not None else None turn_analyzer = LocalSmartTurnAnalyzerV3(params=smart_params) return TurnAnalyzerUserTurnStopStrategy(turn_analyzer=turn_analyzer, **params) + elif strategy_type_lower == "krisp_viva_turn": + # KrispVivaTurn uses Krisp's VIVA SDK turn detection v3 (Tt) API, processing audio + # frame-by-frame in real time with an external VAD flag. It depends on the proprietary + # krisp_audio SDK plus a .kef model file (KRISP_VIVA_TURN_MODEL_PATH) and license key + # (KRISP_VIVA_API_KEY), so import lazily to keep the SDK optional. + from pipecat.audio.turn.krisp_viva_turn import KrispTurnParams, KrispVivaTurn + + params = dict(strategy_params) + # Krisp analyzer tuning params; fall back to KrispTurnParams defaults when absent. + krisp_kwargs = {k: params.pop(k) for k in ("threshold", "frame_duration_ms") if k in params} + krisp_params = KrispTurnParams(**krisp_kwargs) if krisp_kwargs else None + # Constructor kwargs; model_path/api_key fall back to their env vars when omitted. + analyzer_kwargs = {k: params.pop(k) for k in ("model_path", "api_key", "sample_rate") if k in params} + krisp_analyzer = KrispVivaTurn(params=krisp_params, **analyzer_kwargs) + return TurnAnalyzerUserTurnStopStrategy(turn_analyzer=krisp_analyzer, **params) elif strategy_type_lower == "external": # ExternalUserTurnStopStrategy has no required parameters return ExternalUserTurnStopStrategy(**strategy_params) else: raise ValueError( f"Unsupported turn stop strategy: {strategy_type}. " - f"Supported types: 'speech_timeout', 'turn_analyzer', 'external'" + f"Supported types: 'speech_timeout', 'turn_analyzer', 'krisp_viva_turn', 'external'" ) diff --git a/src/eva/models/config.py b/src/eva/models/config.py index 8bd9130b..057ce240 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -168,8 +168,10 @@ class ModelConfig(BaseModel): turn_stop_strategy: str = Field( "turn_analyzer", description=( - "User turn stop strategy: 'speech_timeout', 'turn_analyzer', or 'external'. " + "User turn stop strategy: 'speech_timeout', 'turn_analyzer', 'krisp_viva_turn', or 'external'. " "Defaults to 'turn_analyzer' (TurnAnalyzerUserTurnStopStrategy with LocalSmartTurnAnalyzerV3). " + "'krisp_viva_turn' uses Krisp's VIVA SDK (requires the krisp_audio SDK, " + "KRISP_VIVA_TURN_MODEL_PATH (for local dev), and KRISP_VIVA_API_KEY). " "Set via EVA_MODEL__TURN_STOP_STRATEGY." ), ) diff --git a/tests/unit/assistant/test_turn_config.py b/tests/unit/assistant/test_turn_config.py index 72e135a6..a0b1370e 100644 --- a/tests/unit/assistant/test_turn_config.py +++ b/tests/unit/assistant/test_turn_config.py @@ -226,5 +226,71 @@ def test_unsupported_strategy_raises(self): def test_unsupported_strategy_error_lists_supported(self): """ValueError message lists supported strategies.""" - with pytest.raises(ValueError, match="speech_timeout.*turn_analyzer.*external"): + with pytest.raises(ValueError, match="speech_timeout.*turn_analyzer.*krisp_viva_turn.*external"): create_turn_stop_strategy("unknown", {}) + + +# --------------------------------------------------------------------------- +# create_turn_stop_strategy — krisp_viva_turn +# --------------------------------------------------------------------------- + + +class TestCreateTurnStopStrategyKrispViva: + """Tests for the 'krisp_viva_turn' branch of create_turn_stop_strategy. + + KrispVivaTurn depends on the proprietary ``krisp_audio`` SDK, which is not a + project dependency. The factory imports ``pipecat.audio.turn.krisp_viva_turn`` + lazily, so these tests inject a fake module into ``sys.modules``. + """ + + @staticmethod + def _install_fake_krisp(monkeypatch): + """Inject a fake pipecat.audio.turn.krisp_viva_turn module and return it.""" + import sys + + fake_module = MagicMock() + fake_module.KrispVivaTurn.return_value = MagicMock() + monkeypatch.setitem(sys.modules, "pipecat.audio.turn.krisp_viva_turn", fake_module) + return fake_module + + def test_krisp_viva_turn_strategy(self, monkeypatch): + """'krisp_viva_turn' returns a TurnAnalyzerUserTurnStopStrategy with a KrispVivaTurn.""" + fake_module = self._install_fake_krisp(monkeypatch) + + result = create_turn_stop_strategy("krisp_viva_turn", {}) + + assert isinstance(result, TurnAnalyzerUserTurnStopStrategy) + # No tuning params given -> KrispTurnParams left at its defaults (params=None). + fake_module.KrispVivaTurn.assert_called_once_with(params=None) + fake_module.KrispTurnParams.assert_not_called() + + def test_krisp_viva_turn_case_insensitive(self, monkeypatch): + """Strategy type is matched case-insensitively.""" + self._install_fake_krisp(monkeypatch) + assert isinstance(create_turn_stop_strategy("KRISP_VIVA_TURN", {}), TurnAnalyzerUserTurnStopStrategy) + + def test_krisp_viva_turn_tuning_params(self, monkeypatch): + """Tuning params threshold / frame_duration_ms are forwarded to KrispTurnParams.""" + fake_module = self._install_fake_krisp(monkeypatch) + + create_turn_stop_strategy("krisp_viva_turn", {"threshold": 0.7, "frame_duration_ms": 30}) + + fake_module.KrispTurnParams.assert_called_once_with(threshold=0.7, frame_duration_ms=30) + # The constructed params object is passed to the analyzer. + assert fake_module.KrispVivaTurn.call_args.kwargs["params"] is fake_module.KrispTurnParams.return_value + + def test_krisp_viva_turn_constructor_params(self, monkeypatch): + """model_path / api_key / sample_rate are forwarded to the KrispVivaTurn constructor.""" + fake_module = self._install_fake_krisp(monkeypatch) + + create_turn_stop_strategy( + "krisp_viva_turn", + {"model_path": "/models/turn.kef", "api_key": "secret", "sample_rate": 16000}, + ) + + call_kwargs = fake_module.KrispVivaTurn.call_args.kwargs + assert call_kwargs["model_path"] == "/models/turn.kef" + assert call_kwargs["api_key"] == "secret" + assert call_kwargs["sample_rate"] == 16000 + # None of these should leak into the strategy as tuning params. + fake_module.KrispTurnParams.assert_not_called() diff --git a/vendor/krisp/README.md b/vendor/krisp/README.md new file mode 100644 index 00000000..96a01e38 --- /dev/null +++ b/vendor/krisp/README.md @@ -0,0 +1,43 @@ +# Krisp VIVA SDK — vendor files + +The `krisp_viva_turn` turn-stop strategy needs Krisp's **licensed** VIVA SDK, which +is **not** on PyPI and **not** public — it is downloaded per-account from the Krisp +developer portal. The binaries are therefore **git-ignored** (see `.gitignore`) and +must be placed here manually before running the container. + +## Download Krisp files + +1. Log in to https://developers.krisp.ai/versions +2. Download the VIVA Python SDK zip file as well as the model zip files +3. Extract the necessary files + ```sh + cd vendor/krisp + for file in ~/Downloads/krisp-viva-*-models*.zip; do unzip -jo "$file"; done + for file in ~/Downloads/krisp-viva-uar-python-sdk-*.zip; do unzip -jo "$file" '*/dist/krisp_audio-*-cp311-cp311-linux_x86_64.whl'; done + ``` + +Result (wheel version and `.kef` set will vary by download): + +``` +vendor/krisp/ +├── README.md (tracked) +├── krisp_audio-1.11.0-cp311-cp311-linux_x86_64.whl (git-ignored) +└── krisp-viva-tp-v3.kef (git-ignored) +``` + +> [!WARNING] +> At most one `.whl` may be present here. The `eva` wrapper skips installing Krisp if it doesn't find one, and it errors out if it finds more than one, since it wouldn't know which to install. + +## How it's used + +In the Docker container: +- This directory is mounted to `/app/vendor/krisp`. +- The `eva` command (which is actually `scripts/docker_eva_wrapper.sh`) installs the `.whl`, if present. +- The `KRISP_VIVA_TURN_MODEL_PATH` env var points at the `.kef` file directly on that mount, e.g. `vendor/krisp/krisp-viva-tp-v3.kef`. +- The `KRISP_VIVA_API_KEY` env var provides the API key. + +## Runtime requirement + +The SDK validates its license against `sdkapi.krisp.ai` and reports usage to +`analytics.krisp.ai` at runtime. The container must have outbound HTTPS to both, or +license validation fails.