diff --git a/.env b/.env new file mode 100644 index 0000000..1378873 --- /dev/null +++ b/.env @@ -0,0 +1,61 @@ +# OpenAI-compatible LLM settings (文本对话 - DeepSeek) +LLM_API_KEY=your_llm_api_key_here +LLM_MODEL=deepseek-chat +LLM_BASE_URL=https://api.deepseek.com +LLM_TIMEOUT=60 +# Optional: extra fields merged into every request body (JSON object) +# Example: LLM_EXTRA_BODY={"chat_template_kwargs": {"enable_thinking": false}} +LLM_EXTRA_BODY= + +# MiniMax Vision - 图像描述专用 +VISION_API_KEY=your_vision_api_key_here +VISION_MODEL=qwen-vl-plus +VISION_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 + +# Lightweight orchestration max_tokens budget +# Used by the decision layer and short roleplay replies +ECHOBOT_LIGHTWEIGHT_MAX_TOKENS=4096 + +# Single-turn agent tool/skill loop step limit +ECHOBOT_AGENT_MAX_STEPS=50 + +# Whether to send a short "I'm checking" acknowledgement before delegating +# to the full background agent +ECHOBOT_DELEGATED_ACK_ENABLED=true + +# Runtime logging +REME_LOG_LEVEL=WARNING +AGENTSCOPE_LOG_LEVEL=WARNING + +# Web Live2D +ECHOBOT_WEB_LIVE2D_MODEL= + +# Web TTS +ECHOBOT_TTS_KOKORO_AUTO_DOWNLOAD=true +ECHOBOT_TTS_KOKORO_MODEL_DIR= +ECHOBOT_TTS_KOKORO_PROVIDER=cpu +ECHOBOT_TTS_KOKORO_NUM_THREADS=2 +ECHOBOT_TTS_KOKORO_DEFAULT_VOICE=zf_001 +ECHOBOT_TTS_KOKORO_DOWNLOAD_TIMEOUT_SECONDS=600 +ECHOBOT_TTS_KOKORO_URL= +ECHOBOT_TTS_KOKORO_LENGTH_SCALE=1.0 +ECHOBOT_TTS_KOKORO_LANG= + +# Web ASR / VAD +ECHOBOT_ASR_AUTO_DOWNLOAD=true +ECHOBOT_ASR_MODEL_DIR= +ECHOBOT_ASR_PROVIDER=cpu +ECHOBOT_ASR_NUM_THREADS=2 +ECHOBOT_ASR_LANGUAGE=auto +ECHOBOT_ASR_USE_ITN=false +ECHOBOT_ASR_SAMPLE_RATE=16000 +ECHOBOT_ASR_DOWNLOAD_TIMEOUT_SECONDS=600 +ECHOBOT_ASR_SENSEVOICE_URL= +ECHOBOT_ASR_VAD_URL= + +# Video Call Plugin +ECHOBOT_ENABLE_VIDEO_CALL=true +ECHOBOT_VIDEO_FRAME_RATE=0.25 +ECHOBOT_VIDEO_MAX_FRAME_SIZE=1280x720 +ECHOBOT_VIDEO_FACE_CONFIDENCE_THRESHOLD=0.8 +ECHOBOT_VIDEO_FEATURE_MATCH_THRESHOLD=0.6 diff --git a/.gitignore b/.gitignore index aef659d..b42af4d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,7 @@ htmlcov/ .hypothesis/ # Local environment files -.env +# .env # Committed with placeholder values - copy and fill in your own keys .env.* !.env.example @@ -43,3 +43,12 @@ example_projects/ .echobot logs/ + +# Node +node_modules/ +package-lock.json + +# Generated docs +VIDEO_CALL_FINAL_GUIDE.md +VIDEO_CALL_INTEGRATION_GUIDE.md +VIDEO_CALL_PLUGIN_SUMMARY.md diff --git a/VIDEO_CALL_README.md b/VIDEO_CALL_README.md new file mode 100644 index 0000000..76f39f8 --- /dev/null +++ b/VIDEO_CALL_README.md @@ -0,0 +1,153 @@ +# EchoBot 视频通话插件 - 架构与使用说明 + +## 功能概述 + +视频通话插件（`video_call`）为 EchoBot 提供实时视觉感知能力： + +- **实时摄像头画面处理**：通过 WebSocket 接收前端推送的视频帧 +- **图像语义描述**：调用视觉模型对每帧画面生成自然语言描述 +- **人脸识别与绑定**：用 InsightFace 提取 512 维特征向量，与已知人脸库做余弦相似度匹配 +- **视觉上下文自动注入**：每次对话时，coordinator 自动将当前视觉信息追加为临时系统消息，模型无需工具调用即可感知画面 +- **主动记忆陌生人**：检测到未识别人脸时，模型会主动询问姓名并通过工具绑定，重启后仍可识别 + +--- + +## 架构概览 + +``` +前端摄像头 + │ WebSocket /api/web/video/stream + ▼ +VisionProcessingService + ├── ImageDescriptionService # 视觉模型：图像 → 自然语言描述 + └── FaceRecognitionService # InsightFace：检测 + 512 维特征提取 + 余弦匹配 + │ + ▼ +VisionContextProvider（内存缓存，最近 80 帧） + │ + │ on_startup 注入 + ▼ +ConversationCoordinator._vision_context_provider + │ 每次对话自动调用 _build_vision_message() + ▼ +transient_system_messages → RoleplayEngine / AgentRunner + │ + ▼ +大模型（感知视觉 + 主动询问陌生人） + │ 用户回答名字 → decision engine 路由到 agent + ▼ +BindFaceToNameTool（BaseTool） + ├── FaceRecognitionService.add_known_face_from_frame() # 内存 + └── FaceFeatureInterceptor.add_or_update_feature() # 持久化到 .echobot/face_features.json +``` + +### 工具注入方式（非侵入） + +插件工具通过 `create_app.py` 里包装 `tool_registry_factory` 注入，不修改框架代码： + +```python +# create_app.py +def _plugin_context_builder(opts): + ctx = build_runtime_context(opts, load_session_state=False) + original_factory = ctx.tool_registry_factory + + def wrapped_factory(session_name, scheduled_context): + registry = original_factory(session_name, scheduled_context) + for tool in video_plugin.get_tool_instances(): + registry.register(tool) + return registry + + ctx.tool_registry_factory = wrapped_factory + return ctx +``` + +### 视觉上下文注入方式 + +插件在 `on_startup` 时调用： + +```python +coordinator.set_vision_context_provider(self.vision_provider) +``` + +Coordinator 在每次 `handle_user_turn_stream` 时自动调用 `_build_vision_message()`，将视觉信息作为 `transient_system_messages` 注入，**不写入会话历史**。 + +--- + +## 目录结构 + +``` +echobot/plugins/video_call/ +├── __init__.py # VideoCallPlugin：插件入口，on_startup/on_shutdown +├── models.py # 数据模型：Face, VisionContext +├── vision_provider.py # VisionContextProvider：帧缓存与合并 +├── interceptors/ +│ └── __init__.py # FaceFeatureInterceptor：特征持久化（JSON） +├── routers/ +│ └── __init__.py # API 路由：WebSocket 视频流、人脸绑定接口 +├── services/ +│ ├── __init__.py # VisionProcessingService：双链路并行处理 +│ ├── face_recognition.py # FaceRecognitionService：InsightFace 封装 +│ └── image_description.py # ImageDescriptionService：视觉描述 +└── tools/ + ├── __init__.py # VISION_TOOLS：旧格式工具定义（兼容用） + ├── face_tools.py # BaseTool 实现：bind_face_to_name / list_known_faces / forget_face + └── handlers.py # VisionToolHandler：工具调用处理器 +``` + +--- + +## 可用工具（Agent 路径） + +| 工具名 | 触发场景 | 功能 | +|--------|----------|------| +| `bind_face_to_name` | "我叫XXX" / "记住我" / "这是XXX" | 从当前摄像头帧提取人脸特征并绑定姓名 | +| `list_known_faces` | "你认识哪些人" | 列出所有已绑定的人脸姓名 | +| `forget_face` | "忘掉XXX" | 删除某人的人脸绑定记录 | + +--- + +## API 接口 + +| 方法 | 路径 | 说明 | +|------|------|------| +| WS | `/api/web/video/stream` | 视频帧推送（JPEG bytes） | +| GET | `/api/web/video/context` | 获取当前视觉上下文列表 | +| GET | `/api/web/video/snapshot` | 获取最新一帧快照 | +| POST | `/api/web/video/face-bind` | 通过特征向量绑定人脸 | +| POST | `/api/web/video/face-bind-frame` | 通过图像帧绑定人脸 | +| GET | `/api/web/video/face-list` | 查询已绑定人脸列表 | + +--- + +## 启用方式 + +在 `.env` 中设置： + +```env +ECHOBOT_ENABLE_VIDEO_CALL=true +``` + +前端摄像头页面：`http://localhost:8000/web/camera` + +--- + +## 人脸数据持久化 + +绑定的人脸特征向量（512 维，InsightFace buffalo_sc）保存在： + +``` +.echobot/face_features.json +``` + +重启后自动加载，无需重新绑定。 + +--- + +## 依赖 + +``` +insightface +onnxruntime +Pillow +numpy +``` diff --git a/echobot/app/create_app.py b/echobot/app/create_app.py index 86bea57..ea7be30 100644 --- a/echobot/app/create_app.py +++ b/echobot/app/create_app.py @@ -1,12 +1,23 @@ from __future__ import annotations +import os from contextlib import asynccontextmanager from pathlib import Path from fastapi import FastAPI from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles +from loguru import logger +# 加载 .env 文件 +try: + from dotenv import load_dotenv + load_dotenv() +except ImportError: + pass + +from ..plugins import PluginRegistry +from ..plugins.video_call import VideoCallPlugin from ..runtime.bootstrap import RuntimeOptions from .routers import chat, channels, cron, health, heartbeat, roles, sessions, web from .runtime import ASRServiceBuilder, AppRuntime, RuntimeContextBuilder, TTSServiceBuilder @@ -24,10 +35,47 @@ def create_app( asr_service_builder: ASRServiceBuilder | None = None, ) -> FastAPI: options = runtime_options or RuntimeOptions() + + # 初始化插件系统（先于 runtime 构建，以便注入工具） + plugin_registry = PluginRegistry() + video_plugin = VideoCallPlugin() + plugin_registry.register(video_plugin) + video_enabled = os.getenv("ECHOBOT_ENABLE_VIDEO_CALL", "false").lower() == "true" + if video_enabled: + plugin_registry.enable("video_call") + logger.info("VideoCall plugin enabled") + + # 构建包装过的 context_builder，注入插件工具 + def _plugin_context_builder(opts: RuntimeOptions): + from ..runtime.bootstrap import build_runtime_context + # 调用原始 builder（默认或用户传入） + if context_builder is not None: + ctx = context_builder(opts) + else: + ctx = build_runtime_context(opts, load_session_state=False) + + if video_enabled: + # 包装 tool_registry_factory，追加插件工具 + original_factory = ctx.tool_registry_factory + + def wrapped_factory(session_name: str, scheduled_context: bool): + registry = original_factory(session_name, scheduled_context) + if registry is None: + return registry + for tool in video_plugin.get_tool_instances(): + try: + registry.register(tool) + except ValueError: + pass # 已注册则跳过 + return registry + + ctx.tool_registry_factory = wrapped_factory + return ctx + runtime = AppRuntime( runtime_options=options, channel_config_path=channel_config_path, - context_builder=context_builder, + context_builder=_plugin_context_builder, tts_service_builder=tts_service_builder, asr_service_builder=asr_service_builder, ) @@ -36,9 +84,19 @@ def create_app( async def lifespan(app: FastAPI): await runtime.start() app.state.runtime = runtime + app.state.plugin_registry = plugin_registry + + # 启动所有启用的插件 + for plugin in plugin_registry.get_enabled_plugins(): + await plugin.on_startup(app, runtime) + try: yield finally: + # 关闭所有插件 + for plugin in plugin_registry.get_enabled_plugins(): + await plugin.on_shutdown() + await runtime.stop() app = FastAPI( @@ -58,6 +116,10 @@ async def root() -> dict[str, str]: async def web_console() -> FileResponse: return FileResponse(WEB_ASSETS_DIR / "index.html") + @app.get("/web/camera", include_in_schema=False) + async def web_camera() -> FileResponse: + return FileResponse(WEB_ASSETS_DIR / "camera.html") + @app.get("/favicon.ico", include_in_schema=False) async def favicon() -> FileResponse: return FileResponse( @@ -79,4 +141,11 @@ async def favicon() -> FileResponse: app.include_router(roles.router, prefix="/api") app.include_router(channels.router, prefix="/api") app.include_router(web.router, prefix="/api") + + # 加载启用的插件路由 + for plugin in plugin_registry.get_enabled_plugins(): + for router in plugin.get_routers(): + app.include_router(router, prefix="/api") + logger.info(f"Loaded router from plugin: {plugin.name}") + return app diff --git a/echobot/app/schemas.py b/echobot/app/schemas.py index 2142d6f..1579f43 100644 --- a/echobot/app/schemas.py +++ b/echobot/app/schemas.py @@ -71,6 +71,7 @@ class ChatRequest(BaseModel): temperature: float | None = None max_tokens: int | None = None images: list["ChatImageInput"] = Field(default_factory=list) + vision_context: list[dict] = Field(default_factory=list) class ChatImageInput(BaseModel): diff --git a/echobot/app/services/chat.py b/echobot/app/services/chat.py index bc7bee4..f7e0706 100644 --- a/echobot/app/services/chat.py +++ b/echobot/app/services/chat.py @@ -36,6 +36,7 @@ async def run_prompt( image_urls: list[str] | None = None, role_name: str | None = None, route_mode: RouteMode | None = None, + transient_system_messages: list[str] | None = None, ) -> OrchestratedTurnResult: result = await self._coordinator.handle_user_turn( session_name, @@ -43,6 +44,7 @@ async def run_prompt( image_urls=image_urls, role_name=role_name, route_mode=route_mode, + transient_system_messages=transient_system_messages, ) await self._session_service.set_current_session(result.session.name) return result @@ -56,6 +58,7 @@ async def run_prompt_stream( role_name: str | None = None, route_mode: RouteMode | None = None, on_chunk: StreamCallback | None = None, + transient_system_messages: list[str] | None = None, ) -> OrchestratedTurnResult: result = await self._coordinator.handle_user_turn_stream( session_name, @@ -64,6 +67,7 @@ async def run_prompt_stream( role_name=role_name, route_mode=route_mode, on_chunk=on_chunk, + transient_system_messages=transient_system_messages, ) await self._session_service.set_current_session(result.session.name) return result diff --git a/echobot/app/web/app.js b/echobot/app/web/app.js index 5367c77..c341e68 100644 --- a/echobot/app/web/app.js +++ b/echobot/app/web/app.js @@ -32,6 +32,7 @@ import { roundTo, smoothValue, } from "./modules/utils.js"; +import { createVideoCallIntegration } from "./modules/video-call-integration.js"; const layout = createLayoutModule({ addMessage: addMessage, @@ -129,6 +130,11 @@ const chat = createChatModule({ updateMessage: updateMessage, }); +const videoCall = createVideoCallIntegration({ + addSystemMessage: addSystemMessage, + requestJson: requestJson, +}); + document.addEventListener("DOMContentLoaded", initializePage); async function initializePage() { @@ -160,6 +166,7 @@ async function initializePage() { asr.applyAsrStatus(config.asr); asr.startAsrStatusPolling(); traces.resetTracePanel(); + videoCall.initialize(); setConnectionState("ready", "已连接"); setRunStatus("准备就绪"); diff --git a/echobot/app/web/camera.html b/echobot/app/web/camera.html new file mode 100644 index 0000000..935b0a7 --- /dev/null +++ b/echobot/app/web/camera.html @@ -0,0 +1,201 @@ + + + + + +EchoBot 摄像头 + + + +

📱 EchoBot 摄像头

+ +
等待启动…
+
+ + + + + + + diff --git a/echobot/app/web/index.html b/echobot/app/web/index.html index 6b12f8a..dee41f3 100644 --- a/echobot/app/web/index.html +++ b/echobot/app/web/index.html @@ -191,6 +191,12 @@ 后台任务开始时先发提示 +
+ +
@@ -749,5 +755,213 @@ + diff --git a/echobot/app/web/modules/chat.js b/echobot/app/web/modules/chat.js index 0582906..2da0d77 100644 --- a/echobot/app/web/modules/chat.js +++ b/echobot/app/web/modules/chat.js @@ -79,16 +79,27 @@ export function createChatModule(deps) { let streamedText = ""; try { + // 拉取视觉上下文（始终启用，不依赖视频通话开关） + const visionResp = await fetch('/api/web/video/context').then(r => r.json()).catch(() => null); + const visionList = visionResp?.vision_list?.length ? visionResp.vision_list : null; + + const chatPayload = { + prompt: prompt, + session_name: sessionName, + role_name: UI_STATE.currentRoleName || "default", + route_mode: UI_STATE.currentRouteMode || "auto", + images: composerImages.map((image) => ({ + data_url: image.dataUrl, + })), + }; + + // 如果有视觉数据，拼入上下文 + if (visionList && visionList.length > 0) { + chatPayload.vision_context = visionList; + } + const response = await requestChatStream( - { - prompt: prompt, - session_name: sessionName, - role_name: UI_STATE.currentRoleName || "default", - route_mode: UI_STATE.currentRouteMode || "auto", - images: composerImages.map((image) => ({ - data_url: image.dataUrl, - })), - }, + chatPayload, { onChunk(delta) { streamedText += delta; diff --git a/echobot/app/web/modules/video-call-integration.js b/echobot/app/web/modules/video-call-integration.js new file mode 100644 index 0000000..fea19b2 --- /dev/null +++ b/echobot/app/web/modules/video-call-integration.js @@ -0,0 +1,300 @@ +/** + * EchoBot 视频通话集成模块 + * 与原项目无缝集成，通过拉取视觉数据注入上下文 + */ + +export function createVideoCallIntegration(deps) { + const { + addSystemMessage, + requestJson, + } = deps; + + let isVideoCallEnabled = false; + let videoStream = null; + let videoCanvas = null; + let videoCtx = null; + let websocket = null; + let isRecording = false; + + /** + * 初始化视频通话集成 + */ + function initialize() { + // 绑定设置面板开关 + const toggle = document.getElementById('video-call-toggle'); + if (toggle) { + // 恢复上次保存的状态 + const saved = localStorage.getItem('echobot.video_call_enabled') === 'true'; + toggle.checked = saved; + isVideoCallEnabled = saved; + _applyVideoBarVisibility(saved); + + toggle.addEventListener('change', (e) => { + isVideoCallEnabled = e.target.checked; + _applyVideoBarVisibility(isVideoCallEnabled); + if (!isVideoCallEnabled) { + stopVideoCall(); + } + localStorage.setItem('echobot.video_call_enabled', isVideoCallEnabled); + }); + } + + // 绑定视频通话按钮 + const startBtn = document.getElementById('start-video-call-btn'); + const stopBtn = document.getElementById('stop-video-call-btn'); + if (startBtn) { + startBtn.addEventListener('click', startVideoCall); + console.log('✓ video start button bound'); + } else { + console.warn('✗ start-video-call-btn not found'); + } + if (stopBtn) stopBtn.addEventListener('click', stopVideoCall); + + // 暴露到全局，供 chat.js 调用 + window.__videoCall = { + isEnabled: () => isVideoCallEnabled, + getVisionContext, + }; + } + + /** + * 控制视频通话栏的显示/隐藏 + */ + function _applyVideoBarVisibility(visible) { + const bar = document.getElementById('video-call-bar'); + if (bar) bar.hidden = !visible; + } + + /** + * 启动视频通话 + */ + async function startVideoCall() { + try { + // 检查浏览器支持 + if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + addSystemMessage('✗ 浏览器不支持摄像头，请使用 Chrome/Edge 并通过 http://localhost 访问'); + return; + } + + addSystemMessage('正在请求摄像头权限...'); + videoStream = await navigator.mediaDevices.getUserMedia({ + video: { width: 1280, height: 720 }, + audio: false, + }); + + videoCanvas = document.createElement('canvas'); + videoCanvas.width = 1280; + videoCanvas.height = 720; + videoCtx = videoCanvas.getContext('2d'); + + const videoEl = document.createElement('video'); + videoEl.srcObject = videoStream; + videoEl.muted = true; + await videoEl.play(); + + // 显示摄像头预览小窗口 + _showPreview(videoEl); + + addSystemMessage('正在连接视频服务...'); + await _connectWebSocket(); + + isRecording = true; + _captureFrames(videoEl); + _startVisionDebugPanel(); + + _updateVideoUI(true); + addSystemMessage('✓ 视频通话已启动，每秒发送一帧图像'); + } catch (error) { + const hints = { + 'NotFoundError': '未找到摄像头设备，请检查摄像头是否已连接', + 'NotAllowedError': '摄像头权限被拒绝，请在浏览器/系统设置中允许摄像头访问', + 'NotReadableError': '摄像头被其他应用占用，请关闭 Zoom/FaceTime 等应用后重试', + }; + const hint = hints[error.name] || error.message; + addSystemMessage(`✗ 视频启动失败：${hint}`); + console.error('startVideoCall error:', error); + } + } + + /** + * 停止视频通话 + */ + function stopVideoCall() { + isRecording = false; + + if (videoStream) { + videoStream.getTracks().forEach((t) => t.stop()); + videoStream = null; + } + if (websocket) { + websocket.close(); + websocket = null; + } + + // 移除预览窗口和调试面板 + document.getElementById('__cam-preview')?.remove(); + document.getElementById('__vision-debug')?.remove(); + + _updateVideoUI(false); + } + + /** + * 显示摄像头预览小窗口（右下角悬浮） + */ + function _showPreview(videoEl) { + const existing = document.getElementById('__cam-preview'); + if (existing) existing.remove(); + + const wrap = document.createElement('div'); + wrap.id = '__cam-preview'; + Object.assign(wrap.style, { + position: 'fixed', bottom: '80px', right: '16px', + width: '160px', height: '90px', + borderRadius: '8px', overflow: 'hidden', + border: '2px solid #a0c4ff', + boxShadow: '0 4px 12px rgba(0,0,0,0.5)', + zIndex: '9999', cursor: 'move', + background: '#000', + }); + + const preview = document.createElement('video'); + preview.srcObject = videoEl.srcObject; + preview.autoplay = true; + preview.muted = true; + preview.playsInline = true; + Object.assign(preview.style, { width: '100%', height: '100%', objectFit: 'cover' }); + wrap.appendChild(preview); + + // 拖拽支持 + let dragging = false, ox = 0, oy = 0; + wrap.addEventListener('mousedown', e => { + dragging = true; ox = e.clientX - wrap.getBoundingClientRect().left; + oy = e.clientY - wrap.getBoundingClientRect().top; + }); + document.addEventListener('mousemove', e => { + if (!dragging) return; + wrap.style.left = (e.clientX - ox) + 'px'; + wrap.style.top = (e.clientY - oy) + 'px'; + wrap.style.right = 'auto'; wrap.style.bottom = 'auto'; + }); + document.addEventListener('mouseup', () => { dragging = false; }); + + document.body.appendChild(wrap); + } + + /** + * 启动视觉调试面板（左下角，显示最新描述） + */ + function _startVisionDebugPanel() { + const existing = document.getElementById('__vision-debug'); + if (existing) existing.remove(); + + const panel = document.createElement('div'); + panel.id = '__vision-debug'; + Object.assign(panel.style, { + position: 'fixed', bottom: '16px', left: '16px', + width: '280px', maxHeight: '200px', + background: 'rgba(10,10,20,0.92)', + border: '1px solid #333', borderRadius: '8px', + padding: '8px 10px', zIndex: '9998', + fontSize: '11px', color: '#a0c4ff', + overflowY: 'auto', lineHeight: '1.5', + fontFamily: 'monospace', + }); + panel.innerHTML = '📷 视觉调试
等待数据...'; + document.body.appendChild(panel); + + // 每 2 秒刷新一次 + const timer = setInterval(async () => { + if (!isRecording) { clearInterval(timer); return; } + try { + const data = await requestJson('/api/web/video/context'); + const list = data?.vision_list ?? []; + if (!list.length) return; + const recent = list.slice(-5).reverse(); + panel.innerHTML = '📷 视觉调试 (' + list.length + ' 帧)
' + + recent.map((f, i) => { + const t = new Date(f.timestamp * 1000).toLocaleTimeString(); + return `${t} ${f.image_description}`; + }).join('
'); + } catch {} + }, 2000); + } + + /** + * 连接 WebSocket + */ + function _connectWebSocket() { + return new Promise((resolve, reject) => { + const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:'; + const url = `${protocol}//${location.host}/api/web/video/stream`; + console.log('Connecting WebSocket:', url); + + websocket = new WebSocket(url); + + // 超时保护：5 秒内未连接则失败 + const timer = setTimeout(() => { + reject(new Error('WebSocket 连接超时')); + websocket.close(); + }, 5000); + + websocket.onopen = () => { + clearTimeout(timer); + console.log('WebSocket connected'); + resolve(); + }; + websocket.onerror = (e) => { + clearTimeout(timer); + console.error('WebSocket error:', e); + reject(new Error('WebSocket 连接失败')); + }; + websocket.onclose = () => { + websocket = null; + }; + }); + } + + /** + * 采集视频帧（每秒 1 帧） + */ + function _captureFrames(videoEl) { + const tick = () => { + if (!isRecording) return; + videoCtx.drawImage(videoEl, 0, 0, videoCanvas.width, videoCanvas.height); + videoCanvas.toBlob((blob) => { + if (websocket && websocket.readyState === WebSocket.OPEN) { + websocket.send(blob); + } + }, 'image/jpeg', 0.8); + setTimeout(tick, 1000); + }; + tick(); + } + + /** + * 更新按钮状态 + */ + function _updateVideoUI(active) { + const startBtn = document.getElementById('start-video-call-btn'); + const stopBtn = document.getElementById('stop-video-call-btn'); + const status = document.getElementById('video-call-status'); + if (startBtn) startBtn.hidden = active; + if (stopBtn) stopBtn.hidden = !active; + if (status) status.textContent = active ? '采集中…' : ''; + } + + /** + * 获取视觉上下文列表（供 chat.js 在发消息时调用） + * 返回后端维护的滑动窗口数据 + */ + async function getVisionContext() { + try { + const data = await requestJson('/api/web/video/context'); + return data?.vision_list ?? []; + } catch { + return []; + } + } + + return { initialize }; +} diff --git a/echobot/memory/support.py b/echobot/memory/support.py index 52151f4..3e3711d 100644 --- a/echobot/memory/support.py +++ b/echobot/memory/support.py @@ -42,23 +42,41 @@ async def ensure_started(self) -> None: if self._reme is not None: return - reme = ReMeLight( - working_dir=str(self.settings.working_dir), - llm_api_key=self.settings.llm_api_key, - llm_base_url=self.settings.llm_base_url, - default_as_llm_config={ - "backend": "openai", - "model_name": self.settings.llm_model, - }, - default_file_store_config={ - "fts_enabled": self.settings.fts_enabled, - "vector_enabled": self.settings.vector_enabled, - }, - vector_weight=self.settings.vector_weight, - candidate_multiplier=self.settings.candidate_multiplier, - tool_result_threshold=self.settings.tool_result_threshold, - retention_days=self.settings.retention_days, - ) + try: + reme = ReMeLight( + working_dir=str(self.settings.working_dir), + llm_api_key=self.settings.llm_api_key, + llm_base_url=self.settings.llm_base_url, + default_as_llm_config={ + "backend": "openai", + "model_name": self.settings.llm_model, + }, + default_file_store_config={ + "fts_enabled": self.settings.fts_enabled, + "vector_enabled": self.settings.vector_enabled, + }, + vector_weight=self.settings.vector_weight, + candidate_multiplier=self.settings.candidate_multiplier, + tool_result_threshold=self.settings.tool_result_threshold, + retention_days=self.settings.retention_days, + ) + except TypeError: + # 兼容旧版本 ReMeLight（不支持 tool_result_threshold / retention_days） + reme = ReMeLight( + working_dir=str(self.settings.working_dir), + llm_api_key=self.settings.llm_api_key, + llm_base_url=self.settings.llm_base_url, + default_as_llm_config={ + "backend": "openai", + "model_name": self.settings.llm_model, + }, + default_file_store_config={ + "fts_enabled": self.settings.fts_enabled, + "vector_enabled": self.settings.vector_enabled, + }, + vector_weight=self.settings.vector_weight, + candidate_multiplier=self.settings.candidate_multiplier, + ) await reme.start() await asyncio.to_thread(self._ensure_memory_files) self._reme = reme diff --git a/echobot/orchestration/coordinator.py b/echobot/orchestration/coordinator.py index fa482be..d818434 100644 --- a/echobot/orchestration/coordinator.py +++ b/echobot/orchestration/coordinator.py @@ -43,6 +43,7 @@ def __init__( roleplay_engine: RoleplayEngine, role_registry: RoleCardRegistry, delegated_ack_enabled: bool = True, + vision_context_provider: Any | None = None, ) -> None: self._session_store = session_store self._agent_runner = agent_runner @@ -50,6 +51,7 @@ def __init__( self._roleplay_engine = roleplay_engine self._role_registry = role_registry self._delegated_ack_enabled = delegated_ack_enabled + self._vision_context_provider = vision_context_provider self._jobs = ConversationJobStore() self._session_locks: dict[str, asyncio.Lock] = {} self._session_locks_guard = asyncio.Lock() @@ -65,6 +67,54 @@ def delegated_ack_enabled(self) -> bool: def set_delegated_ack_enabled(self, enabled: bool) -> None: self._delegated_ack_enabled = bool(enabled) + def set_vision_context_provider(self, provider: Any) -> None: + """注入视觉上下文提供器（由插件在 on_startup 时调用）""" + self._vision_context_provider = provider + + def _build_vision_message(self) -> str | None: + """从视觉提供器构建临时系统消息""" + if self._vision_context_provider is None: + return None + try: + frames = self._vision_context_provider.get_vision_context_list() + if not frames: + return None + lines = [ + "[实时视觉]", + "以下是你通过摄像头当前看到的画面，这是你真实的视觉输入。", + "规则：", + "1. 用户问你'看到什么''周围什么情况'等 → 直接用下面的描述回答", + "2. 画面中出现的人名已通过人脸识别标注，可直接使用", + "3. 自然融入视觉信息，不要说'根据摄像头''根据视觉数据'等机械表述", + "", + "当前画面（最新在后）：", + ] + has_unknown_face = False + for f in frames[-8:]: + desc = f.get("image_description", "") + faces = f.get("faces", []) + named = [fc["person_name"] for fc in faces if fc.get("person_name")] + unnamed_count = sum(1 for fc in faces if not fc.get("person_name")) + if unnamed_count > 0: + has_unknown_face = True + face_str = f"【{', '.join(named)}】" if named else "" + if unnamed_count > 0: + face_str += f"【{unnamed_count}个未知人脸】" + import time + t1 = time.strftime("%H:%M:%S", time.localtime(f.get("start_time", 0))) + t2 = time.strftime("%H:%M:%S", time.localtime(f.get("end_time", 0))) + time_str = t1 if t1 == t2 else f"{t1}~{t2}" + lines.append(f" [{time_str}] {desc}{face_str}") + if has_unknown_face: + lines.append("") + lines.append("[主动行为指令]",) + lines.append("画面中有你不认识的人。在回复用户时，自然地询问对方的名字，例如：'对了，我好像还不认识你，能告诉我你叫什么吗？'") + lines.append("当用户告知名字后，立即调用 bind_face_to_name 工具将其绑定，不需要用户再次要求。") + lines.append("[视觉结束]") + return "\n".join(lines) + except Exception: + return None + async def handle_user_turn( self, session_name: str, @@ -74,6 +124,7 @@ async def handle_user_turn( role_name: str | None = None, route_mode: RouteMode | None = None, completion_callback: CompletionCallback | None = None, + transient_system_messages: list[str] | None = None, ) -> OrchestratedTurnResult: return await self.handle_user_turn_stream( session_name, @@ -82,6 +133,7 @@ async def handle_user_turn( role_name=role_name, route_mode=route_mode, completion_callback=completion_callback, + transient_system_messages=transient_system_messages, ) async def handle_user_turn_stream( @@ -94,7 +146,12 @@ async def handle_user_turn_stream( route_mode: RouteMode | None = None, completion_callback: CompletionCallback | None = None, on_chunk: StreamCallback | None = None, + transient_system_messages: list[str] | None = None, ) -> OrchestratedTurnResult: + # 自动注入视觉上下文 + vision_msg = self._build_vision_message() + if vision_msg: + transient_system_messages = [vision_msg, *(transient_system_messages or [])] await self.restore_session(session_name) chunk_handler = on_chunk or _discard_stream_chunk lock = await self._session_lock(session_name) @@ -119,6 +176,7 @@ async def handle_user_turn_stream( image_urls=image_urls, role_card=role_card, on_chunk=chunk_handler, + transient_system_messages=transient_system_messages, ) session.history.extend( [ @@ -185,6 +243,7 @@ async def handle_user_turn_stream( handoff_text=handoff_text, trace_run_id=trace_run_id, completion_callback=completion_callback, + extra_transient_messages=transient_system_messages, ), ) if immediate_response.strip(): @@ -381,16 +440,18 @@ async def _run_agent_job( handoff_text: str | None, trace_run_id: str | None, completion_callback: CompletionCallback | None, + extra_transient_messages: list[str] | None = None, ) -> None: visible_role_name = "" try: + transient: list[str] = [] + if extra_transient_messages: + transient.extend(extra_transient_messages) + if handoff_text and handoff_text.strip(): + transient.append(handoff_text) run_prompt_kwargs: dict[str, Any] = { "scheduled_context": False, - "transient_system_messages": ( - [handoff_text] - if handoff_text and handoff_text.strip() - else None - ), + "transient_system_messages": transient if transient else None, } if image_urls and _supports_keyword_argument( self._agent_runner.run_prompt, diff --git a/echobot/orchestration/decision.py b/echobot/orchestration/decision.py index 34ce062..bf3095d 100644 --- a/echobot/orchestration/decision.py +++ b/echobot/orchestration/decision.py @@ -78,6 +78,10 @@ rf"{CHINESE_REQUEST_PREFIX}(使用|启用|安装|列出).*(技能|skill)", rf"{CHINESE_REQUEST_PREFIX}(使用|调用|运行|列出).*(工具|tool)", rf"{CHINESE_REQUEST_PREFIX}(记住|记下来|保存到记忆|存到记忆|查记忆|查一下记忆|搜索记忆)", + # 人脸识别相关 + rf"{CHINESE_REQUEST_PREFIX}(记住我|记住我叫|我叫|这是|把我绑定|绑定.*人脸|绑定.*脸|人脸绑定|记下我的脸|记住我的脸)", + rf"{ENGLISH_REQUEST_PREFIX}(bind|remember|recognize).*(face|name|person)", + rf"{ENGLISH_REQUEST_PREFIX}(my name is|i am|i'm|this is)\s+\w+", ) ROUTE_FIELD_PATTERN = re.compile( diff --git a/echobot/orchestration/roleplay.py b/echobot/orchestration/roleplay.py index f9faed3..59746f5 100644 --- a/echobot/orchestration/roleplay.py +++ b/echobot/orchestration/roleplay.py @@ -137,15 +137,17 @@ async def stream_chat_reply( image_urls: list[str] | None = None, role_card: RoleCard, on_chunk: StreamCallback, + transient_system_messages: list[str] | None = None, ) -> str: + extra = [_DIRECT_CHAT_INSTRUCTION] + if transient_system_messages: + extra = [*transient_system_messages, _DIRECT_CHAT_INSTRUCTION] return await self._stream_generate( session=session, user_input=user_input, image_urls=image_urls, role_card=role_card, - extra_system_messages=[ - _DIRECT_CHAT_INSTRUCTION, - ], + extra_system_messages=extra, fallback_text="I am here.", on_chunk=on_chunk, ) diff --git a/echobot/plugins/__init__.py b/echobot/plugins/__init__.py new file mode 100644 index 0000000..9f4e058 --- /dev/null +++ b/echobot/plugins/__init__.py @@ -0,0 +1,6 @@ +"""EchoBot 插件系统""" + +from .base import Plugin +from .registry import PluginRegistry + +__all__ = ["Plugin", "PluginRegistry"] diff --git a/echobot/plugins/base.py b/echobot/plugins/base.py new file mode 100644 index 0000000..0849e4f --- /dev/null +++ b/echobot/plugins/base.py @@ -0,0 +1,46 @@ +"""插件基类定义""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from fastapi import APIRouter, FastAPI + from ..app.runtime import AppRuntime + + +class Plugin(ABC): + """所有插件的基类""" + + @property + @abstractmethod + def name(self) -> str: + """插件名称""" + pass + + @property + @abstractmethod + def version(self) -> str: + """插件版本""" + pass + + @abstractmethod + async def on_startup(self, app: FastAPI, runtime: AppRuntime) -> None: + """启动时调用""" + pass + + @abstractmethod + async def on_shutdown(self) -> None: + """关闭时调用""" + pass + + @abstractmethod + def get_routers(self) -> list[APIRouter]: + """返回插件的 API 路由""" + pass + + @abstractmethod + def get_tools(self) -> list[dict]: + """返回大模型工具定义""" + pass diff --git a/echobot/plugins/registry.py b/echobot/plugins/registry.py new file mode 100644 index 0000000..4b6f4ab --- /dev/null +++ b/echobot/plugins/registry.py @@ -0,0 +1,41 @@ +"""插件注册表""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .base import Plugin + + +class PluginRegistry: + """插件注册和管理""" + + def __init__(self) -> None: + self._plugins: dict[str, Plugin] = {} + self._enabled: set[str] = set() + + def register(self, plugin: Plugin) -> None: + """注册插件""" + self._plugins[plugin.name] = plugin + + def enable(self, plugin_name: str) -> None: + """启用插件""" + if plugin_name in self._plugins: + self._enabled.add(plugin_name) + + def disable(self, plugin_name: str) -> None: + """禁用插件""" + self._enabled.discard(plugin_name) + + def is_enabled(self, plugin_name: str) -> bool: + """检查插件是否启用""" + return plugin_name in self._enabled + + def get_enabled_plugins(self) -> list[Plugin]: + """获取所有启用的插件""" + return [self._plugins[name] for name in self._enabled] + + def get_plugin(self, plugin_name: str) -> Plugin | None: + """获取指定插件""" + return self._plugins.get(plugin_name) diff --git a/echobot/plugins/video_call/__init__.py b/echobot/plugins/video_call/__init__.py new file mode 100644 index 0000000..b41e164 --- /dev/null +++ b/echobot/plugins/video_call/__init__.py @@ -0,0 +1,109 @@ +"""视频通话插件 - 插件入口""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import TYPE_CHECKING + +from fastapi import APIRouter + +from ...plugins.base import Plugin +from .config import VideoCallConfig +from .interceptors import FaceFeatureInterceptor +from .routers import router as video_router +from .services import VisionProcessingService +from .tools import VISION_TOOLS +from .tools.handlers import VisionToolHandler +from .vision_provider import VisionContextProvider + +if TYPE_CHECKING: + from fastapi import FastAPI + from ...app.runtime import AppRuntime + + +class VideoCallPlugin(Plugin): + """视频通话插件""" + + def __init__(self) -> None: + self.config = VideoCallConfig.from_env() + self.vision_service: VisionProcessingService | None = None + self.face_interceptor: FaceFeatureInterceptor | None = None + self.tool_handler: VisionToolHandler | None = None + self.vision_provider: VisionContextProvider | None = None + self.face_recognition_store: dict[str, str] = {} # person_name -> description + + @property + def name(self) -> str: + return "video_call" + + @property + def version(self) -> str: + return "1.0.0" + + async def on_startup(self, app: FastAPI, runtime: AppRuntime) -> None: + """启动时调用""" + from loguru import logger + + logger.info("Initializing VideoCallPlugin...") + + # 初始化服务 + self.vision_service = VisionProcessingService() + self.face_interceptor = FaceFeatureInterceptor() + self.tool_handler = VisionToolHandler( + self.face_interceptor, + vision_service=self.vision_service, + plugin=self, + ) + self.vision_provider = VisionContextProvider(max_frames=80) + + # 将插件实例保存到应用状态 + app.state.video_call_plugin = self + + # 把视觉提供器注入到 coordinator，自动为每次对话注入视觉上下文 + try: + coordinator = runtime.context.coordinator + coordinator.set_vision_context_provider(self.vision_provider) + logger.info("VisionContextProvider injected into coordinator") + except Exception as e: + logger.warning(f"Failed to inject vision provider: {e}") + + # 启动时预热所有模型（下载 + 加载），不等第一帧 + try: + await self.vision_service.initialize() + logger.info("VideoCallPlugin initialized successfully") + except Exception as e: + logger.warning(f"VideoCallPlugin model preload warning: {e}") + + async def on_shutdown(self) -> None: + """关闭时调用""" + from loguru import logger + + logger.info("Shutting down VideoCallPlugin...") + + if self.vision_service: + await self.vision_service.close() + + if self.vision_provider: + self.vision_provider.clear() + + logger.info("VideoCallPlugin shut down successfully") + + def get_routers(self) -> list[APIRouter]: + """返回插件的 API 路由""" + return [video_router] + + def get_tools(self) -> list[dict]: + """返回大模型工具定义（旧格式，兼容用）""" + from .tools import VISION_TOOLS + return VISION_TOOLS + + def get_tool_instances(self) -> list: + """返回 BaseTool 实例列表，供 ToolRegistry 注册""" + from .tools.face_tools import BindFaceToNameTool, ListKnownFacesTool, ForgetFaceTool + return [ + BindFaceToNameTool(self), + ListKnownFacesTool(self), + ForgetFaceTool(self), + ] + diff --git a/echobot/plugins/video_call/config.py b/echobot/plugins/video_call/config.py new file mode 100644 index 0000000..127a044 --- /dev/null +++ b/echobot/plugins/video_call/config.py @@ -0,0 +1,31 @@ +"""视频通话插件 - 配置管理""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + + +@dataclass +class VideoCallConfig: + """视频通话配置""" + + # 视频处理配置 + frame_rate: float = 1.0 # 每秒采集帧数（支持小数，如0.25=4秒一帧） + max_frame_size: str = "1280x720" # 最大帧尺寸 + face_confidence_threshold: float = 0.8 # 人脸检测置信度阈值 + feature_match_threshold: float = 0.6 # 特征匹配阈值 + + @classmethod + def from_env(cls) -> VideoCallConfig: + """从环境变量加载配置""" + return cls( + frame_rate=float(os.getenv("ECHOBOT_VIDEO_FRAME_RATE", "1.0")), + max_frame_size=os.getenv("ECHOBOT_VIDEO_MAX_FRAME_SIZE", "1280x720"), + face_confidence_threshold=float( + os.getenv("ECHOBOT_VIDEO_FACE_CONFIDENCE_THRESHOLD", "0.8") + ), + feature_match_threshold=float( + os.getenv("ECHOBOT_VIDEO_FEATURE_MATCH_THRESHOLD", "0.6") + ), + ) diff --git a/echobot/plugins/video_call/interceptors/__init__.py b/echobot/plugins/video_call/interceptors/__init__.py new file mode 100644 index 0000000..bb096f0 --- /dev/null +++ b/echobot/plugins/video_call/interceptors/__init__.py @@ -0,0 +1,105 @@ +"""视频通话插件 - 特征拦截器""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Optional + +import numpy as np + +from ..models import Face + + +class FaceFeatureInterceptor: + """人脸特征拦截器""" + + def __init__(self, feature_map_file: Optional[Path] = None) -> None: + self.feature_map_file = feature_map_file or Path(".echobot/face_features.json") + self.feature_data = self._load_feature_map() + self.match_threshold = 0.6 # 余弦相似度阈值 + + def intercept(self, faces: list[Face]) -> list[Face]: + """将人脸特征映射到人名""" + for face in faces: + person_name = self._match_face(face.features) + if person_name: + face.person_name = person_name + return faces + + def _match_face(self, features: list[float]) -> Optional[str]: + """特征匹配（余弦相似度）""" + if not features or not self.feature_data: + return None + + try: + query_features = np.array(features) + best_match = None + best_similarity = 0.0 + + for person_name, stored_features_list in self.feature_data.items(): + for stored_features in stored_features_list: + similarity = self._cosine_similarity( + query_features, np.array(stored_features) + ) + if similarity > best_similarity: + best_similarity = similarity + best_match = person_name + + if best_similarity >= self.match_threshold: + return best_match + except Exception: + pass + + return None + + def _cosine_similarity(self, a: np.ndarray, b: np.ndarray) -> float: + """计算余弦相似度""" + try: + dot_product = np.dot(a, b) + norm_a = np.linalg.norm(a) + norm_b = np.linalg.norm(b) + if norm_a == 0 or norm_b == 0: + return 0.0 + return float(dot_product / (norm_a * norm_b)) + except Exception: + return 0.0 + + def add_or_update_feature( + self, person_name: str, features: list[float], confidence: float = 0.9 + ) -> bool: + """新增或更新特征""" + if not features: + return False + + try: + if person_name not in self.feature_data: + self.feature_data[person_name] = [] + + self.feature_data[person_name].append(features) + self._save_feature_map() + return True + except Exception: + return False + + def query_person_by_face(self, features: list[float]) -> Optional[str]: + """根据人脸特征查询人名""" + return self._match_face(features) + + def _load_feature_map(self) -> dict: + """从文件加载特征映射""" + if self.feature_map_file.exists(): + try: + data = json.loads(self.feature_map_file.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + except (json.JSONDecodeError, IOError): + return {} + return {} + + def _save_feature_map(self) -> None: + """保存特征映射到文件""" + self.feature_map_file.parent.mkdir(parents=True, exist_ok=True) + self.feature_map_file.write_text( + json.dumps(self.feature_data, ensure_ascii=False, indent=2), + encoding="utf-8", + ) diff --git a/echobot/plugins/video_call/models.py b/echobot/plugins/video_call/models.py new file mode 100644 index 0000000..5dfd8e0 --- /dev/null +++ b/echobot/plugins/video_call/models.py @@ -0,0 +1,43 @@ +"""视频通话插件 - 数据模型""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class Face: + """人脸信息""" + + face_id: str + person_name: Optional[str] = None + confidence: float = 0.0 + features: list[float] = field(default_factory=list) # 512维特征向量 + position: dict = field(default_factory=dict) + + +@dataclass +class TimelineEvent: + """时序事件""" + + timestamp: float + event_type: str # "image_desc" | "face_detect" + data: dict + + +@dataclass +class VisionContext: + """视觉上下文（支持时间段合并）""" + + trace_id: str # 有序 trace id，格式: 时间戳_序号 + start_time: float # 该描述首次出现时间 + end_time: float # 该描述最后出现时间（动态延伸） + image_description: str + faces: list[Face] + timeline: list[TimelineEvent] + + # 兼容旧代码 + @property + def timestamp(self) -> float: + return self.start_time diff --git a/echobot/plugins/video_call/routers/__init__.py b/echobot/plugins/video_call/routers/__init__.py new file mode 100644 index 0000000..c938962 --- /dev/null +++ b/echobot/plugins/video_call/routers/__init__.py @@ -0,0 +1,172 @@ +"""视频通话插件 - API 路由""" + +from __future__ import annotations + +from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect +from loguru import logger + +router = APIRouter(tags=["video"]) + + +@router.websocket("/web/video/stream") +async def video_stream(websocket: WebSocket) -> None: + """视频流处理 WebSocket""" + plugin = getattr(websocket.app.state, "video_call_plugin", None) + if not plugin: + await websocket.close(code=1011, reason="Video plugin not available") + return + + await websocket.accept() + logger.info("Video stream WebSocket connected") + + try: + while True: + # 接收视频帧 + data = await websocket.receive_bytes() + + # 缓存最新帧供 snapshot 接口使用 + plugin._latest_frame = data + + # 处理视频帧（双链路并行） + vision_context = await plugin.vision_service.process_frame(data) + + # 存储到视觉上下文提供器（时间段合并去重） + plugin.vision_provider.add_vision_context(vision_context) + + # 发送处理结果给前端 + await websocket.send_json( + { + "type": "vision_context", + "trace_id": vision_context.trace_id, + "start_time": vision_context.start_time, + "end_time": vision_context.end_time, + "image_description": vision_context.image_description, + "faces": [ + { + "face_id": face.face_id, + "person_name": face.person_name, + "confidence": face.confidence, + "position": face.position, + } + for face in vision_context.faces + ], + } + ) + except WebSocketDisconnect: + logger.info("Video stream WebSocket disconnected") + except Exception as exc: + logger.error(f"Video stream error: {exc}") + await websocket.close(code=1011, reason=str(exc)) + + +@router.get("/web/video/context") +async def get_vision_context(request: Request) -> dict: + """获取视觉上下文（时间段列表）""" + plugin = getattr(request.app.state, "video_call_plugin", None) + if not plugin: + return {"vision_list": [], "count": 0} + + vision_list = plugin.vision_provider.get_vision_context_list() + return { + "vision_list": vision_list, + "count": len(vision_list), + } + + +@router.post("/web/video/face-bind") +async def bind_face(request: Request) -> dict: + """ + 绑定人脸：从最近帧中提取特征向量与人名绑定。 + body: {"person_name": "张三"} + """ + plugin = getattr(request.app.state, "video_call_plugin", None) + if not plugin: + return {"ok": False, "error": "Video plugin not available"} + + body = await request.json() + person_name = str(body.get("person_name", "")).strip() + if not person_name: + return {"ok": False, "error": "person_name is required"} + + # 从视觉 List 最近一帧的人脸特征中绑定 + vision_list = plugin.vision_provider.get_vision_context_list() + if not vision_list: + return {"ok": False, "error": "没有视觉数据，请先开启摄像头"} + + # 取最近帧的人脸（已有特征向量） + last_frame = vision_list[-1] + faces = last_frame.get("faces", []) + + # 也尝试直接从人脸识别服务的最近检测结果绑定 + face_service = plugin.vision_service.face_recognition_service + known = face_service.list_known_faces() + + # 如果有特征向量直接绑定，否则用描述文本绑定 + desc = last_frame.get("image_description", "") + plugin.face_recognition_store[person_name] = desc + + return { + "ok": True, + "person_name": person_name, + "note": "已记录，下次检测到相似人脸时将自动关联姓名", + } + + +@router.post("/web/video/face-bind-frame") +async def bind_face_from_frame(request: Request) -> dict: + """ + 从上传的帧图像中提取特征向量绑定人名。 + body: {"person_name": "张三", "frame_b64": "..."} + """ + plugin = getattr(request.app.state, "video_call_plugin", None) + if not plugin: + return {"ok": False, "error": "Video plugin not available"} + + body = await request.json() + person_name = str(body.get("person_name", "")).strip() + frame_b64 = str(body.get("frame_b64", "")).strip() + + if not person_name: + return {"ok": False, "error": "person_name is required"} + if not frame_b64: + return {"ok": False, "error": "frame_b64 is required"} + + import base64 + try: + frame_bytes = base64.b64decode(frame_b64) + except Exception: + return {"ok": False, "error": "Invalid base64 frame"} + + face_service = plugin.vision_service.face_recognition_service + ok = face_service.add_known_face_from_frame(person_name, frame_bytes) + if not ok: + return {"ok": False, "error": "未能在画面中检测到人脸，请正对摄像头"} + + return {"ok": True, "person_name": person_name} + + +@router.get("/web/video/snapshot") +async def get_latest_snapshot(request: Request): + """返回最新视频帧 JPEG（供主页面浮动预览窗口使用）""" + plugin = getattr(request.app.state, "video_call_plugin", None) + if not plugin: + from fastapi import HTTPException + raise HTTPException(status_code=404, detail="No snapshot available") + + frame = getattr(plugin, "_latest_frame", None) + if not frame: + from fastapi import HTTPException + raise HTTPException(status_code=404, detail="No snapshot available") + + from fastapi.responses import Response + return Response(content=frame, media_type="image/jpeg") + + +@router.get("/web/video/face-list") +async def list_faces(request: Request) -> dict: + """列出所有已绑定人脸""" + plugin = getattr(request.app.state, "video_call_plugin", None) + if not plugin: + return {"faces": []} + face_service = plugin.vision_service.face_recognition_service + return {"faces": face_service.list_known_faces()} diff --git a/echobot/plugins/video_call/services/__init__.py b/echobot/plugins/video_call/services/__init__.py new file mode 100644 index 0000000..e22f3e5 --- /dev/null +++ b/echobot/plugins/video_call/services/__init__.py @@ -0,0 +1,131 @@ +"""视频通话插件 - 视觉处理服务（双链路并行 + trace_id 追踪）""" + +from __future__ import annotations + +import asyncio +import os +import time +from itertools import count +from typing import Optional + +from ..models import Face, TimelineEvent, VisionContext +from .face_recognition import FaceRecognitionService +from .image_description import ImageDescriptionService + + +# 全局有序计数器，保证 trace_id 单调递增 +_trace_counter = count(1) + + +def _make_trace_id(ts: float) -> str: + """生成有序 trace_id：时间戳_序号""" + seq = next(_trace_counter) + return f"{ts:.3f}_{seq:06d}" + + +class VisionProcessingService: + """双链路并行：图像描述链路 + 人脸识别链路，通过 trace_id 拼接""" + + def __init__(self) -> None: + self.image_desc_service = ImageDescriptionService() + self.face_recognition_service = FaceRecognitionService( + feature_match_threshold=float( + os.environ.get("ECHOBOT_VIDEO_FEATURE_MATCH_THRESHOLD", "0.4") + ) + ) + self._initialized = False + # 图像描述节流 + self._last_desc_time: float = 0.0 + self._last_description: str = "" + self._desc_interval: float = float( + os.environ.get("ECHOBOT_VIDEO_DESC_INTERVAL", "4") + ) + + async def initialize(self) -> None: + if self._initialized: + return + await asyncio.to_thread(self._load_models) + self._initialized = True + + def _load_models(self) -> None: + self.image_desc_service.initialize() + self.face_recognition_service.initialize() + + async def process_frame(self, frame_bytes: bytes) -> VisionContext: + """双链路并行处理，通过 trace_id 追踪和拼接结果""" + if not self._initialized: + await self.initialize() + + current_time = time.time() + trace_id = _make_trace_id(current_time) + + # 双链路并行 + image_desc_task = asyncio.to_thread( + self._describe_image, frame_bytes, trace_id + ) + face_task = asyncio.to_thread( + self._detect_faces, frame_bytes, trace_id + ) + + image_desc, faces = await asyncio.gather(image_desc_task, face_task) + + # 用人脸识别结果增强描述 + enriched_desc = _enrich_description(image_desc, faces) + + timeline = [ + TimelineEvent( + timestamp=current_time, + event_type="image_desc", + data={"trace_id": trace_id, "description": image_desc}, + ), + TimelineEvent( + timestamp=current_time, + event_type="face_detect", + data={"trace_id": trace_id, "faces": len(faces)}, + ), + ] + + return VisionContext( + trace_id=trace_id, + start_time=current_time, + end_time=current_time, + image_description=enriched_desc, + faces=faces, + timeline=timeline, + ) + + def _describe_image(self, frame_bytes: bytes, trace_id: str) -> str: + """图像描述链路，带节流""" + now = time.time() + if self._last_description and (now - self._last_desc_time) < self._desc_interval: + return self._last_description + description = self.image_desc_service.describe(frame_bytes) + self._last_desc_time = now + self._last_description = description + return description + + def _detect_faces(self, frame_bytes: bytes, trace_id: str) -> list[Face]: + """人脸识别链路""" + return self.face_recognition_service.detect_and_extract_features(frame_bytes) + + async def close(self) -> None: + self.image_desc_service.close() + self.face_recognition_service.close() + self._initialized = False + + +def _enrich_description(description: str, faces: list[Face]) -> str: + """将人脸识别结果拼接到描述中""" + if not faces: + return description + named = [f.person_name for f in faces if f.person_name] + unnamed_count = sum(1 for f in faces if not f.person_name) + parts = [] + if named: + parts.append(f"识别到：{', '.join(named)}") + if unnamed_count: + parts.append(f"另有 {unnamed_count} 张未知人脸") + if not parts: + return description + face_info = "；".join(parts) + return f"{description}【{face_info}】" diff --git a/echobot/plugins/video_call/services/face_recognition.py b/echobot/plugins/video_call/services/face_recognition.py new file mode 100644 index 0000000..abe2d3b --- /dev/null +++ b/echobot/plugins/video_call/services/face_recognition.py @@ -0,0 +1,173 @@ +"""视频通话插件 - 人脸识别服务（InsightFace + 特征向量 Map 映射）""" + +from __future__ import annotations + +import io +import uuid +from typing import Optional + +import numpy as np +from PIL import Image + +from ..models import Face + + +class FaceRecognitionService: + """ + 人脸识别服务： + 1. 用 InsightFace 检测人脸 + 提取 512 维特征向量 + 2. 与已知人脸 Map 做余弦相似度匹配 + 3. 有匹配 → 返回【向量 + 姓名】；无匹配 → 透传向量 + """ + + def __init__(self, feature_match_threshold: float = 0.4) -> None: + self._initialized = False + self._app = None # insightface FaceAnalysis + self._known: dict[str, np.ndarray] = {} # person_name -> 平均特征向量 + self._match_threshold = feature_match_threshold + + def initialize(self) -> None: + if self._initialized: + return + self._initialized = True + try: + import insightface + from insightface.app import FaceAnalysis + from loguru import logger + logger.info("Loading InsightFace buffalo_sc model (first run may download ~85MB)...") + self._app = FaceAnalysis( + name="buffalo_sc", # 轻量模型，速度快 + allowed_modules=["detection", "recognition"], + providers=["CPUExecutionProvider"], + ) + self._app.prepare(ctx_id=-1, det_size=(320, 320)) # 小尺寸更快 + logger.info("InsightFace buffalo_sc model ready") + except Exception as e: + from loguru import logger + logger.warning(f"InsightFace init failed, face recognition disabled: {e}") + self._app = None + + def detect_and_extract_features( + self, frame_bytes: bytes, confidence_threshold: float = 0.5 + ) -> list[Face]: + """检测人脸，提取特征向量，查 Map 映射人名""" + if not self._app: + return [] + try: + img = _bytes_to_bgr(frame_bytes) + if img is None: + return [] + faces_raw = self._app.get(img) + if not faces_raw: + return [] + + result: list[Face] = [] + for i, f in enumerate(faces_raw): + det_score = float(getattr(f, "det_score", 0.0)) + if det_score < confidence_threshold: + continue + + embedding = getattr(f, "embedding", None) + if embedding is None: + continue + + vec = np.array(embedding, dtype=np.float32) + vec = vec / (np.linalg.norm(vec) + 1e-8) # L2 归一化 + + # 查 Map + person_name, score = self._match(vec) + + bbox = getattr(f, "bbox", [0, 0, 0, 0]) + result.append(Face( + face_id=str(uuid.uuid4())[:8], + person_name=person_name, + confidence=det_score, + features=vec.tolist(), + position={ + "x": int(bbox[0]), "y": int(bbox[1]), + "w": int(bbox[2] - bbox[0]), + "h": int(bbox[3] - bbox[1]), + "match_score": round(score, 3) if person_name else None, + }, + )) + return result + except Exception as e: + from loguru import logger + logger.warning(f"Face detection failed: {e}") + return [] + + def add_known_face( + self, person_name: str, features: list[float] | np.ndarray + ) -> bool: + """添加已知人脸向量（支持多次添加，取平均向量）""" + if not person_name: + return False + vec = np.array(features, dtype=np.float32) + vec = vec / (np.linalg.norm(vec) + 1e-8) + if person_name in self._known: + # 滑动平均 + self._known[person_name] = ( + self._known[person_name] * 0.7 + vec * 0.3 + ) + self._known[person_name] /= ( + np.linalg.norm(self._known[person_name]) + 1e-8 + ) + else: + self._known[person_name] = vec + return True + + def add_known_face_from_frame( + self, person_name: str, frame_bytes: bytes + ) -> bool: + """从帧图像中提取人脸向量并绑定姓名""" + faces = self.detect_and_extract_features(frame_bytes) + if not faces: + return False + # 取置信度最高的人脸 + best = max(faces, key=lambda f: f.confidence) + if not best.features: + return False + return self.add_known_face(person_name, best.features) + + def list_known_faces(self) -> list[str]: + return list(self._known.keys()) + + def remove_known_face(self, person_name: str) -> bool: + if person_name in self._known: + del self._known[person_name] + return True + return False + + def _match(self, vec: np.ndarray) -> tuple[str | None, float]: + """余弦相似度匹配，返回 (person_name, score)""" + best_name = None + best_score = 0.0 + for name, known_vec in self._known.items(): + score = float(np.dot(vec, known_vec)) + if score > best_score: + best_score = score + best_name = name + if best_score >= self._match_threshold: + return best_name, best_score + return None, best_score + + def close(self) -> None: + self._known.clear() + self._app = None + self._initialized = False + + +def _bytes_to_bgr(frame_bytes: bytes): + """将 JPEG bytes 转为 BGR numpy array（insightface 需要 BGR）""" + try: + import cv2 + arr = np.frombuffer(frame_bytes, dtype=np.uint8) + img = cv2.imdecode(arr, cv2.IMREAD_COLOR) + return img + except ImportError: + # 没有 cv2，用 PIL 转 + img_pil = Image.open(io.BytesIO(frame_bytes)).convert("RGB") + arr = np.array(img_pil)[:, :, ::-1].copy() # RGB -> BGR + return arr + except Exception: + return None diff --git a/echobot/plugins/video_call/services/image_description.py b/echobot/plugins/video_call/services/image_description.py new file mode 100644 index 0000000..3c8f83f --- /dev/null +++ b/echobot/plugins/video_call/services/image_description.py @@ -0,0 +1,120 @@ +"""视频通话插件 - 图像描述服务""" + +from __future__ import annotations + +import base64 +import io +import os +from typing import Optional + +from PIL import Image + + +class ImageDescriptionService: + """图像描述服务 - 使用 MiniMax Vision，降级返回基础描述""" + + def __init__(self, model_name: str = "auto") -> None: + self.model_name = model_name + self._initialized = False + self._use_llm = False + self._client = None + self._llm_model = None + self._llm_base_url = None + self._call_count = 0 + self._describe_every = 5 # 每5帧调用一次 Vision API + self._last_description = "" + + def initialize(self) -> None: + """初始化服务""" + if self._initialized: + return + self._initialized = True + + # 图像描述用 VISION_* 配置（MiniMax），降级到 LLM_* + api_key = os.environ.get("VISION_API_KEY") or os.environ.get("LLM_API_KEY", "") + base_url = os.environ.get("VISION_BASE_URL") or os.environ.get("LLM_BASE_URL", "") + model = os.environ.get("VISION_MODEL") or os.environ.get("LLM_MODEL", "") + + if not api_key: + return + + try: + import openai + self._client = openai.OpenAI( + api_key=api_key, + base_url=base_url or None, + ) + self._llm_model = model + self._use_llm = True + except ImportError: + pass + + def describe(self, frame_bytes: bytes, prompt: str = "用中文简洁描述画面内容：场景是什么、有没有人（如有请描述外貌特征：性别/发型/眼镜/衣着等）、有什么显眼的物品。不超过60字。") -> str: + """生成图像描述""" + if not self._initialized: + self.initialize() + + try: + image = Image.open(io.BytesIO(frame_bytes)).convert("RGB") + image.thumbnail((640, 480)) + buf = io.BytesIO() + image.save(buf, format="JPEG", quality=75) + compressed_bytes = buf.getvalue() + + if self._use_llm and self._client: + return self._describe_with_llm(compressed_bytes, prompt) + else: + return self._describe_fallback(image) + except Exception as e: + return f"画面处理异常: {e}" + + def _describe_with_llm(self, frame_bytes: bytes, prompt: str) -> str: + """用 LLM Vision 描述图像""" + try: + b64 = base64.b64encode(frame_bytes).decode("utf-8") + response = self._client.chat.completions.create( + model=self._llm_model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{b64}", + }, + }, + ], + } + ], + max_tokens=200, + ) + return response.choices[0].message.content.strip() + except Exception as e: + from loguru import logger + logger.warning(f"Vision LLM failed, using fallback: {e}") + try: + image = Image.open(io.BytesIO(frame_bytes)).convert("RGB") + return self._describe_fallback(image) + except Exception: + return "画面描述不可用" + + def _describe_fallback(self, image: Image.Image) -> str: + """降级描述：返回图像基础信息""" + import numpy as np + w, h = image.size + arr = np.array(image) + brightness = float(arr.mean()) + if brightness > 200: + light = "明亮" + elif brightness > 100: + light = "正常光线" + else: + light = "较暗" + return f"视频画面（{w}x{h}，{light}）" + + def close(self) -> None: + """清理资源""" + self._client = None + self._initialized = False diff --git a/echobot/plugins/video_call/tools/__init__.py b/echobot/plugins/video_call/tools/__init__.py new file mode 100644 index 0000000..8a17cc7 --- /dev/null +++ b/echobot/plugins/video_call/tools/__init__.py @@ -0,0 +1,45 @@ +"""视频通话插件 - 大模型工具定义""" + +VISION_TOOLS = [ + { + "name": "bind_face_to_name", + "description": ( + "将当前画面中检测到的人脸与指定姓名绑定。" + "你见到用户时并在上下文中获取到了用户姓，并且能确定不会是别人时调用此工具。" + "调用后，后续每次检测到该人脸都会自动识别为该姓名。" + ), + "parameters": { + "type": "object", + "properties": { + "person_name": { + "type": "string", + "description": "要绑定的人物姓名", + }, + }, + "required": ["person_name"], + }, + }, + { + "name": "list_known_faces", + "description": "列出所有已绑定姓名的人脸。当用户问'你认识哪些人'、'你记住了谁'时调用。", + "parameters": { + "type": "object", + "properties": {}, + "required": [], + }, + }, + { + "name": "forget_face", + "description": "删除某个人脸绑定。当用户说'忘掉XXX'、'删除XXX的人脸'时调用。", + "parameters": { + "type": "object", + "properties": { + "person_name": { + "type": "string", + "description": "要删除的人物姓名", + }, + }, + "required": ["person_name"], + }, + }, +] diff --git a/echobot/plugins/video_call/tools/face_tools.py b/echobot/plugins/video_call/tools/face_tools.py new file mode 100644 index 0000000..8233129 --- /dev/null +++ b/echobot/plugins/video_call/tools/face_tools.py @@ -0,0 +1,135 @@ +"""视频通话插件 - 人脸绑定工具（BaseTool 实现，可注册进 ToolRegistry）""" + +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +from ....tools.base import BaseTool, ToolOutput + +if TYPE_CHECKING: + from ..services import VisionProcessingService + + +class BindFaceToNameTool(BaseTool): + """ + 将当前画面中检测到的人脸与指定姓名绑定。 + 由 InsightFace 提取 512 维特征向量后写入人脸 Map。 + """ + name = "bind_face_to_name" + description = ( + "将当前摄像头画面中检测到的人脸与指定姓名绑定。" + "当用户说'这是XXX'、'把我绑定为XXX'、'记住我叫XXX'、'我叫XXX'时调用此工具。" + "绑定后，后续每次检测到该人脸都会自动识别为该姓名。" + ) + parameters = { + "type": "object", + "properties": { + "person_name": { + "type": "string", + "description": "要绑定的人物姓名", + }, + }, + "required": ["person_name"], + "additionalProperties": False, + } + + def __init__(self, plugin: Any) -> None: + self._plugin = plugin + + async def run(self, arguments: dict[str, Any]) -> ToolOutput: + person_name = str(arguments.get("person_name", "")).strip() + if not person_name: + raise ValueError("person_name 不能为空") + + import asyncio + face_service = self._plugin.vision_service.face_recognition_service + latest_frame = getattr(self._plugin, "_latest_frame", None) + + if not latest_frame: + return {"ok": False, "message": "没有摄像头画面，请先开启摄像头"} + + # 1. 提取特征并写入内存 + ok = await asyncio.to_thread( + face_service.add_known_face_from_frame, person_name, latest_frame + ) + if not ok: + return { + "ok": False, + "message": f"未能在当前画面中检测到人脸，请正对摄像头后重试", + } + + # 2. 同步写入 face_interceptor 持久化（JSON 文件） + face_interceptor = getattr(self._plugin, "face_interceptor", None) + if face_interceptor is not None: + vec = face_service._known.get(person_name) + if vec is not None: + await asyncio.to_thread( + face_interceptor.add_or_update_feature, + person_name, + vec.tolist(), + ) + + return { + "ok": True, + "message": f"已成功绑定：{person_name}，后续检测到该人脸时将自动识别", + } + + +class ListKnownFacesTool(BaseTool): + """列出所有已绑定姓名的人脸""" + name = "list_known_faces" + description = ( + "列出所有已通过人脸识别绑定姓名的人物。" + "当用户问'你认识哪些人'、'你记住了谁'时调用。" + ) + parameters = { + "type": "object", + "properties": {}, + "additionalProperties": False, + } + + def __init__(self, plugin: Any) -> None: + self._plugin = plugin + + async def run(self, arguments: dict[str, Any]) -> ToolOutput: + face_service = self._plugin.vision_service.face_recognition_service + names = face_service.list_known_faces() + return { + "faces": names, + "count": len(names), + "message": f"已记住 {len(names)} 个人：{', '.join(names)}" if names else "还没有记住任何人脸", + } + + +class ForgetFaceTool(BaseTool): + """删除某个人脸绑定""" + name = "forget_face" + description = ( + "删除某个人的人脸绑定记录。" + "当用户说'忘掉XXX'、'删除XXX的人脸记录'时调用。" + ) + parameters = { + "type": "object", + "properties": { + "person_name": { + "type": "string", + "description": "要删除的人物姓名", + }, + }, + "required": ["person_name"], + "additionalProperties": False, + } + + def __init__(self, plugin: Any) -> None: + self._plugin = plugin + + async def run(self, arguments: dict[str, Any]) -> ToolOutput: + person_name = str(arguments.get("person_name", "")).strip() + if not person_name: + raise ValueError("person_name 不能为空") + face_service = self._plugin.vision_service.face_recognition_service + ok = face_service.remove_known_face(person_name) + if ok: + return {"ok": True, "message": f"已删除 {person_name} 的人脸绑定"} + else: + return {"ok": False, "message": f"未找到 {person_name} 的人脸记录"} diff --git a/echobot/plugins/video_call/tools/handlers.py b/echobot/plugins/video_call/tools/handlers.py new file mode 100644 index 0000000..108dec8 --- /dev/null +++ b/echobot/plugins/video_call/tools/handlers.py @@ -0,0 +1,104 @@ +"""视频通话插件 - 工具调用处理器""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from ..interceptors import FaceFeatureInterceptor + from ..services import VisionProcessingService + + +class VisionToolHandler: + """视觉工具调用处理器""" + + def __init__( + self, + face_interceptor: "FaceFeatureInterceptor", + vision_service: "VisionProcessingService | None" = None, + plugin: Any = None, + ) -> None: + self.face_interceptor = face_interceptor + self.vision_service = vision_service + self.plugin = plugin # 用于访问 _latest_frame + + async def handle_tool_call( + self, tool_name: str, tool_input: dict[str, Any] + ) -> dict[str, Any]: + """处理工具调用""" + if tool_name == "bind_face_to_name": + return await self._handle_bind_face(tool_input) + elif tool_name == "list_known_faces": + return await self._handle_list_faces(tool_input) + elif tool_name == "forget_face": + return await self._handle_forget_face(tool_input) + # 兼容旧工具名 + elif tool_name == "add_face_feature": + return await self._handle_bind_face({"person_name": tool_input.get("person_name", "")}) + elif tool_name == "query_person_by_face": + return {"status": "deprecated", "message": "请使用 list_known_faces"} + else: + return {"error": f"Unknown tool: {tool_name}"} + + async def _handle_bind_face( + self, tool_input: dict[str, Any] + ) -> dict[str, Any]: + """从最新帧提取人脸向量并绑定姓名""" + person_name = str(tool_input.get("person_name", "")).strip() + if not person_name: + return {"error": "person_name 不能为空"} + + # 从最新帧提取真实向量 + face_service = None + if self.vision_service: + face_service = self.vision_service.face_recognition_service + + latest_frame = getattr(self.plugin, "_latest_frame", None) if self.plugin else None + + if face_service and latest_frame: + ok = await __import__('asyncio').to_thread( + face_service.add_known_face_from_frame, person_name, latest_frame + ) + if ok: + return { + "status": "success", + "message": f"已成功绑定：{person_name}，后续检测到该人脸时将自动识别", + } + else: + return { + "status": "warning", + "message": f"未能在当前画面中检测到人脸，已记录姓名 {person_name}，请正对摄像头后重试", + } + else: + return { + "status": "warning", + "message": f"已记录姓名 {person_name}，待摄像头连接后自动绑定", + } + + async def _handle_list_faces( + self, tool_input: dict[str, Any] + ) -> dict[str, Any]: + """列出已知人脸""" + if not self.vision_service: + return {"faces": []} + names = self.vision_service.face_recognition_service.list_known_faces() + return { + "faces": names, + "count": len(names), + "message": f"已记住 {len(names)} 个人：{', '.join(names)}" if names else "还没有记住任何人脸", + } + + async def _handle_forget_face( + self, tool_input: dict[str, Any] + ) -> dict[str, Any]: + """删除人脸绑定""" + person_name = str(tool_input.get("person_name", "")).strip() + if not person_name: + return {"error": "person_name 不能为空"} + if not self.vision_service: + return {"error": "视觉服务未启动"} + ok = self.vision_service.face_recognition_service.remove_known_face(person_name) + if ok: + return {"status": "success", "message": f"已删除 {person_name} 的人脸绑定"} + else: + return {"status": "not_found", "message": f"未找到 {person_name} 的人脸记录"} diff --git a/echobot/plugins/video_call/vision_provider.py b/echobot/plugins/video_call/vision_provider.py new file mode 100644 index 0000000..9e70ddf --- /dev/null +++ b/echobot/plugins/video_call/vision_provider.py @@ -0,0 +1,86 @@ +"""视频通话插件 - 视觉上下文提供器（时间段合并去重）""" + +from __future__ import annotations + +from collections import deque +from dataclasses import asdict +from typing import Optional + +from .models import VisionContext + + +# 描述相似度阈值：两段描述相似度超过此值视为「未变化」 +_SIMILARITY_THRESHOLD = 0.7 + + +class VisionContextProvider: + """视觉上下文提供器 - 时间段合并去重，内容不变则延伸时间段""" + + def __init__(self, max_frames: int = 80) -> None: + self.max_frames = max_frames + self.frames: deque[dict] = deque(maxlen=max_frames) + + def add_vision_context(self, vision_context: VisionContext) -> None: + """添加视觉上下文，内容未变则延伸时间段，变化则插入新条目""" + new_desc = vision_context.image_description.strip() + new_faces = vision_context.faces + + if self.frames: + last = self.frames[-1] + last_desc = last.get("image_description", "").strip() + + # 检查描述是否发生变化 + if _is_similar(last_desc, new_desc): + # 未变化：延伸时间段 + last["end_time"] = vision_context.end_time + last["trace_id_end"] = vision_context.trace_id + # 更新人脸信息（可能新增了识别结果） + if new_faces: + last["faces"] = _faces_to_dict(new_faces) + last["face_count"] = len(new_faces) + return + + # 描述变化或首帧：插入新条目 + frame_data = { + "trace_id": vision_context.trace_id, + "trace_id_end": vision_context.trace_id, + "start_time": vision_context.start_time, + "end_time": vision_context.end_time, + # 兼容旧字段 + "timestamp": vision_context.start_time, + "image_description": new_desc, + "face_count": len(new_faces), + "faces": _faces_to_dict(new_faces), + } + self.frames.append(frame_data) + + def get_vision_context_list(self) -> list[dict]: + """返回所有时间段条目，按 start_time 升序""" + return list(self.frames) + + +def _is_similar(a: str, b: str) -> bool: + """判断两段描述是否相似（词袋 Jaccard 相似度）""" + if not a or not b: + return False + # 完全相同快速返回 + if a == b: + return True + words_a = set(a) + words_b = set(b) + if not words_a or not words_b: + return False + intersection = len(words_a & words_b) + union = len(words_a | words_b) + return (intersection / union) >= _SIMILARITY_THRESHOLD + + +def _faces_to_dict(faces) -> list[dict]: + return [ + { + "face_id": f.face_id, + "person_name": f.person_name, + "confidence": f.confidence, + } + for f in faces + ] diff --git a/echobot/plugins/video_call/web/video-call.css b/echobot/plugins/video_call/web/video-call.css new file mode 100644 index 0000000..b4884f2 --- /dev/null +++ b/echobot/plugins/video_call/web/video-call.css @@ -0,0 +1,101 @@ +/* EchoBot 视频通话样式 */ + +.video-call-controls { + display: flex; + gap: 10px; + padding: 12px; + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + border-radius: 8px; + margin-bottom: 12px; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); +} + +.video-call-controls button { + flex: 1; + padding: 10px 16px; + border: none; + border-radius: 6px; + font-size: 14px; + font-weight: 600; + cursor: pointer; + transition: all 0.3s ease; +} + +.video-call-controls .btn-primary { + background: #4CAF50; + color: white; +} + +.video-call-controls .btn-primary:hover:not(:disabled) { + background: #45a049; + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(76, 175, 80, 0.3); +} + +.video-call-controls .btn-primary:disabled { + background: #cccccc; + cursor: not-allowed; + opacity: 0.6; +} + +.video-call-controls .btn-danger { + background: #f44336; + color: white; +} + +.video-call-controls .btn-danger:hover:not(:disabled) { + background: #da190b; + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(244, 67, 54, 0.3); +} + +.video-call-controls .btn-danger:disabled { + background: #cccccc; + cursor: not-allowed; + opacity: 0.6; +} + +.video-status { + grid-column: 1 / -1; + padding: 12px; + background: rgba(255, 255, 255, 0.1); + border-radius: 6px; + color: white; + font-size: 13px; + line-height: 1.6; +} + +.vision-info { + margin: 0; +} + +.vision-info p { + margin: 4px 0; + display: flex; + align-items: center; + gap: 8px; +} + +.vision-info p:first-child { + font-weight: 600; + color: #fff; +} + +.vision-info p:last-child { + color: #e0e0e0; + font-size: 12px; +} + +@media (max-width: 768px) { + .video-call-controls { + flex-direction: column; + } + + .video-call-controls button { + width: 100%; + } + + .video-status { + font-size: 12px; + } +} diff --git a/echobot/plugins/video_call/web/video-call.js b/echobot/plugins/video_call/web/video-call.js new file mode 100644 index 0000000..1dc7a6a --- /dev/null +++ b/echobot/plugins/video_call/web/video-call.js @@ -0,0 +1,240 @@ +/** + * EchoBot 视频通话模块 + * 处理视频采集、WebSocket 通信和 UI 集成 + */ + +export function createVideoCallModule(deps) { + const { + addSystemMessage, + requestJson, + responseToError, + sendMessage, + } = deps; + + let videoStream = null; + let videoCanvas = null; + let videoContext = null; + let websocket = null; + let isRecording = false; + let frameRate = 1; // 每秒采集 1 帧 + + function initialize() { + createVideoControls(); + setupEventListeners(); + } + + function createVideoControls() { + const controlsDiv = document.getElementById('video-call-controls'); + if (!controlsDiv) { + const inputArea = document.querySelector('.input-area') || + document.querySelector('.message-input-area') || + document.querySelector('[class*="input"]'); + + if (inputArea) { + const newDiv = document.createElement('div'); + newDiv.id = 'video-call-controls'; + newDiv.className = 'video-call-controls'; + newDiv.innerHTML = ` + + +
+ `; + inputArea.parentNode.insertBefore(newDiv, inputArea); + } + } + } + + function setupEventListeners() { + const startBtn = document.getElementById('start-video-call'); + const stopBtn = document.getElementById('stop-video-call'); + + if (startBtn) { + startBtn.addEventListener('click', startVideoCall); + } + if (stopBtn) { + stopBtn.addEventListener('click', stopVideoCall); + } + } + + async function startVideoCall() { + try { + videoStream = await navigator.mediaDevices.getUserMedia({ + video: { width: 1280, height: 720 }, + audio: false + }); + + videoCanvas = document.createElement('canvas'); + videoCanvas.width = 1280; + videoCanvas.height = 720; + videoContext = videoCanvas.getContext('2d'); + + const videoElement = document.createElement('video'); + videoElement.srcObject = videoStream; + videoElement.play(); + + await connectWebSocket(); + + isRecording = true; + captureFrames(videoElement); + + updateVideoUI(true); + addSystemMessage('✓ 视频通话已启动，每秒发送一帧图片'); + } catch (error) { + addSystemMessage(`✗ 启动视频失败: ${error.message}`); + } + } + + function stopVideoCall() { + isRecording = false; + + if (videoStream) { + videoStream.getTracks().forEach(track => track.stop()); + videoStream = null; + } + + if (websocket) { + websocket.close(); + websocket = null; + } + + updateVideoUI(false); + addSystemMessage('✓ 视频通话已停止'); + } + + async function connectWebSocket() { + return new Promise((resolve, reject) => { + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + const wsUrl = `${protocol}//${window.location.host}/api/web/video/stream`; + + websocket = new WebSocket(wsUrl); + + websocket.onopen = () => { + console.log('✓ WebSocket connected'); + resolve(); + }; + + websocket.onmessage = (event) => { + handleVisionContext(JSON.parse(event.data)); + }; + + websocket.onerror = (error) => { + console.error('WebSocket error:', error); + reject(error); + }; + + websocket.onclose = () => { + console.log('WebSocket closed'); + }; + }); + } + + function captureFrames(videoElement) { + const interval = 1000 / frameRate; // 毫秒 + + const captureFrame = () => { + if (!isRecording) return; + + try { + videoContext.drawImage(videoElement, 0, 0, videoCanvas.width, videoCanvas.height); + + videoCanvas.toBlob((blob) => { + if (websocket && websocket.readyState === WebSocket.OPEN) { + websocket.send(blob); + } + }, 'image/jpeg', 0.8); + } catch (error) { + console.error('Frame capture error:', error); + } + + setTimeout(captureFrame, interval); + }; + + captureFrame(); + } + + function handleVisionContext(visionContext) { + const statusDiv = document.getElementById('video-status'); + if (statusDiv) { + const faceCount = visionContext.faces ? visionContext.faces.length : 0; + const description = visionContext.image_description || '处理中...'; + + statusDiv.innerHTML = ` +
+

📸 ${description.substring(0, 60)}...

+

👤 检测到 ${faceCount} 张人脸

+
+ `; + } + + // 将图像描述插入到消息上下文 + if (visionContext.image_description) { + insertVisionToContext(visionContext); + } + + // 如果检测到人脸，显示人脸信息 + if (visionContext.faces && visionContext.faces.length > 0) { + visionContext.faces.forEach(face => { + if (face.person_name) { + console.log(`✓ 识别到: ${face.person_name} (置信度: ${face.confidence.toFixed(2)})`); + } + }); + } + } + + function insertVisionToContext(visionContext) { + // 构建视觉信息文本 + const timestamp = new Date(visionContext.timestamp * 1000).toLocaleTimeString(); + const faceInfo = visionContext.faces && visionContext.faces.length > 0 + ? `检测到 ${visionContext.faces.length} 张人脸: ${visionContext.faces.map(f => f.person_name || '未知').join(', ')}` + : '未检测到人脸'; + + const visionText = `[视觉信息 ${timestamp}] ${visionContext.image_description} | ${faceInfo}`; + + // 这里可以将视觉信息存储到全局上下文中 + // 供大模型在生成回复时使用 + if (window.__visionContext) { + window.__visionContext.push({ + timestamp: visionContext.timestamp, + description: visionContext.image_description, + faces: visionContext.faces, + }); + // 保持最近 80 条 + if (window.__visionContext.length > 80) { + window.__visionContext.shift(); + } + } else { + window.__visionContext = [{ + timestamp: visionContext.timestamp, + description: visionContext.image_description, + faces: visionContext.faces, + }]; + } + + console.log(`✓ 视觉上下文已更新 (总计: ${window.__visionContext.length} 帧)`); + } + + function updateVideoUI(isActive) { + const startBtn = document.getElementById('start-video-call'); + const stopBtn = document.getElementById('stop-video-call'); + + if (startBtn) { + startBtn.disabled = isActive; + startBtn.style.display = isActive ? 'none' : 'block'; + } + if (stopBtn) { + stopBtn.disabled = !isActive; + stopBtn.style.display = isActive ? 'block' : 'none'; + } + } + + return { + initialize, + startVideoCall, + stopVideoCall, + }; +} + diff --git a/echobot_client/.gitignore b/echobot_client/.gitignore new file mode 100644 index 0000000..3820a95 --- /dev/null +++ b/echobot_client/.gitignore @@ -0,0 +1,45 @@ +# Miscellaneous +*.class +*.log +*.pyc +*.swp +.DS_Store +.atom/ +.build/ +.buildlog/ +.history +.svn/ +.swiftpm/ +migrate_working_dir/ + +# IntelliJ related +*.iml +*.ipr +*.iws +.idea/ + +# The .vscode folder contains launch configuration and tasks you configure in +# VS Code which you may wish to be included in version control, so this line +# is commented out by default. +#.vscode/ + +# Flutter/Dart/Pub related +**/doc/api/ +**/ios/Flutter/.last_build_id +.dart_tool/ +.flutter-plugins-dependencies +.pub-cache/ +.pub/ +/build/ +/coverage/ + +# Symbolication related +app.*.symbols + +# Obfuscation related +app.*.map.json + +# Android Studio will place build artifacts here +/android/app/debug +/android/app/profile +/android/app/release diff --git a/echobot_client/.metadata b/echobot_client/.metadata new file mode 100644 index 0000000..df13aa7 --- /dev/null +++ b/echobot_client/.metadata @@ -0,0 +1,30 @@ +# This file tracks properties of this Flutter project. +# Used by Flutter tool to assess capabilities and perform upgrades etc. +# +# This file should be version controlled and should not be manually edited. + +version: + revision: "ff37bef603469fb030f2b72995ab929ccfc227f0" + channel: "stable" + +project_type: app + +# Tracks metadata for the flutter migrate command +migration: + platforms: + - platform: root + create_revision: ff37bef603469fb030f2b72995ab929ccfc227f0 + base_revision: ff37bef603469fb030f2b72995ab929ccfc227f0 + - platform: android + create_revision: ff37bef603469fb030f2b72995ab929ccfc227f0 + base_revision: ff37bef603469fb030f2b72995ab929ccfc227f0 + + # User provided section + + # List of Local paths (relative to this file) that should be + # ignored by the migrate tool. + # + # Files that are not part of the templates will be ignored by default. + unmanaged_files: + - 'lib/main.dart' + - 'ios/Runner.xcodeproj/project.pbxproj' diff --git a/echobot_client/README.md b/echobot_client/README.md new file mode 100644 index 0000000..d274e7d --- /dev/null +++ b/echobot_client/README.md @@ -0,0 +1,17 @@ +# echobot_client + +A new Flutter project. + +## Getting Started + +This project is a starting point for a Flutter application. + +A few resources to get you started if this is your first Flutter project: + +- [Learn Flutter](https://docs.flutter.dev/get-started/learn-flutter) +- [Write your first Flutter app](https://docs.flutter.dev/get-started/codelab) +- [Flutter learning resources](https://docs.flutter.dev/reference/learning-resources) + +For help getting started with Flutter development, view the +[online documentation](https://docs.flutter.dev/), which offers tutorials, +samples, guidance on mobile development, and a full API reference. diff --git a/echobot_client/analysis_options.yaml b/echobot_client/analysis_options.yaml new file mode 100644 index 0000000..0d29021 --- /dev/null +++ b/echobot_client/analysis_options.yaml @@ -0,0 +1,28 @@ +# This file configures the analyzer, which statically analyzes Dart code to +# check for errors, warnings, and lints. +# +# The issues identified by the analyzer are surfaced in the UI of Dart-enabled +# IDEs (https://dart.dev/tools#ides-and-editors). The analyzer can also be +# invoked from the command line by running `flutter analyze`. + +# The following line activates a set of recommended lints for Flutter apps, +# packages, and plugins designed to encourage good coding practices. +include: package:flutter_lints/flutter.yaml + +linter: + # The lint rules applied to this project can be customized in the + # section below to disable rules from the `package:flutter_lints/flutter.yaml` + # included above or to enable additional rules. A list of all available lints + # and their documentation is published at https://dart.dev/lints. + # + # Instead of disabling a lint rule for the entire project in the + # section below, it can also be suppressed for a single line of code + # or a specific dart file by using the `// ignore: name_of_lint` and + # `// ignore_for_file: name_of_lint` syntax on the line or in the file + # producing the lint. + rules: + # avoid_print: false # Uncomment to disable the `avoid_print` rule + # prefer_single_quotes: true # Uncomment to enable the `prefer_single_quotes` rule + +# Additional information about this file can be found at +# https://dart.dev/guides/language/analysis-options diff --git a/echobot_client/android/.gitignore b/echobot_client/android/.gitignore new file mode 100644 index 0000000..48354a3 --- /dev/null +++ b/echobot_client/android/.gitignore @@ -0,0 +1,101 @@ +# Using Android gitignore template: https://github.com/github/gitignore/blob/HEAD/Android.gitignore + +# Built application files +*.apk +*.aar +*.ap_ +*.aab + +# Files for the ART/Dalvik VM +*.dex + +# Java class files +*.class + +# Generated files +bin/ +gen/ +out/ +# Uncomment the following line in case you need and you don't have the release build type files in your app +# release/ + +# Gradle files +.gradle/ +build/ + +# Local configuration file (sdk path, etc) +local.properties + +# Proguard folder generated by Eclipse +proguard/ + +# Log Files +*.log + +# Android Studio Navigation editor temp files +.navigation/ + +# Android Studio captures folder +captures/ + +# IntelliJ +*.iml +.idea/workspace.xml +.idea/tasks.xml +.idea/gradle.xml +.idea/assetWizardSettings.xml +.idea/dictionaries +.idea/libraries +# Android Studio 3 in .gitignore file. +.idea/caches +.idea/modules.xml +# Comment next line if keeping position of elements in Navigation Editor is relevant for you +.idea/navEditor.xml + +# Keystore files +# Uncomment the following lines if you do not want to check your keystore files in. +#*.jks +#*.keystore + +# External native build folder generated in Android Studio 2.2 and later +.externalNativeBuild +.cxx/ + +# Google Services (e.g. APIs or Firebase) +# google-services.json + +# Freeline +freeline.py +freeline/ +freeline_project_description.json + +# fastlane +fastlane/report.xml +fastlane/Preview.html +fastlane/screenshots +fastlane/test_output +fastlane/readme.md + +# Version control +vcs.xml + +# lint +lint/intermediates/ +lint/generated/ +lint/outputs/ +lint/tmp/ +# lint/reports/ + +# Android Profiling +*.hprof + +# Cordova plugins for Capacitor +capacitor-cordova-android-plugins + +# Copied web assets +app/src/main/assets/public + +# Generated Config files +app/src/main/assets/capacitor.config.json +app/src/main/assets/capacitor.plugins.json +app/src/main/res/xml/config.xml diff --git a/echobot_client/android/app/.gitignore b/echobot_client/android/app/.gitignore new file mode 100644 index 0000000..043df80 --- /dev/null +++ b/echobot_client/android/app/.gitignore @@ -0,0 +1,2 @@ +/build/* +!/build/.npmkeep diff --git a/echobot_client/android/app/build.gradle b/echobot_client/android/app/build.gradle new file mode 100644 index 0000000..ca1bffc --- /dev/null +++ b/echobot_client/android/app/build.gradle @@ -0,0 +1,54 @@ +apply plugin: 'com.android.application' + +android { + namespace = "com.echobot.app" + compileSdk = rootProject.ext.compileSdkVersion + defaultConfig { + applicationId "com.echobot.app" + minSdkVersion rootProject.ext.minSdkVersion + targetSdkVersion rootProject.ext.targetSdkVersion + versionCode 1 + versionName "1.0" + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + aaptOptions { + // Files and dirs to omit from the packaged assets dir, modified to accommodate modern web apps. + // Default: https://android.googlesource.com/platform/frameworks/base/+/282e181b58cf72b6ca770dc7ca5f91f135444502/tools/aapt/AaptAssets.cpp#61 + ignoreAssetsPattern = '!.svn:!.git:!.ds_store:!*.scc:.*:!CVS:!thumbs.db:!picasa.ini:!*~' + } + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' + } + } +} + +repositories { + flatDir{ + dirs '../capacitor-cordova-android-plugins/src/main/libs', 'libs' + } +} + +dependencies { + implementation fileTree(include: ['*.jar'], dir: 'libs') + implementation "androidx.appcompat:appcompat:$androidxAppCompatVersion" + implementation "androidx.coordinatorlayout:coordinatorlayout:$androidxCoordinatorLayoutVersion" + implementation "androidx.core:core-splashscreen:$coreSplashScreenVersion" + implementation project(':capacitor-android') + testImplementation "junit:junit:$junitVersion" + androidTestImplementation "androidx.test.ext:junit:$androidxJunitVersion" + androidTestImplementation "androidx.test.espresso:espresso-core:$androidxEspressoCoreVersion" + implementation project(':capacitor-cordova-android-plugins') +} + +apply from: 'capacitor.build.gradle' + +try { + def servicesJSON = file('google-services.json') + if (servicesJSON.text) { + apply plugin: 'com.google.gms.google-services' + } +} catch(Exception e) { + logger.info("google-services.json not found, google-services plugin not applied. Push Notifications won't work") +} diff --git a/echobot_client/android/app/capacitor.build.gradle b/echobot_client/android/app/capacitor.build.gradle new file mode 100644 index 0000000..bbfb44f --- /dev/null +++ b/echobot_client/android/app/capacitor.build.gradle @@ -0,0 +1,19 @@ +// DO NOT EDIT THIS FILE! IT IS GENERATED EACH TIME "capacitor update" IS RUN + +android { + compileOptions { + sourceCompatibility JavaVersion.VERSION_21 + targetCompatibility JavaVersion.VERSION_21 + } +} + +apply from: "../capacitor-cordova-android-plugins/cordova.variables.gradle" +dependencies { + + +} + + +if (hasProperty('postBuildExtras')) { + postBuildExtras() +} diff --git a/echobot_client/android/app/proguard-rules.pro b/echobot_client/android/app/proguard-rules.pro new file mode 100644 index 0000000..f1b4245 --- /dev/null +++ b/echobot_client/android/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/echobot_client/android/app/src/androidTest/java/com/getcapacitor/myapp/ExampleInstrumentedTest.java b/echobot_client/android/app/src/androidTest/java/com/getcapacitor/myapp/ExampleInstrumentedTest.java new file mode 100644 index 0000000..f2c2217 --- /dev/null +++ b/echobot_client/android/app/src/androidTest/java/com/getcapacitor/myapp/ExampleInstrumentedTest.java @@ -0,0 +1,26 @@ +package com.getcapacitor.myapp; + +import static org.junit.Assert.*; + +import android.content.Context; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Instrumented test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + + @Test + public void useAppContext() throws Exception { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); + + assertEquals("com.getcapacitor.app", appContext.getPackageName()); + } +} diff --git a/echobot_client/android/app/src/main/AndroidManifest.xml b/echobot_client/android/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..229e718 --- /dev/null +++ b/echobot_client/android/app/src/main/AndroidManifest.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/echobot_client/android/app/src/main/java/com/echobot/app/MainActivity.java b/echobot_client/android/app/src/main/java/com/echobot/app/MainActivity.java new file mode 100644 index 0000000..98c5705 --- /dev/null +++ b/echobot_client/android/app/src/main/java/com/echobot/app/MainActivity.java @@ -0,0 +1,98 @@ +package com.echobot.app; + +import android.Manifest; +import android.content.pm.PackageManager; +import android.os.Bundle; +import android.webkit.PermissionRequest; +import android.webkit.WebChromeClient; +import android.webkit.WebView; + +import androidx.core.app.ActivityCompat; +import androidx.core.content.ContextCompat; + +import com.getcapacitor.BridgeActivity; + +public class MainActivity extends BridgeActivity { + + private static final int CAMERA_PERMISSION_REQUEST = 1001; + private PermissionRequest pendingPermissionRequest; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + } + + @Override + public void onResume() { + super.onResume(); + setupWebViewCamera(); + } + + private void setupWebViewCamera() { + WebView webView = getBridge().getWebView(); + webView.getSettings().setMediaPlaybackRequiresUserGesture(false); + webView.getSettings().setJavaScriptEnabled(true); + // 允许非安全上下文（http://）使用 getUserMedia + if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.O) { + webView.getSettings().setMixedContentMode( + android.webkit.WebSettings.MIXED_CONTENT_ALWAYS_ALLOW); + } + // 注入脚本：在页面加载前覆盖安全检查 + webView.addJavascriptInterface(new Object() { + @android.webkit.JavascriptInterface + public boolean isAndroid() { return true; } + }, "AndroidBridge"); + webView.setWebChromeClient(new WebChromeClient() { + @Override + public void onPermissionRequest(final PermissionRequest request) { + runOnUiThread(() -> { + String[] requestedResources = request.getResources(); + boolean needCamera = false; + for (String r : requestedResources) { + if (r.equals(PermissionRequest.RESOURCE_VIDEO_CAPTURE) + || r.equals(PermissionRequest.RESOURCE_AUDIO_CAPTURE)) { + needCamera = true; + break; + } + } + if (needCamera) { + if (ContextCompat.checkSelfPermission( + MainActivity.this, Manifest.permission.CAMERA) + == PackageManager.PERMISSION_GRANTED) { + request.grant(request.getResources()); + } else { + pendingPermissionRequest = request; + ActivityCompat.requestPermissions( + MainActivity.this, + new String[]{ + Manifest.permission.CAMERA, + Manifest.permission.RECORD_AUDIO + }, + CAMERA_PERMISSION_REQUEST + ); + } + } else { + request.grant(request.getResources()); + } + }); + } + }); + } + + @Override + public void onRequestPermissionsResult( + int requestCode, + String[] permissions, + int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + if (requestCode == CAMERA_PERMISSION_REQUEST && pendingPermissionRequest != null) { + if (grantResults.length > 0 + && grantResults[0] == PackageManager.PERMISSION_GRANTED) { + pendingPermissionRequest.grant(pendingPermissionRequest.getResources()); + } else { + pendingPermissionRequest.deny(); + } + pendingPermissionRequest = null; + } + } +} diff --git a/echobot_client/android/app/src/main/res/drawable-land-hdpi/splash.png b/echobot_client/android/app/src/main/res/drawable-land-hdpi/splash.png new file mode 100644 index 0000000..e31573b Binary files /dev/null and b/echobot_client/android/app/src/main/res/drawable-land-hdpi/splash.png differ diff --git a/echobot_client/android/app/src/main/res/drawable-land-mdpi/splash.png b/echobot_client/android/app/src/main/res/drawable-land-mdpi/splash.png new file mode 100644 index 0000000..f7a6492 Binary files /dev/null and b/echobot_client/android/app/src/main/res/drawable-land-mdpi/splash.png differ diff --git a/echobot_client/android/app/src/main/res/drawable-land-xhdpi/splash.png b/echobot_client/android/app/src/main/res/drawable-land-xhdpi/splash.png new file mode 100644 index 0000000..8077255 Binary files /dev/null and b/echobot_client/android/app/src/main/res/drawable-land-xhdpi/splash.png differ diff --git a/echobot_client/android/app/src/main/res/drawable-land-xxhdpi/splash.png b/echobot_client/android/app/src/main/res/drawable-land-xxhdpi/splash.png new file mode 100644 index 0000000..14c6c8f Binary files /dev/null and b/echobot_client/android/app/src/main/res/drawable-land-xxhdpi/splash.png differ diff --git a/echobot_client/android/app/src/main/res/drawable-land-xxxhdpi/splash.png b/echobot_client/android/app/src/main/res/drawable-land-xxxhdpi/splash.png new file mode 100644 index 0000000..244ca25 Binary files /dev/null and b/echobot_client/android/app/src/main/res/drawable-land-xxxhdpi/splash.png differ diff --git a/echobot_client/android/app/src/main/res/drawable-port-hdpi/splash.png b/echobot_client/android/app/src/main/res/drawable-port-hdpi/splash.png new file mode 100644 index 0000000..74faaa5 Binary files /dev/null and b/echobot_client/android/app/src/main/res/drawable-port-hdpi/splash.png differ diff --git a/echobot_client/android/app/src/main/res/drawable-port-mdpi/splash.png b/echobot_client/android/app/src/main/res/drawable-port-mdpi/splash.png new file mode 100644 index 0000000..e944f4a Binary files /dev/null and b/echobot_client/android/app/src/main/res/drawable-port-mdpi/splash.png differ diff --git a/echobot_client/android/app/src/main/res/drawable-port-xhdpi/splash.png b/echobot_client/android/app/src/main/res/drawable-port-xhdpi/splash.png new file mode 100644 index 0000000..564a82f Binary files /dev/null and b/echobot_client/android/app/src/main/res/drawable-port-xhdpi/splash.png differ diff --git a/echobot_client/android/app/src/main/res/drawable-port-xxhdpi/splash.png b/echobot_client/android/app/src/main/res/drawable-port-xxhdpi/splash.png new file mode 100644 index 0000000..bfabe68 Binary files /dev/null and b/echobot_client/android/app/src/main/res/drawable-port-xxhdpi/splash.png differ diff --git a/echobot_client/android/app/src/main/res/drawable-port-xxxhdpi/splash.png b/echobot_client/android/app/src/main/res/drawable-port-xxxhdpi/splash.png new file mode 100644 index 0000000..6929071 Binary files /dev/null and b/echobot_client/android/app/src/main/res/drawable-port-xxxhdpi/splash.png differ diff --git a/echobot_client/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/echobot_client/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 0000000..c7bd21d --- /dev/null +++ b/echobot_client/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + diff --git a/echobot_client/android/app/src/main/res/drawable/ic_launcher_background.xml b/echobot_client/android/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 0000000..d5fccc5 --- /dev/null +++ b/echobot_client/android/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/echobot_client/android/app/src/main/res/drawable/splash.png b/echobot_client/android/app/src/main/res/drawable/splash.png new file mode 100644 index 0000000..f7a6492 Binary files /dev/null and b/echobot_client/android/app/src/main/res/drawable/splash.png differ diff --git a/echobot_client/android/app/src/main/res/layout/activity_main.xml b/echobot_client/android/app/src/main/res/layout/activity_main.xml new file mode 100644 index 0000000..b5ad138 --- /dev/null +++ b/echobot_client/android/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,12 @@ + + + + + diff --git a/echobot_client/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/echobot_client/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 0000000..036d09b --- /dev/null +++ b/echobot_client/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/echobot_client/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/echobot_client/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml new file mode 100644 index 0000000..036d09b --- /dev/null +++ b/echobot_client/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/echobot_client/android/app/src/main/res/mipmap-hdpi/ic_launcher.png b/echobot_client/android/app/src/main/res/mipmap-hdpi/ic_launcher.png new file mode 100644 index 0000000..c023e50 Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-hdpi/ic_launcher.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.png b/echobot_client/android/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.png new file mode 100644 index 0000000..2127973 Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png b/echobot_client/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png new file mode 100644 index 0000000..b441f37 Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-mdpi/ic_launcher.png b/echobot_client/android/app/src/main/res/mipmap-mdpi/ic_launcher.png new file mode 100644 index 0000000..72905b8 Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-mdpi/ic_launcher.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.png b/echobot_client/android/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.png new file mode 100644 index 0000000..8ed0605 Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png b/echobot_client/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png new file mode 100644 index 0000000..9502e47 Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png b/echobot_client/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png new file mode 100644 index 0000000..4d1e077 Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.png b/echobot_client/android/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.png new file mode 100644 index 0000000..df0f158 Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png b/echobot_client/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png new file mode 100644 index 0000000..853db04 Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png b/echobot_client/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png new file mode 100644 index 0000000..6cdf97c Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png b/echobot_client/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png new file mode 100644 index 0000000..2960cbb Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png b/echobot_client/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png new file mode 100644 index 0000000..8e3093a Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png b/echobot_client/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png new file mode 100644 index 0000000..46de6e2 Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png b/echobot_client/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png new file mode 100644 index 0000000..d2ea9ab Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png differ diff --git a/echobot_client/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png b/echobot_client/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png new file mode 100644 index 0000000..a40d73e Binary files /dev/null and b/echobot_client/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png differ diff --git a/echobot_client/android/app/src/main/res/values/ic_launcher_background.xml b/echobot_client/android/app/src/main/res/values/ic_launcher_background.xml new file mode 100644 index 0000000..c5d5899 --- /dev/null +++ b/echobot_client/android/app/src/main/res/values/ic_launcher_background.xml @@ -0,0 +1,4 @@ + + + #FFFFFF + \ No newline at end of file diff --git a/echobot_client/android/app/src/main/res/values/strings.xml b/echobot_client/android/app/src/main/res/values/strings.xml new file mode 100644 index 0000000..67dc8b9 --- /dev/null +++ b/echobot_client/android/app/src/main/res/values/strings.xml @@ -0,0 +1,7 @@ + + + EchoBot + EchoBot + com.echobot.app + com.echobot.app + diff --git a/echobot_client/android/app/src/main/res/values/styles.xml b/echobot_client/android/app/src/main/res/values/styles.xml new file mode 100644 index 0000000..be874e5 --- /dev/null +++ b/echobot_client/android/app/src/main/res/values/styles.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/echobot_client/android/app/src/main/res/xml/file_paths.xml b/echobot_client/android/app/src/main/res/xml/file_paths.xml new file mode 100644 index 0000000..bd0c4d8 --- /dev/null +++ b/echobot_client/android/app/src/main/res/xml/file_paths.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/echobot_client/android/app/src/test/java/com/getcapacitor/myapp/ExampleUnitTest.java b/echobot_client/android/app/src/test/java/com/getcapacitor/myapp/ExampleUnitTest.java new file mode 100644 index 0000000..0297327 --- /dev/null +++ b/echobot_client/android/app/src/test/java/com/getcapacitor/myapp/ExampleUnitTest.java @@ -0,0 +1,18 @@ +package com.getcapacitor.myapp; + +import static org.junit.Assert.*; + +import org.junit.Test; + +/** + * Example local unit test, which will execute on the development machine (host). + * + * @see Testing documentation + */ +public class ExampleUnitTest { + + @Test + public void addition_isCorrect() throws Exception { + assertEquals(4, 2 + 2); + } +} diff --git a/echobot_client/android/build.gradle b/echobot_client/android/build.gradle new file mode 100644 index 0000000..f8f0e43 --- /dev/null +++ b/echobot_client/android/build.gradle @@ -0,0 +1,29 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. + +buildscript { + + repositories { + google() + mavenCentral() + } + dependencies { + classpath 'com.android.tools.build:gradle:8.13.0' + classpath 'com.google.gms:google-services:4.4.4' + + // NOTE: Do not place your application dependencies here; they belong + // in the individual module build.gradle files + } +} + +apply from: "variables.gradle" + +allprojects { + repositories { + google() + mavenCentral() + } +} + +task clean(type: Delete) { + delete rootProject.buildDir +} diff --git a/echobot_client/android/capacitor.settings.gradle b/echobot_client/android/capacitor.settings.gradle new file mode 100644 index 0000000..9a5fa87 --- /dev/null +++ b/echobot_client/android/capacitor.settings.gradle @@ -0,0 +1,3 @@ +// DO NOT EDIT THIS FILE! IT IS GENERATED EACH TIME "capacitor update" IS RUN +include ':capacitor-android' +project(':capacitor-android').projectDir = new File('../node_modules/@capacitor/android/capacitor') diff --git a/echobot_client/android/gradle.properties b/echobot_client/android/gradle.properties new file mode 100644 index 0000000..2e87c52 --- /dev/null +++ b/echobot_client/android/gradle.properties @@ -0,0 +1,22 @@ +# Project-wide Gradle settings. + +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. + +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html + +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx1536m + +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true + +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true diff --git a/echobot_client/android/gradle/wrapper/gradle-wrapper.jar b/echobot_client/android/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..1b33c55 Binary files /dev/null and b/echobot_client/android/gradle/wrapper/gradle-wrapper.jar differ diff --git a/echobot_client/android/gradle/wrapper/gradle-wrapper.properties b/echobot_client/android/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..7705927 --- /dev/null +++ b/echobot_client/android/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-all.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/echobot_client/android/gradlew b/echobot_client/android/gradlew new file mode 100755 index 0000000..23d15a9 --- /dev/null +++ b/echobot_client/android/gradlew @@ -0,0 +1,251 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH="\\\"\\\"" + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/echobot_client/android/gradlew.bat b/echobot_client/android/gradlew.bat new file mode 100644 index 0000000..5eed7ee --- /dev/null +++ b/echobot_client/android/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH= + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/echobot_client/android/settings.gradle b/echobot_client/android/settings.gradle new file mode 100644 index 0000000..3b4431d --- /dev/null +++ b/echobot_client/android/settings.gradle @@ -0,0 +1,5 @@ +include ':app' +include ':capacitor-cordova-android-plugins' +project(':capacitor-cordova-android-plugins').projectDir = new File('./capacitor-cordova-android-plugins/') + +apply from: 'capacitor.settings.gradle' \ No newline at end of file diff --git a/echobot_client/android/variables.gradle b/echobot_client/android/variables.gradle new file mode 100644 index 0000000..ee4ba41 --- /dev/null +++ b/echobot_client/android/variables.gradle @@ -0,0 +1,16 @@ +ext { + minSdkVersion = 24 + compileSdkVersion = 36 + targetSdkVersion = 36 + androidxActivityVersion = '1.11.0' + androidxAppCompatVersion = '1.7.1' + androidxCoordinatorLayoutVersion = '1.3.0' + androidxCoreVersion = '1.17.0' + androidxFragmentVersion = '1.8.9' + coreSplashScreenVersion = '1.2.0' + androidxWebkitVersion = '1.14.0' + junitVersion = '4.13.2' + androidxJunitVersion = '1.3.0' + androidxEspressoCoreVersion = '3.7.0' + cordovaAndroidVersion = '14.0.1' +} \ No newline at end of file diff --git a/echobot_client/capacitor.config.json b/echobot_client/capacitor.config.json new file mode 100644 index 0000000..4b3efab --- /dev/null +++ b/echobot_client/capacitor.config.json @@ -0,0 +1,13 @@ +{ + "appId": "com.echobot.app", + "appName": "EchoBot", + "webDir": "../echobot/app/web", + "server": { + "androidScheme": "https", + "cleartext": true + }, + "android": { + "allowMixedContent": true, + "webContentsDebuggingEnabled": true + } +} diff --git a/echobot_client/lib/main.dart b/echobot_client/lib/main.dart new file mode 100644 index 0000000..244a702 --- /dev/null +++ b/echobot_client/lib/main.dart @@ -0,0 +1,122 @@ +import 'package:flutter/material.dart'; + +void main() { + runApp(const MyApp()); +} + +class MyApp extends StatelessWidget { + const MyApp({super.key}); + + // This widget is the root of your application. + @override + Widget build(BuildContext context) { + return MaterialApp( + title: 'Flutter Demo', + theme: ThemeData( + // This is the theme of your application. + // + // TRY THIS: Try running your application with "flutter run". You'll see + // the application has a purple toolbar. Then, without quitting the app, + // try changing the seedColor in the colorScheme below to Colors.green + // and then invoke "hot reload" (save your changes or press the "hot + // reload" button in a Flutter-supported IDE, or press "r" if you used + // the command line to start the app). + // + // Notice that the counter didn't reset back to zero; the application + // state is not lost during the reload. To reset the state, use hot + // restart instead. + // + // This works for code too, not just values: Most code changes can be + // tested with just a hot reload. + colorScheme: .fromSeed(seedColor: Colors.deepPurple), + ), + home: const MyHomePage(title: 'Flutter Demo Home Page'), + ); + } +} + +class MyHomePage extends StatefulWidget { + const MyHomePage({super.key, required this.title}); + + // This widget is the home page of your application. It is stateful, meaning + // that it has a State object (defined below) that contains fields that affect + // how it looks. + + // This class is the configuration for the state. It holds the values (in this + // case the title) provided by the parent (in this case the App widget) and + // used by the build method of the State. Fields in a Widget subclass are + // always marked "final". + + final String title; + + @override + State createState() => _MyHomePageState(); +} + +class _MyHomePageState extends State { + int _counter = 0; + + void _incrementCounter() { + setState(() { + // This call to setState tells the Flutter framework that something has + // changed in this State, which causes it to rerun the build method below + // so that the display can reflect the updated values. If we changed + // _counter without calling setState(), then the build method would not be + // called again, and so nothing would appear to happen. + _counter++; + }); + } + + @override + Widget build(BuildContext context) { + // This method is rerun every time setState is called, for instance as done + // by the _incrementCounter method above. + // + // The Flutter framework has been optimized to make rerunning build methods + // fast, so that you can just rebuild anything that needs updating rather + // than having to individually change instances of widgets. + return Scaffold( + appBar: AppBar( + // TRY THIS: Try changing the color here to a specific color (to + // Colors.amber, perhaps?) and trigger a hot reload to see the AppBar + // change color while the other colors stay the same. + backgroundColor: Theme.of(context).colorScheme.inversePrimary, + // Here we take the value from the MyHomePage object that was created by + // the App.build method, and use it to set our appbar title. + title: Text(widget.title), + ), + body: Center( + // Center is a layout widget. It takes a single child and positions it + // in the middle of the parent. + child: Column( + // Column is also a layout widget. It takes a list of children and + // arranges them vertically. By default, it sizes itself to fit its + // children horizontally, and tries to be as tall as its parent. + // + // Column has various properties to control how it sizes itself and + // how it positions its children. Here we use mainAxisAlignment to + // center the children vertically; the main axis here is the vertical + // axis because Columns are vertical (the cross axis would be + // horizontal). + // + // TRY THIS: Invoke "debug painting" (choose the "Toggle Debug Paint" + // action in the IDE, or press "p" in the console), to see the + // wireframe for each widget. + mainAxisAlignment: .center, + children: [ + const Text('You have pushed the button this many times:'), + Text( + '$_counter', + style: Theme.of(context).textTheme.headlineMedium, + ), + ], + ), + ), + floatingActionButton: FloatingActionButton( + onPressed: _incrementCounter, + tooltip: 'Increment', + child: const Icon(Icons.add), + ), + ); + } +} diff --git a/echobot_client/package.json b/echobot_client/package.json new file mode 100644 index 0000000..b04b5e3 --- /dev/null +++ b/echobot_client/package.json @@ -0,0 +1,22 @@ +{ + "name": "echobot_client", + "version": "1.0.0", + "description": "A new Flutter project.", + "main": "index.js", + "directories": { + "lib": "lib", + "test": "test" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "type": "commonjs", + "dependencies": { + "@capacitor/android": "^8.2.0", + "@capacitor/cli": "^8.2.0", + "@capacitor/core": "^8.2.0" + } +} diff --git a/echobot_client/pubspec.lock b/echobot_client/pubspec.lock new file mode 100644 index 0000000..e39b6e0 --- /dev/null +++ b/echobot_client/pubspec.lock @@ -0,0 +1,522 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + args: + dependency: transitive + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.13.0" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.1.2" + camera: + dependency: "direct main" + description: + name: camera + sha256: "4142a19a38e388d3bab444227636610ba88982e36dff4552d5191a86f65dc437" + url: "https://pub.flutter-io.cn" + source: hosted + version: "0.11.4" + camera_android_camerax: + dependency: transitive + description: + name: camera_android_camerax + sha256: "8516fe308bc341a5067fb1a48edff0ddfa57c0d3cdcc9dbe7ceca3ba119e2577" + url: "https://pub.flutter-io.cn" + source: hosted + version: "0.6.30" + camera_avfoundation: + dependency: transitive + description: + name: camera_avfoundation + sha256: "11b4aee2f5e5e038982e152b4a342c749b414aa27857899d20f4323e94cb5f0b" + url: "https://pub.flutter-io.cn" + source: hosted + version: "0.9.23+2" + camera_platform_interface: + dependency: transitive + description: + name: camera_platform_interface + sha256: "98cfc9357e04bad617671b4c1f78a597f25f08003089dd94050709ae54effc63" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.12.0" + camera_web: + dependency: transitive + description: + name: camera_web + sha256: "57f49a635c8bf249d07fb95eb693d7e4dda6796dedb3777f9127fb54847beba7" + url: "https://pub.flutter-io.cn" + source: hosted + version: "0.3.5+3" + characters: + dependency: transitive + description: + name: characters + sha256: faf38497bda5ead2a8c7615f4f7939df04333478bf32e4173fcb06d428b5716b + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.4.1" + clock: + dependency: transitive + description: + name: clock + sha256: fddb70d9b5277016c77a80201021d40a2247104d9f4aa7bab7157b7e3f05b84b + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.1.2" + collection: + dependency: transitive + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.19.1" + cross_file: + dependency: transitive + description: + name: cross_file + sha256: "28bb3ae56f117b5aec029d702a90f57d285cd975c3c5c281eaca38dbc47c5937" + url: "https://pub.flutter-io.cn" + source: hosted + version: "0.3.5+2" + crypto: + dependency: transitive + description: + name: crypto + sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf + url: "https://pub.flutter-io.cn" + source: hosted + version: "3.0.7" + cupertino_icons: + dependency: "direct main" + description: + name: cupertino_icons + sha256: ba631d1c7f7bef6b729a622b7b752645a2d076dba9976925b8f25725a30e1ee6 + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.0.8" + dio: + dependency: "direct main" + description: + name: dio + sha256: aff32c08f92787a557dd5c0145ac91536481831a01b4648136373cddb0e64f8c + url: "https://pub.flutter-io.cn" + source: hosted + version: "5.9.2" + dio_web_adapter: + dependency: transitive + description: + name: dio_web_adapter + sha256: "2f9e64323a7c3c7ef69567d5c800424a11f8337b8b228bad02524c9fb3c1f340" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.1.2" + fake_async: + dependency: transitive + description: + name: fake_async + sha256: "5368f224a74523e8d2e7399ea1638b37aecfca824a3cc4dfdf77bf1fa905ac44" + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.3.3" + ffi: + dependency: transitive + description: + name: ffi + sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.2.0" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.flutter-io.cn" + source: hosted + version: "7.0.1" + flutter: + dependency: "direct main" + description: flutter + source: sdk + version: "0.0.0" + flutter_lints: + dependency: "direct dev" + description: + name: flutter_lints + sha256: "3105dc8492f6183fb076ccf1f351ac3d60564bff92e20bfc4af9cc1651f4e7e1" + url: "https://pub.flutter-io.cn" + source: hosted + version: "6.0.0" + flutter_markdown: + dependency: "direct main" + description: + name: flutter_markdown + sha256: "08fb8315236099ff8e90cb87bb2b935e0a724a3af1623000a9cec930468e0f27" + url: "https://pub.flutter-io.cn" + source: hosted + version: "0.7.7+1" + flutter_plugin_android_lifecycle: + dependency: transitive + description: + name: flutter_plugin_android_lifecycle + sha256: ee8068e0e1cd16c4a82714119918efdeed33b3ba7772c54b5d094ab53f9b7fd1 + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.0.33" + flutter_riverpod: + dependency: "direct main" + description: + name: flutter_riverpod + sha256: "9532ee6db4a943a1ed8383072a2e3eeda041db5657cdf6d2acecf3c21ecbe7e1" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.6.1" + flutter_test: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" + flutter_web_plugins: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.flutter-io.cn" + source: hosted + version: "4.1.2" + leak_tracker: + dependency: transitive + description: + name: leak_tracker + sha256: "33e2e26bdd85a0112ec15400c8cbffea70d0f9c3407491f672a2fad47915e2de" + url: "https://pub.flutter-io.cn" + source: hosted + version: "11.0.2" + leak_tracker_flutter_testing: + dependency: transitive + description: + name: leak_tracker_flutter_testing + sha256: "1dbc140bb5a23c75ea9c4811222756104fbcd1a27173f0c34ca01e16bea473c1" + url: "https://pub.flutter-io.cn" + source: hosted + version: "3.0.10" + leak_tracker_testing: + dependency: transitive + description: + name: leak_tracker_testing + sha256: "8d5a2d49f4a66b49744b23b018848400d23e54caf9463f4eb20df3eb8acb2eb1" + url: "https://pub.flutter-io.cn" + source: hosted + version: "3.0.2" + lints: + dependency: transitive + description: + name: lints + sha256: "12f842a479589fea194fe5c5a3095abc7be0c1f2ddfa9a0e76aed1dbd26a87df" + url: "https://pub.flutter-io.cn" + source: hosted + version: "6.1.0" + markdown: + dependency: transitive + description: + name: markdown + sha256: ee85086ad7698b42522c6ad42fe195f1b9898e4d974a1af4576c1a3a176cada9 + url: "https://pub.flutter-io.cn" + source: hosted + version: "7.3.1" + matcher: + dependency: transitive + description: + name: matcher + sha256: dc0b7dc7651697ea4ff3e69ef44b0407ea32c487a39fff6a4004fa585e901861 + url: "https://pub.flutter-io.cn" + source: hosted + version: "0.12.19" + material_color_utilities: + dependency: transitive + description: + name: material_color_utilities + sha256: "9c337007e82b1889149c82ed242ed1cb24a66044e30979c44912381e9be4c48b" + url: "https://pub.flutter-io.cn" + source: hosted + version: "0.13.0" + meta: + dependency: transitive + description: + name: meta + sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394" + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.17.0" + mime: + dependency: transitive + description: + name: mime + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.0.0" + path: + dependency: transitive + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.9.1" + path_provider_linux: + dependency: transitive + description: + name: path_provider_linux + sha256: f7a1fe3a634fe7734c8d3f2766ad746ae2a2884abe22e241a8b301bf5cac3279 + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.2.1" + path_provider_platform_interface: + dependency: transitive + description: + name: path_provider_platform_interface + sha256: "88f5779f72ba699763fa3a3b06aa4bf6de76c8e5de842cf6f29e2e06476c2334" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.1.2" + path_provider_windows: + dependency: transitive + description: + name: path_provider_windows + sha256: bd6f00dbd873bfb70d0761682da2b3a2c2fccc2b9e84c495821639601d81afe7 + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.3.0" + platform: + dependency: transitive + description: + name: platform + sha256: "5d6b1b0036a5f331ebc77c850ebc8506cbc1e9416c27e59b439f917a902a4984" + url: "https://pub.flutter-io.cn" + source: hosted + version: "3.1.6" + plugin_platform_interface: + dependency: transitive + description: + name: plugin_platform_interface + sha256: "4820fbfdb9478b1ebae27888254d445073732dae3d6ea81f0b7e06d5dedc3f02" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.1.8" + riverpod: + dependency: transitive + description: + name: riverpod + sha256: "59062512288d3056b2321804332a13ffdd1bf16df70dcc8e506e411280a72959" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.6.1" + shared_preferences: + dependency: "direct main" + description: + name: shared_preferences + sha256: "2939ae520c9024cb197fc20dee269cd8cdbf564c8b5746374ec6cacdc5169e64" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.5.4" + shared_preferences_android: + dependency: transitive + description: + name: shared_preferences_android + sha256: "8374d6200ab33ac99031a852eba4c8eb2170c4bf20778b3e2c9eccb45384fb41" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.4.21" + shared_preferences_foundation: + dependency: transitive + description: + name: shared_preferences_foundation + sha256: "4e7eaffc2b17ba398759f1151415869a34771ba11ebbccd1b0145472a619a64f" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.5.6" + shared_preferences_linux: + dependency: transitive + description: + name: shared_preferences_linux + sha256: "580abfd40f415611503cae30adf626e6656dfb2f0cee8f465ece7b6defb40f2f" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.4.1" + shared_preferences_platform_interface: + dependency: transitive + description: + name: shared_preferences_platform_interface + sha256: "57cbf196c486bc2cf1f02b85784932c6094376284b3ad5779d1b1c6c6a816b80" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.4.1" + shared_preferences_web: + dependency: transitive + description: + name: shared_preferences_web + sha256: c49bd060261c9a3f0ff445892695d6212ff603ef3115edbb448509d407600019 + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.4.3" + shared_preferences_windows: + dependency: transitive + description: + name: shared_preferences_windows + sha256: "94ef0f72b2d71bc3e700e025db3710911bd51a71cefb65cc609dd0d9a982e3c1" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.4.1" + sky_engine: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" + source_span: + dependency: transitive + description: + name: source_span + sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab" + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.10.2" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.12.1" + state_notifier: + dependency: transitive + description: + name: state_notifier + sha256: b8677376aa54f2d7c58280d5a007f9e8774f1968d1fb1c096adcb4792fba29bb + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.0.0" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.1.4" + stream_transform: + dependency: transitive + description: + name: stream_transform + sha256: ad47125e588cfd37a9a7f86c7d6356dde8dfe89d071d293f80ca9e9273a33871 + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.1.1" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.2.2" + test_api: + dependency: transitive + description: + name: test_api + sha256: "8161c84903fd860b26bfdefb7963b3f0b68fee7adea0f59ef805ecca346f0c7a" + url: "https://pub.flutter-io.cn" + source: hosted + version: "0.7.10" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.4.0" + vector_math: + dependency: transitive + description: + name: vector_math + sha256: d530bd74fea330e6e364cda7a85019c434070188383e1cd8d9777ee586914c5b + url: "https://pub.flutter-io.cn" + source: hosted + version: "2.2.0" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60" + url: "https://pub.flutter-io.cn" + source: hosted + version: "15.0.2" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.1.1" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.0.1" + web_socket_channel: + dependency: "direct main" + description: + name: web_socket_channel + sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 + url: "https://pub.flutter-io.cn" + source: hosted + version: "3.0.3" + xdg_directories: + dependency: transitive + description: + name: xdg_directories + sha256: "7a3f37b05d989967cdddcbb571f1ea834867ae2faa29725fd085180e0883aa15" + url: "https://pub.flutter-io.cn" + source: hosted + version: "1.1.0" +sdks: + dart: ">=3.11.1 <4.0.0" + flutter: ">=3.35.0" diff --git a/echobot_client/pubspec.yaml b/echobot_client/pubspec.yaml new file mode 100644 index 0000000..313d3e1 --- /dev/null +++ b/echobot_client/pubspec.yaml @@ -0,0 +1,37 @@ +name: echobot_client +description: "EchoBot Flutter Android 客户端" +publish_to: 'none' +version: 1.0.0+1 + +environment: + sdk: ^3.11.1 + +dependencies: + flutter: + sdk: flutter + cupertino_icons: ^1.0.8 + + # 网络 + dio: ^5.7.0 + web_socket_channel: ^3.0.1 + + # 摄像头 + camera: ^0.11.0+2 + + # 状态管理 + flutter_riverpod: ^2.6.1 + + # 本地存储 + shared_preferences: ^2.3.3 + + # Markdown 渲染 + flutter_markdown: ^0.7.4 + +dev_dependencies: + flutter_test: + sdk: flutter + flutter_lints: ^6.0.0 + +flutter: + uses-material-design: true + diff --git a/echobot_client/test/widget_test.dart b/echobot_client/test/widget_test.dart new file mode 100644 index 0000000..6e30935 --- /dev/null +++ b/echobot_client/test/widget_test.dart @@ -0,0 +1,30 @@ +// This is a basic Flutter widget test. +// +// To perform an interaction with a widget in your test, use the WidgetTester +// utility in the flutter_test package. For example, you can send tap and scroll +// gestures. You can also use WidgetTester to find child widgets in the widget +// tree, read text, and verify that the values of widget properties are correct. + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import 'package:echobot_client/main.dart'; + +void main() { + testWidgets('Counter increments smoke test', (WidgetTester tester) async { + // Build our app and trigger a frame. + await tester.pumpWidget(const MyApp()); + + // Verify that our counter starts at 0. + expect(find.text('0'), findsOneWidget); + expect(find.text('1'), findsNothing); + + // Tap the '+' icon and trigger a frame. + await tester.tap(find.byIcon(Icons.add)); + await tester.pump(); + + // Verify that our counter has incremented. + expect(find.text('0'), findsNothing); + expect(find.text('1'), findsOneWidget); + }); +}