diff --git a/.gitignore b/.gitignore index c6b64d1dd..f9cc46524 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,4 @@ ms_agent/app/temp_workspace/ webui/work_dir/ .ms_agent_snapshots/ +.ms_agent/ diff --git a/ms_agent/agent/base.py b/ms_agent/agent/base.py index 0c6ee3fc3..090b7ddde 100644 --- a/ms_agent/agent/base.py +++ b/ms_agent/agent/base.py @@ -51,6 +51,19 @@ def __init__(self, self.config.output_dir = self.output_dir except Exception: pass + # Merge the work-dir project patch (e.g. a persisted /model override) so + # config overrides round-trip from /.ms_agent/config.yaml — + # anchored to the project (the work dir), not the config file's + # directory. This keeps running a shared/template config from picking up + # (or scattering) overrides in that config's folder. + try: + from omegaconf import OmegaConf + from ms_agent.config.resolver import ConfigResolver + patch = ConfigResolver()._load_project_patch(self.output_dir) + if patch is not None: + self.config = OmegaConf.merge(self.config, patch) + except Exception: + pass @abstractmethod async def run( diff --git a/ms_agent/agent/llm_agent.py b/ms_agent/agent/llm_agent.py index 44e34e250..5bf9f48ec 100644 --- a/ms_agent/agent/llm_agent.py +++ b/ms_agent/agent/llm_agent.py @@ -172,6 +172,16 @@ def __init__( # Gates both the initial >>> prompt and the mid-turn InputCallback. self._interactive = False + # Optional injected PermissionHandler (TUI/WebUI/Server). When None, + # _select_permission_handler() picks by mode + interactivity. + self._permission_handler = kwargs.get('permission_handler', None) + + # Optional console I/O sink (TUI). When None, streaming tokens go to + # sys.stdout and the interactive prompt uses input() — i.e. current CLI + # behavior. A ConsoleIO with write()/read_prompt() reroutes both so a + # rich TUI can own rendering without changing the agent lifecycle. + self._console_io = kwargs.get('console_io', None) + # Personalization (lazy-loaded in _build_personalization_section) self._profile_manager = ProfileManager() @@ -456,7 +466,8 @@ def register_callback_from_config(self): if self._interactive: self.callbacks.append(callbacks_mapping[_callback]( self.config, - command_router=self._get_command_router())) + command_router=self._get_command_router(), + io=self._console_io)) else: self.callbacks.append(callbacks_mapping[_callback]( self.config)) @@ -469,7 +480,8 @@ def register_callback_from_config(self): self.callbacks.append( input_cls( self.config, - command_router=self._get_command_router())) + command_router=self._get_command_router(), + io=self._console_io)) async def on_task_begin(self, messages: List[Message]): self.log_output(f'Agent {self.tag} task beginning.') @@ -573,16 +585,39 @@ async def parallel_tool_call(self, self.log_output(_new_message.content) return messages + def _select_permission_handler(self, mode: str): + """Pick the PermissionHandler by mode + runtime environment. + + - An explicitly injected handler (``set_permission_handler`` / the + ``permission_handler`` kwarg) always wins — this is how the TUI / + WebUI / Server supply their own confirmation UI. + - ``interactive`` (alias ``restricted``) in an interactive terminal + session -> ``CLIPermissionHandler`` (the ``[y/s/a/e/n]`` prompt), + so non-whitelisted tools actually ask the user. + - Everything else (``auto`` / ``strict`` / non-interactive) -> + ``AutoPermissionHandler`` (SafetyGuard still enforces the floor). + """ + if self._permission_handler is not None: + return self._permission_handler + from ms_agent.permission import ( + AutoPermissionHandler, + CLIPermissionHandler, + ) + # ``PermissionConfig.from_dict`` already normalizes ``restricted`` -> + # ``interactive``; accept both so a direct caller passing the raw alias + # still gets the interactive prompt (not a silent AutoPermissionHandler). + if mode in ('interactive', 'restricted') and self._interactive: + return CLIPermissionHandler() + return AutoPermissionHandler() + def _build_permission_objects(self): """Create SafetyGuard and PermissionEnforcer from config if configured.""" from ms_agent.permission import ( - AutoPermissionHandler, PermissionConfig, PermissionEnforcer, PermissionMemory, SafetyGuard, ) - from ms_agent.permission.config import SafetyConfig raw = {} if hasattr(self.config, 'permission'): @@ -605,12 +640,16 @@ def _build_permission_objects(self): workspace_root=workspace_root, ) - handler = AutoPermissionHandler() + handler = self._select_permission_handler(perm_config.mode) memory = PermissionMemory(project_path=workspace_root) enforcer = PermissionEnforcer(config=perm_config, handler=handler, memory=memory) return safety_guard, enforcer, perm_config + def set_permission_handler(self, handler) -> None: + """Inject a custom PermissionHandler (TUI/WebUI/Server) before run.""" + self._permission_handler = handler + async def prepare_tools(self): """Initialize and connect the tool manager.""" import uuid @@ -1053,10 +1092,20 @@ def _init_session_log(self) -> None: session_cfg, 'dir', None ) if session_cfg else None if session_dir is None: - session_dir = os.path.join( - getattr(self.config, 'output_dir', 'output'), - 'sessions', - ) + # Sessions live globally, keyed by the work dir (Claude Code style), + # decoupled from output_dir. An explicit session_log.dir (set by the + # WebUI/Server per project) still wins. Legacy /sessions + # is migrated forward if present. + from ms_agent.project.paths import global_sessions_dir + output_dir = getattr(self.config, 'output_dir', 'output') + session_dir = str(global_sessions_dir(output_dir)) + legacy_dir = os.path.join(output_dir, 'sessions') + if os.path.isdir(legacy_dir) and not os.path.isdir(session_dir): + try: + import shutil + shutil.copytree(legacy_dir, session_dir) + except Exception: + pass session_key = getattr(session_cfg, 'session_key', None) if session_cfg else None self.session_log = SessionLog(session_dir, session_key=session_key) @@ -1309,8 +1358,11 @@ async def step( if _printed_reasoning_header and not _printed_reasoning_footer: self._write_thinking_footer() _printed_reasoning_footer = True - sys.stdout.write(new_content) - sys.stdout.flush() + if self._console_io is not None: + self._console_io.write(new_content) + else: + sys.stdout.write(new_content) + sys.stdout.flush() _content = _response_message.content messages[-1] = _response_message yield messages @@ -1327,7 +1379,10 @@ async def step( self._write_reasoning(final_reasoning, dim=True) self._write_thinking_footer() - sys.stdout.write('\n') + if self._console_io is not None: + self._console_io.end_stream() + else: + sys.stdout.write('\n') else: _response_message = self.llm.generate(messages, tools=tools) if self.show_reasoning: @@ -1603,7 +1658,10 @@ async def run_loop(self, messages: Union[List[Message], str], messages = configured elif self._interactive: from ms_agent.command.interactive import InteractiveSession - session = InteractiveSession(self._get_command_router()) + session = InteractiveSession( + self._get_command_router(), + source='tui' if self._console_io is not None else 'cli', + io=self._console_io) turn = await session.run_turn( messages=None, runtime=self.runtime) if turn.action == 'quit': diff --git a/ms_agent/callbacks/input_callback.py b/ms_agent/callbacks/input_callback.py index a8e1f6d09..c37744e11 100644 --- a/ms_agent/callbacks/input_callback.py +++ b/ms_agent/callbacks/input_callback.py @@ -31,13 +31,17 @@ def __init__( self, config: DictConfig, command_router: Optional['CommandRouter'] = None, + io: object = None, ): super().__init__(config) if command_router is None: # Fallback for standalone / test instantiation. In normal CLI use # the agent injects its own router so there is a single instance. command_router = self._build_default_router() - self._session = InteractiveSession(command_router) + self._session = InteractiveSession( + command_router, + source='tui' if io is not None else 'cli', + io=io) @staticmethod def _build_default_router() -> 'CommandRouter': diff --git a/ms_agent/cli/cli.py b/ms_agent/cli/cli.py index b43b4fa4f..6e371bb63 100644 --- a/ms_agent/cli/cli.py +++ b/ms_agent/cli/cli.py @@ -7,6 +7,7 @@ from ms_agent.cli.cron import CronCMD from ms_agent.cli.plugin import PluginCMD from ms_agent.cli.run import RunCMD +from ms_agent.cli.tui import TuiCMD from ms_agent.cli.ui import UICMD @@ -28,6 +29,7 @@ def run_cmd(): ACPProxyCmd.define_args(subparsers) ACPRegistryCmd.define_args(subparsers) RunCMD.define_args(subparsers) + TuiCMD.define_args(subparsers) AppCMD.define_args(subparsers) UICMD.define_args(subparsers) CronCMD.define_args(subparsers) diff --git a/ms_agent/cli/run.py b/ms_agent/cli/run.py index ec72e401f..7f9f2cfbc 100644 --- a/ms_agent/cli/run.py +++ b/ms_agent/cli/run.py @@ -260,7 +260,6 @@ def _execute_with_config(self): if tl is None or not OmegaConf.is_config(tl): localsearch_config = { 'paths': paths, - 'work_path': './.sirchmunk', 'mode': 'FAST', } config.tools['localsearch'] = OmegaConf.create( diff --git a/ms_agent/cli/tui.py b/ms_agent/cli/tui.py new file mode 100644 index 000000000..ed1a3139d --- /dev/null +++ b/ms_agent/cli/tui.py @@ -0,0 +1,77 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import argparse + +from .base import CLICommand + + +def subparser_func(args): + """Factory for the `tui` sub-command.""" + return TuiCMD(args) + + +class TuiCMD(CLICommand): + """Terminal UI: a lightweight REPL over the LLMAgent lifecycle. + + Note: `ui` is taken by the (gradio) webui, so this command is `tui`. + """ + + name = 'tui' + + def __init__(self, args): + self.args = args + + @staticmethod + def define_args(parsers: argparse.ArgumentParser): + parser: argparse.ArgumentParser = parsers.add_parser(TuiCMD.name) + parser.add_argument( + '--config', + type=str, + default=None, + help='Path to an agent.yaml (or a task dir / modelscope id). ' + 'Defaults to the built-in agent.yaml.') + parser.add_argument( + '--work-dir', + '--work_dir', + dest='work_dir', + type=str, + default=None, + help='Working/project directory (== output_dir). Defaults to the ' + 'current directory. Framework internals go to /.ms_agent/; ' + 'sessions are global under ~/.ms_agent/projects/.') + parser.add_argument( + '--env', + type=str, + default=None, + help='Path to a .env file (defaults to ./.env).') + parser.add_argument( + '--permission_mode', + type=str, + default='restricted', + choices=['auto', 'strict', 'restricted', 'interactive'], + help='Permission mode for tool calls. Default `restricted` so ' + 'non-whitelisted tools ask for confirmation.') + parser.add_argument( + '--trust_remote_code', + action='store_true', + help='Allow loading external code referenced by the config.') + parser.set_defaults(func=subparser_func) + + def execute(self): + import importlib.resources as importlib_resources + + config = self.args.config + if not config: + # Fall back to the packaged default agent.yaml. + default_config = importlib_resources.files('ms_agent').joinpath( + 'agent', 'agent.yaml') + with importlib_resources.as_file(default_config) as p: + config = str(p) + + from ms_agent.tui.app import main + main( + config, + env_file=self.args.env, + permission_mode=self.args.permission_mode, + trust_remote_code=self.args.trust_remote_code, + work_dir=self.args.work_dir, + ) diff --git a/ms_agent/command/builtin/config_cmds.py b/ms_agent/command/builtin/config_cmds.py index d63adc507..30a7200a3 100644 --- a/ms_agent/command/builtin/config_cmds.py +++ b/ms_agent/command/builtin/config_cmds.py @@ -26,23 +26,30 @@ def _persist_model_to_config(config, new_model: str, service=None): """Persist the model (and optional service) change to the project patch. Writes ``llm.model`` (and ``llm.service`` when a provider switch happened) - into ``/.ms-agent/config.yaml`` rather than mutating the - version-controlled source YAML. ``Config.from_task`` merges this patch back - (patch wins) on the next run, so the override survives without ever touching - the committed config. Returns the patch path on success, or None if no - source directory is known or the write fails. + into ``/.ms_agent/config.yaml`` rather than mutating the + version-controlled source YAML. Anchored to the **work dir** (``output_dir``) + — the project — not the config file's directory, so running a shared or + packaged template config (e.g. ``demos/``) never scatters a ``.ms_agent/`` + next to it. The work-dir patch is merged back (patch wins) on the next run. + Returns the patch path on success, or None if no dir is known / write fails. """ from omegaconf import OmegaConf - local_dir = getattr(config, 'local_dir', None) - if not local_dir: + # Prefer the work dir; fall back to the config's own dir only if unset. + base_dir = (getattr(config, 'output_dir', None) + or getattr(config, 'local_dir', None)) + if not base_dir: return None - patch_dir = os.path.join(str(local_dir), '.ms-agent') + # Write to the new .ms_agent/ dir; migrate an existing legacy + # .ms-agent/config.yaml so a single patch file remains authoritative. + patch_dir = os.path.join(str(base_dir), '.ms_agent') patch_path = os.path.join(patch_dir, 'config.yaml') + legacy_path = os.path.join(str(base_dir), '.ms-agent', 'config.yaml') try: os.makedirs(patch_dir, exist_ok=True) - patch = (OmegaConf.load(patch_path) - if os.path.isfile(patch_path) else OmegaConf.create({})) + source = patch_path if os.path.isfile(patch_path) else legacy_path + patch = (OmegaConf.load(source) + if os.path.isfile(source) else OmegaConf.create({})) OmegaConf.update(patch, 'llm.model', new_model, merge=True) if service: OmegaConf.update(patch, 'llm.service', service, merge=True) diff --git a/ms_agent/command/builtin/context_cmds.py b/ms_agent/command/builtin/context_cmds.py index 0fd7a2a6c..86ba0a704 100644 --- a/ms_agent/command/builtin/context_cmds.py +++ b/ms_agent/command/builtin/context_cmds.py @@ -104,28 +104,50 @@ async def cmd_tools(ctx: CommandContext) -> CommandResult: async def cmd_compact(ctx: CommandContext) -> CommandResult: + """Prune old tool outputs from the live context to free tokens. + + Keeps the most recent tool results in full and elides older large ones in + place (the same idea as ContextAssembler's ToolOutputPruner, applied on + demand). Automatic per-round compaction still runs when session_log + + compaction are configured; this is the manual trigger. + """ messages = ctx.extra.get('messages') if not messages: return CommandResult( type=CommandResultType.MESSAGE, content='No messages available.' ) - try: - from ms_agent.session.context_assembler import ContextAssembler # noqa: F401 - except ImportError: + keep_recent = 3 + prune_threshold = 200 + tool_idxs = [ + i for i, m in enumerate(messages) + if getattr(m, 'role', None) == 'tool' + ] + to_prune = tool_idxs[:-keep_recent] if len(tool_idxs) > keep_recent else [] + + pruned = 0 + freed = 0 + for i in to_prune: + m = messages[i] + content = m.content if isinstance(m.content, str) else str( + m.content or '') + if len(content) > prune_threshold and not content.startswith( + '[compacted'): + placeholder = f'[compacted tool output: {len(content)} chars elided]' + freed += len(content) - len(placeholder) + messages[i].content = placeholder + pruned += 1 + + if pruned == 0: return CommandResult( type=CommandResultType.MESSAGE, - content=( - 'Context compaction not available yet.' - ), + content=(f'Nothing to compact ({len(messages)} messages; no large ' + f'old tool outputs beyond the last {keep_recent}).'), ) - return CommandResult( type=CommandResultType.MESSAGE, - content=( - f'Compaction requested. Current message count: {len(messages)}.\n' - f'(Full implementation available after PR#912 merge)' - ), + content=(f'Compacted {pruned} old tool output(s), ~{freed} chars freed. ' + f'The {keep_recent} most recent are kept in full.'), ) diff --git a/ms_agent/command/interactive.py b/ms_agent/command/interactive.py index 597d7da2f..6935c0fb8 100644 --- a/ms_agent/command/interactive.py +++ b/ms_agent/command/interactive.py @@ -33,9 +33,14 @@ class InteractiveTurn: class InteractiveSession: """Drives a single ``>>>`` prompt turn, handling slash commands in a loop.""" - def __init__(self, router: CommandRouter, source: str = 'cli') -> None: + def __init__(self, router: CommandRouter, source: str = 'cli', + io: Any = None) -> None: self._router = router self._source = source + # Optional ConsoleIO (TUI). When None, use bare input()/print() so CLI + # behavior is unchanged. When set, read_prompt()/print() are delegated + # so a rich TUI owns prompt rendering and slash-completion. + self._io = io async def run_turn( self, @@ -56,7 +61,10 @@ async def run_turn( """ while True: try: - query = input('>>> ').strip() + if self._io is not None: + query = self._io.read_prompt('>>> ').strip() + else: + query = input('>>> ').strip() except (EOFError, KeyboardInterrupt): return InteractiveTurn(action='quit') if not query: @@ -82,10 +90,16 @@ async def run_turn( return InteractiveTurn(action='submit', text=query) if result.type == CommandResultType.QUIT: if result.content: - print(result.content) + self._emit(result.content) return InteractiveTurn(action='quit') if result.type == CommandResultType.SUBMIT_PROMPT: return InteractiveTurn(action='submit', text=result.content) # MESSAGE / MUTATE_STATE: show output and prompt again. if result.content: - print(result.content) + self._emit(result.content) + + def _emit(self, text: str) -> None: + if self._io is not None: + self._io.print(text) + else: + print(text) diff --git a/ms_agent/config/config.py b/ms_agent/config/config.py index 3385128ea..dd2d3831b 100644 --- a/ms_agent/config/config.py +++ b/ms_agent/config/config.py @@ -10,7 +10,6 @@ from ms_agent.prompting import apply_prompt_files from ms_agent.utils import get_logger -from ms_agent.config.resolver import ConfigResolver from ..utils.constants import TOOL_PLUGIN_NAME from .env import Env @@ -95,14 +94,11 @@ def from_task(cls, cls._update_config(config, _dict_config) config.local_dir = config_dir_or_id config.name = name - # Merge the project-level config patch (/.ms-agent/config.yaml) - # written by interactive overrides such as /model. The patch wins over - # the committed YAML so a user override beats the project default, while - # the source file itself is never mutated. - if isinstance(config, DictConfig): - patch = ConfigResolver()._load_project_patch(config_dir_or_id) - if patch is not None: - config = OmegaConf.merge(config, patch) + # The project-level config patch (persisted /model overrides etc.) is + # merged by the agent from /.ms_agent/config.yaml, anchored to + # the work dir — NOT here from the config file's own directory, which + # would leak/scatter overrides when a shared or packaged template config + # is run from a different working directory. See BaseAgent.__init__. config = cls.fill_missing_fields(config) # Prompt files: resolve config.prompt.system from prompts/ directory # if user didn't specify inline prompt.system. @@ -122,6 +118,22 @@ def fill_missing_fields(config: DictConfig) -> DictConfig: config.callbacks = ListConfig([]) return config + @staticmethod + def safe_get_config(config: Union[DictConfig, ListConfig], + dotted_path: str, + default: Any = None) -> Any: + """Safely read a dotted config path, returning ``default`` if missing. + + Example:: + + Config.safe_get_config(cfg, 'tools.file_system.include', []) + """ + try: + value = OmegaConf.select(config, dotted_path, default=default) + except Exception: + return default + return value if value is not None else default + @staticmethod def is_workflow(config: DictConfig) -> bool: assert config.name is not None, 'Cannot find a valid name in this config' @@ -132,16 +144,28 @@ def is_workflow(config: DictConfig) -> bool: @staticmethod def parse_args() -> Dict[str, Any]: - arg_parser = argparse.ArgumentParser() - args, unknown = arg_parser.parse_known_args() - _dict_config = {} - if unknown: - for idx in range(1, len(unknown) - 1, 2): - key = unknown[idx] - value = unknown[idx + 1] - assert key.startswith( - '--'), f'Parameter not correct: {unknown}' - _dict_config[key[2:]] = value + """Best-effort extraction of ad-hoc ``--key value`` overrides from argv. + + This is called by :meth:`from_task` for every config load, including + embedded / non-CLI contexts (pytest, WebUI/Server, TUI). It therefore + must never raise on argv shapes it does not recognize: it scans for + well-formed ``--key value`` pairs and silently ignores everything else + (subcommands, pytest flags, positional test paths, ``-x`` short flags). + Unknown override keys are harmless downstream — ``_update_config`` only + replaces config paths that already exist. + """ + arg_parser = argparse.ArgumentParser(add_help=False) + _, unknown = arg_parser.parse_known_args() + _dict_config: Dict[str, Any] = {} + idx = 0 + while idx < len(unknown): + key = unknown[idx] + if (key.startswith('--') and len(key) > 2 and idx + 1 < len(unknown) + and not unknown[idx + 1].startswith('--')): + _dict_config[key[2:]] = unknown[idx + 1] + idx += 2 + else: + idx += 1 return _dict_config @staticmethod diff --git a/ms_agent/config/mcp_manager.py b/ms_agent/config/mcp_manager.py index 9c8aabec9..47360dca9 100644 --- a/ms_agent/config/mcp_manager.py +++ b/ms_agent/config/mcp_manager.py @@ -43,7 +43,8 @@ def global_mcp_path(self) -> Path: def project_mcp_path(self) -> Path: if self.project_root is None: raise ValueError('project_root is required for project scope') - return self.project_root / '.ms-agent' / 'mcp.json' + from ms_agent.project.paths import project_internal_file + return project_internal_file(self.project_root, 'mcp.json') def _ensure_dir(self, path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) diff --git a/ms_agent/config/model_settings.py b/ms_agent/config/model_settings.py new file mode 100644 index 000000000..2f9445a63 --- /dev/null +++ b/ms_agent/config/model_settings.py @@ -0,0 +1,127 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Persistent model-provider settings (WebUI "Model settings" page / 原型图10). + +Manages the ``providers`` / ``default_model`` sections of +``~/.ms_agent/settings.json`` so a UI can add/remove custom providers and +models and pick a default, on top of the read-only built-in registry +(``ms_agent/llm/spec.py``). Read-modify-write is atomic (tmp + rename) and +never clobbers the other settings.json sections (llm, personalization, ...). +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Dict, List, Optional + + +class ModelSettingsManager: + """CRUD for custom providers/models + default model in settings.json.""" + + def __init__(self, global_dir: str | Path = '~/.ms_agent') -> None: + self._dir = Path(global_dir).expanduser() + self._path = self._dir / 'settings.json' + + # -- raw settings.json io -- + + def _load_raw(self) -> Dict[str, Any]: + if not self._path.exists(): + return {} + try: + with open(self._path, 'r', encoding='utf-8') as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return {} + + def _save_raw(self, data: Dict[str, Any]) -> None: + self._dir.mkdir(parents=True, exist_ok=True) + tmp = self._path.with_suffix('.json.tmp') + with open(tmp, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=2) + os.replace(tmp, self._path) + + # -- providers -- + + def list_custom_providers(self) -> Dict[str, Dict[str, Any]]: + return dict(self._load_raw().get('providers', {}) or {}) + + def list_providers(self) -> List[Dict[str, Any]]: + """Built-in (read-only, from the registry) + custom providers.""" + from ms_agent.llm.spec import get_registry + out: List[Dict[str, Any]] = [] + for spec in get_registry().list_providers(): + out.append({ + 'id': spec.name, + 'name': spec.name, + 'protocol': spec.transport, + 'builtin': True, + 'models': list(getattr(spec, 'keywords', []) or []), + }) + builtin_ids = {p['id'] for p in out} + for pid, p in self.list_custom_providers().items(): + entry = {'id': pid, 'builtin': False, **p} + if pid in builtin_ids: # custom override of a builtin id + entry['overrides_builtin'] = True + out.append(entry) + return out + + def add_provider( + self, + provider_id: str, + *, + name: Optional[str] = None, + protocol: str = 'openai', + api_key: Optional[str] = None, + base_url: Optional[str] = None, + models: Optional[List[str]] = None, + ) -> Dict[str, Any]: + data = self._load_raw() + providers = data.setdefault('providers', {}) + entry = providers.get(provider_id, {}) + entry.update({ + 'name': name or entry.get('name') or provider_id, + 'protocol': protocol, + }) + if api_key is not None: + entry['api_key'] = api_key + if base_url is not None: + entry['base_url'] = base_url + entry['models'] = list(models if models is not None + else entry.get('models', [])) + providers[provider_id] = entry + self._save_raw(data) + return entry + + def remove_provider(self, provider_id: str) -> None: + data = self._load_raw() + if data.get('providers', {}).pop(provider_id, None) is not None: + self._save_raw(data) + + def add_model(self, provider_id: str, model: str) -> None: + data = self._load_raw() + providers = data.setdefault('providers', {}) + entry = providers.setdefault(provider_id, {'name': provider_id, + 'protocol': 'openai', + 'models': []}) + models = entry.setdefault('models', []) + if model not in models: + models.append(model) + self._save_raw(data) + + def remove_model(self, provider_id: str, model: str) -> None: + data = self._load_raw() + entry = data.get('providers', {}).get(provider_id) + if entry and model in entry.get('models', []): + entry['models'].remove(model) + self._save_raw(data) + + # -- default model -- + + def get_default_model(self) -> Optional[str]: + """Returns ``provider/model`` (or bare ``model``), or None.""" + return self._load_raw().get('default_model') + + def set_default_model(self, model: str, provider: Optional[str] = None) -> None: + data = self._load_raw() + data['default_model'] = f'{provider}/{model}' if provider else model + self._save_raw(data) diff --git a/ms_agent/config/resolver.py b/ms_agent/config/resolver.py index edcc32391..13bf6deb3 100644 --- a/ms_agent/config/resolver.py +++ b/ms_agent/config/resolver.py @@ -49,9 +49,28 @@ GLOBAL_MCP_FILE = 'mcp.json' GLOBAL_SKILLS_FILE = 'skills.json' PROJECT_CONFIG_FILE = 'config.yaml' +PROJECT_SETTINGS_LOCAL_FILE = 'settings.local.json' PROJECT_MCP_FILE = 'mcp.json' PROJECT_SKILLS_FILE = 'skills.json' +# New project-local framework dir (underscore, matches ~/.ms_agent) and the +# older hyphenated name kept for read-compat. +PROJECT_INTERNAL_DIR = '.ms_agent' +PROJECT_INTERNAL_DIR_LEGACY = '.ms-agent' + + +def _project_internal_file(project_path: str, filename: str) -> Optional[Path]: + """Return /.ms_agent/, falling back to the legacy + /.ms-agent/. Returns the new-dir path (may not exist) when + neither exists, so callers can treat a missing file uniformly.""" + new = Path(project_path) / PROJECT_INTERNAL_DIR / filename + if new.exists(): + return new + legacy = Path(project_path) / PROJECT_INTERNAL_DIR_LEGACY / filename + if legacy.exists(): + return legacy + return new + class ConfigResolver: """Multi-layer config resolver for server/UI and Playground scenarios.""" @@ -234,14 +253,37 @@ def _load_global_settings(self) -> Optional[DictConfig]: return None def _load_project_patch(self, project_path: str) -> Optional[DictConfig]: - patch_file = Path(project_path) / '.ms-agent' / PROJECT_CONFIG_FILE - if not patch_file.exists(): - return None - try: - return OmegaConf.load(str(patch_file)) - except Exception as e: - logger.warning(f'Failed to load project config patch: {e}') + """Project-level config patch. + + Merges (low -> high priority): + 1. legacy/new ``config.yaml`` (agent schema, direct) + 2. ``settings.local.json`` (settings schema, mapped like the global + settings.json — provider/model/personalization/... ) + Both are optional; new ``.ms_agent/`` is preferred over legacy + ``.ms-agent/``. + """ + layers: List[DictConfig] = [] + + yaml_file = _project_internal_file(project_path, PROJECT_CONFIG_FILE) + if yaml_file and yaml_file.exists(): + try: + layers.append(OmegaConf.load(str(yaml_file))) + except Exception as e: + logger.warning(f'Failed to load project config.yaml: {e}') + + local_file = _project_internal_file( + project_path, PROJECT_SETTINGS_LOCAL_FILE) + if local_file and local_file.exists(): + try: + with open(local_file, 'r', encoding='utf-8') as f: + data = json.load(f) + layers.append(self._settings_to_agent_config(data)) + except (json.JSONDecodeError, OSError) as e: + logger.warning(f'Failed to load settings.local.json: {e}') + + if not layers: return None + return self._merge_layers(layers) # -- merging -- @@ -263,7 +305,7 @@ def _merge_mcp( project_mcp = {} if project_path: project_mcp = self._load_json_safe( - Path(project_path) / '.ms-agent' / PROJECT_MCP_FILE + _project_internal_file(project_path, PROJECT_MCP_FILE) ) if not global_mcp and not project_mcp: @@ -311,7 +353,7 @@ def _merge_skills( project_skills = {} if project_path: project_skills = self._load_json_safe( - Path(project_path) / '.ms-agent' / PROJECT_SKILLS_FILE + _project_internal_file(project_path, PROJECT_SKILLS_FILE) ) if not global_skills and not project_skills: @@ -357,6 +399,18 @@ def _settings_to_agent_config(settings: Dict[str, Any]) -> DictConfig: ] if agent_llm: agent_fields['llm'] = agent_llm + # default_model (from ModelSettingsManager) is a fallback when the llm + # block does not pin a model. Form: "provider/model" or bare "model". + default_model = settings.get('default_model') + if default_model: + agent_llm = agent_fields.setdefault('llm', {}) + if not agent_llm.get('model'): + if '/' in default_model: + prov, mdl = default_model.split('/', 1) + agent_llm.setdefault('service', prov) + agent_llm['model'] = mdl + else: + agent_llm['model'] = default_model if 'output_dir' in settings: agent_fields['output_dir'] = settings['output_dir'] if 'personalization' in settings: diff --git a/ms_agent/config/skills_manager.py b/ms_agent/config/skills_manager.py index cc05ffe0a..02a76f422 100644 --- a/ms_agent/config/skills_manager.py +++ b/ms_agent/config/skills_manager.py @@ -106,7 +106,8 @@ def _global_path(self) -> Path: return self._global_dir / SKILLS_FILE def _project_path(self, project_path: str) -> Path: - return Path(project_path) / PROJECT_META_DIR / SKILLS_FILE + from ms_agent.project.paths import project_internal_file + return project_internal_file(project_path, SKILLS_FILE) def _resolve_path(self, scope: str, project_path: Optional[str]) -> Path: if scope == 'project': diff --git a/ms_agent/hooks/factory.py b/ms_agent/hooks/factory.py index 0327bf202..a5af94d22 100644 --- a/ms_agent/hooks/factory.py +++ b/ms_agent/hooks/factory.py @@ -144,7 +144,8 @@ def build_hook_runtime( )) if 'native' in enabled_sources: - ms_agent_hooks_json = Path(project_path) / '.ms-agent' / 'hooks.json' + from ms_agent.project.paths import project_internal_file + ms_agent_hooks_json = project_internal_file(project_path, 'hooks.json') if ms_agent_hooks_json.is_file(): loaders.append(( 'ms_agent_json', @@ -221,7 +222,11 @@ def _discover_plugin_roots(config: Any, project_path: str) -> list[str]: managed_ids = registry.managed_plugin_ids(project_path) roots: list[str] = [] seen: set[str] = set() - plugins_dir = Path(project_path) / '.ms-agent' / 'plugins' + from ms_agent.project.paths import (project_internal_dir, + legacy_local_internal_dir) + plugins_dir = project_internal_dir(project_path) / 'plugins' + if not plugins_dir.is_dir(): + plugins_dir = legacy_local_internal_dir(project_path) / 'plugins' if plugins_dir.is_dir(): for child in plugins_dir.iterdir(): if not child.is_dir(): diff --git a/ms_agent/hooks/permission_resolve.py b/ms_agent/hooks/permission_resolve.py index b3c775887..0bdaddb09 100644 --- a/ms_agent/hooks/permission_resolve.py +++ b/ms_agent/hooks/permission_resolve.py @@ -59,7 +59,11 @@ async def resolve_hook_permission_decision( permission_enforcer: PermissionEnforcer | None, permission_config: PermissionConfig | None, hook_runtime: HookRuntime | None = None, + force_decision: PermissionDecision | None = None, ) -> PermissionDecision | str: + # ``force_decision`` carries a SafetyGuard ``ask`` (interactive): it must + # reach the handler even when whitelist/memory would otherwise allow, so a + # broad whitelist can't silently bypass a safety confirmation (REVIEW P1-2). if hook_result and hook_result.action == 'deny': return f'Blocked by hook: {hook_result.reason}' @@ -106,5 +110,6 @@ async def resolve_hook_permission_decision( ) if permission_enforcer: - return await permission_enforcer.check(tool_name, args) + return await permission_enforcer.check( + tool_name, args, force_decision=force_decision) return PermissionDecision(action='allow', reason='No permission enforcer') diff --git a/ms_agent/llm/openai_llm.py b/ms_agent/llm/openai_llm.py index d9a9ef927..93afe1586 100644 --- a/ms_agent/llm/openai_llm.py +++ b/ms_agent/llm/openai_llm.py @@ -72,10 +72,18 @@ def __init__( self.model: str = config.llm.model self.max_continue_runs = getattr(config.llm, 'max_continue_runs', None) or MAX_CONTINUE_RUNS + # Resolution: explicit arg -> config field -> env var -> service + # default. The env fallback keeps `service: openai` working with a + # `.env` that only sets OPENAI_BASE_URL/OPENAI_API_KEY (matching the + # provider-router CredentialResolver), instead of silently hitting the + # real OpenAI endpoint. + import os base_url = base_url or getattr( - config.llm, 'openai_base_url', - None) or get_service_config('openai').base_url - api_key = api_key or getattr(config.llm, 'openai_api_key', None) + config.llm, 'openai_base_url', None) or os.environ.get( + 'OPENAI_BASE_URL') or get_service_config('openai').base_url + api_key = api_key or getattr( + config.llm, 'openai_api_key', None) or os.environ.get( + 'OPENAI_API_KEY') self.client = openai.OpenAI( api_key=api_key, diff --git a/ms_agent/llm/transport/openai_compat.py b/ms_agent/llm/transport/openai_compat.py index 7f73cd24d..cba501823 100644 --- a/ms_agent/llm/transport/openai_compat.py +++ b/ms_agent/llm/transport/openai_compat.py @@ -407,7 +407,8 @@ def _stream_continue_generate( message.reasoning_tokens += self._extract_reasoning_tokens( usage) first_run = not messages[-1].to_dict().get(flag, False) - if chunk.choices[0].finish_reason in [ + if self.continue_gen_mode and chunk.choices[ + 0].finish_reason in [ 'length', 'null' ] and (max_runs is None or max_runs != 0): logger.info( @@ -555,7 +556,7 @@ def _continue_generate(self, **kwargs) -> Message: flag = self._continue_flag new_message = self._format_output_message(completion) - if completion.choices[0].finish_reason in [ + if self.continue_gen_mode and completion.choices[0].finish_reason in [ 'length', 'null' ] and (max_runs is None or max_runs != 0): logger.info( diff --git a/ms_agent/memory/default_memory.py b/ms_agent/memory/default_memory.py index 3f76a1bc8..d32b02ad8 100644 --- a/ms_agent/memory/default_memory.py +++ b/ms_agent/memory/default_memory.py @@ -88,9 +88,12 @@ def __init__(self, config: DictConfig): self.run_id: Optional[str] = getattr(memory_config, 'run_id', None) self.compress: Optional[bool] = getattr(config, 'compress', True) self.is_retrieve: Optional[bool] = getattr(config, 'is_retrieve', True) - self.path: Optional[str] = getattr( - memory_config, 'path', - os.path.join(DEFAULT_OUTPUT_DIR, '.default_memory')) + _mem_path = getattr(memory_config, 'path', None) + if not _mem_path: + from ms_agent.project.paths import memory_dir + _work = getattr(config, 'output_dir', None) or DEFAULT_OUTPUT_DIR + _mem_path = str(memory_dir(_work) / 'default') + self.path: Optional[str] = _mem_path self.history_mode = getattr(memory_config, 'history_mode', 'add') self.ignore_roles: List[str] = getattr(memory_config, 'ignore_roles', ['tool', 'system']) diff --git a/ms_agent/memory/unified/orchestrator.py b/ms_agent/memory/unified/orchestrator.py index ee5193b8e..eb979189d 100644 --- a/ms_agent/memory/unified/orchestrator.py +++ b/ms_agent/memory/unified/orchestrator.py @@ -157,11 +157,22 @@ def _parse_config(self, config: Any) -> MemoryConfig: if isinstance(config, MemoryConfig): return config if hasattr(config, "memory") and hasattr(config.memory, "unified_memory"): - return MemoryConfig.from_dict_config(config.memory.unified_memory) + mc = MemoryConfig.from_dict_config(config.memory.unified_memory) + self._default_base_dir(mc, config) + return mc if hasattr(config, "unified_memory"): return MemoryConfig.from_dict_config(config.unified_memory) return MemoryConfig.from_dict_config(config) + @staticmethod + def _default_base_dir(mc: MemoryConfig, config: Any) -> None: + """When base_dir is unset ('.'), root memory under /.ms_agent/memory + instead of the CWD (so MEMORY.md / facts / index don't scatter).""" + if str(getattr(mc, "base_dir", ".")).strip() in (".", "", "./"): + from ms_agent.project.paths import memory_dir + work = getattr(config, "output_dir", None) or "." + mc.base_dir = str(memory_dir(work)) + # =================================================================== # Message conversion helpers diff --git a/ms_agent/permission/config.py b/ms_agent/permission/config.py index 5394fc55d..ae30e7f2a 100644 --- a/ms_agent/permission/config.py +++ b/ms_agent/permission/config.py @@ -116,8 +116,14 @@ def from_dict(cls, d: dict[str, Any], project_root: str | None = None) -> Permis whitelist = tuple(d.get('whitelist', ())) ask_rules = tuple(d.get('ask_rules', ())) user_blacklist = tuple(d.get('blacklist', ())) - blacklist = _DEFAULT_BLACKLIST + tuple( - p for p in user_blacklist if p not in _DEFAULT_BLACKLIST + # The default blacklist blocks network-egress shell commands + # (curl/wget/ssh/...). ``allow_network: true`` opts out of that secure + # default (e.g. to restore legacy "auto = no interception" behavior); + # the user's own blacklist entries still apply. + allow_network = bool(d.get('allow_network', False)) + base_blacklist = () if allow_network else _DEFAULT_BLACKLIST + blacklist = base_blacklist + tuple( + p for p in user_blacklist if p not in base_blacklist ) safety_raw = d.get('safety_rules', {}) diff --git a/ms_agent/permission/enforcer.py b/ms_agent/permission/enforcer.py index c1b788cee..c63fd3c40 100644 --- a/ms_agent/permission/enforcer.py +++ b/ms_agent/permission/enforcer.py @@ -6,6 +6,7 @@ from __future__ import annotations +import asyncio from dataclasses import dataclass from typing import Any, Literal @@ -41,6 +42,24 @@ def __init__( self._handler = handler or AutoPermissionHandler() self._memory = memory or PermissionMemory() self._matcher = PermissionMatcher() + # Parallel tool calls (asyncio.gather in ToolManager.parallel_call_tool) + # would otherwise invoke the interactive handler concurrently — N + # prompts fighting over one terminal deadlocks. Serialize asks with a + # lock created lazily per running loop (the per-turn TUI uses a fresh + # loop each turn, so a single init-time Lock would bind to the wrong one). + self._ask_lock: asyncio.Lock | None = None + self._ask_lock_loop = None + + def _ask_lock_for_loop(self) -> 'asyncio.Lock': + loop = asyncio.get_running_loop() + if self._ask_lock is None or self._ask_lock_loop is not loop: + self._ask_lock = asyncio.Lock() + self._ask_lock_loop = loop + return self._ask_lock + + async def _serialized_ask(self, **kwargs) -> PermissionResponse: + async with self._ask_lock_for_loop(): + return await self._handler.ask(**kwargs) async def check( self, @@ -59,7 +78,7 @@ async def check( if force_decision and force_decision.action == 'ask': suggestions = generate_suggestions(tool_name, tool_args) - response = await self._handler.ask( + response = await self._serialized_ask( tool_name=tool_name, tool_args=tool_args, context=force_decision.reason or '', @@ -86,9 +105,9 @@ async def check( reason='Allowed by remembered permission', ) - # 5. Ask user via handler + # 5. Ask user via handler (serialized against parallel tool calls) suggestions = generate_suggestions(tool_name, tool_args) - response = await self._handler.ask( + response = await self._serialized_ask( tool_name=tool_name, tool_args=tool_args, context='', diff --git a/ms_agent/permission/path_extractors.py b/ms_agent/permission/path_extractors.py index bd1f3c9b9..5cdf4bd11 100644 --- a/ms_agent/permission/path_extractors.py +++ b/ms_agent/permission/path_extractors.py @@ -357,7 +357,7 @@ def _make_filter_entry( def build_extractor_registry() -> dict[str, ExtractorEntry]: - """Build the full 34-command extractor registry.""" + """Build the full 36-command extractor registry.""" registry: dict[str, ExtractorEntry] = {} # Special commands diff --git a/ms_agent/permission/path_validator.py b/ms_agent/permission/path_validator.py index f2fc4a7b5..0eb68c220 100644 --- a/ms_agent/permission/path_validator.py +++ b/ms_agent/permission/path_validator.py @@ -151,12 +151,25 @@ def validate_path( ) -def is_dangerous_removal_path(path: str) -> bool: - """Check if a path is too dangerous for rm/rmdir, even within allowed dirs.""" +def is_dangerous_removal_path( + path: str, extra_patterns: 'tuple[str, ...]' = ()) -> bool: + """Check if a path is too dangerous for rm/rmdir, even within allowed dirs. + + ``extra_patterns`` are configurable (``safety_rules.dangerous_removal_paths``): + each is expanded (``~``) and matched against the normalized path both + literally and as an fnmatch glob (REVIEW P1-8).""" + import fnmatch normalized = _CONSECUTIVE_SLASHES.sub('/', path) if normalized.endswith('/') and len(normalized) > 1: normalized = normalized.rstrip('/') + for pat in extra_patterns or (): + pat_expanded = _CONSECUTIVE_SLASHES.sub( + '/', os.path.expanduser(str(pat))) + if (normalized == pat_expanded.rstrip('/') + or fnmatch.fnmatch(normalized, pat_expanded)): + return True + if normalized == '*': return True if normalized.endswith('/*') or normalized.endswith('\\*'): diff --git a/ms_agent/permission/safety.py b/ms_agent/permission/safety.py index 81a06206a..0f54e8b3f 100644 --- a/ms_agent/permission/safety.py +++ b/ms_agent/permission/safety.py @@ -39,6 +39,7 @@ def __init__( allowed_directories=tuple(self._allowed_dirs), read_only_directories=tuple(self._read_only_dirs), workspace_root=workspace_root, + dangerous_removal_paths=tuple(config.dangerous_removal_paths), ) self._shell_validator = ShellPathValidator( allowed_dirs=self._allowed_dirs, diff --git a/ms_agent/permission/shell_validator.py b/ms_agent/permission/shell_validator.py index e4de48ac5..36e3b46ff 100644 --- a/ms_agent/permission/shell_validator.py +++ b/ms_agent/permission/shell_validator.py @@ -54,6 +54,7 @@ class PathSafetyConfig: allowed_directories: tuple[str, ...] = () read_only_directories: tuple[str, ...] = () workspace_root: str | None = None + dangerous_removal_paths: tuple[str, ...] = () class ShellPathValidator: @@ -287,7 +288,8 @@ def _validate_paths( for path in paths: # Dangerous removal check for rm/rmdir - if cmd_name in ('rm', 'rmdir') and is_dangerous_removal_path(path): + if cmd_name in ('rm', 'rmdir') and is_dangerous_removal_path( + path, self._config.dangerous_removal_paths): return SafetyDecision( action='deny', reason=f'Dangerous removal path: {path}', diff --git a/ms_agent/plugins/config_manager.py b/ms_agent/plugins/config_manager.py index a94d49a16..49dcb1481 100644 --- a/ms_agent/plugins/config_manager.py +++ b/ms_agent/plugins/config_manager.py @@ -34,7 +34,8 @@ def global_plugins_path(self) -> Path: def project_plugins_path(self) -> Path: if self.project_root is None: raise ValueError('project_root is required for project scope') - return self.project_root / PROJECT_META_DIR / PLUGIN_FILE + from ms_agent.project.paths import project_internal_file + return project_internal_file(self.project_root, PLUGIN_FILE) @property def global_plugins_dir(self) -> Path: @@ -44,7 +45,8 @@ def global_plugins_dir(self) -> Path: def project_plugins_dir(self) -> Path: if self.project_root is None: raise ValueError('project_root is required for project scope') - return self.project_root / PROJECT_META_DIR / 'plugins' + from ms_agent.project.paths import project_internal_dir + return project_internal_dir(self.project_root) / 'plugins' @property def global_plugin_data_root(self) -> Path: diff --git a/ms_agent/plugins/installer.py b/ms_agent/plugins/installer.py index 3b16879e2..a27ef93e4 100644 --- a/ms_agent/plugins/installer.py +++ b/ms_agent/plugins/installer.py @@ -228,7 +228,8 @@ def _target_dir( root = Path(project_path or self.project_root or '') if not str(root): raise ValueError('project_path is required for project plugin install') - return root / '.ms-agent' / 'plugins' / plugin_id + from ms_agent.project.paths import project_internal_dir + return project_internal_dir(root) / 'plugins' / plugin_id return self.global_root / 'plugins' / plugin_id def _ensure_dependencies( diff --git a/ms_agent/plugins/runtime.py b/ms_agent/plugins/runtime.py index f871ee20f..75daf818b 100644 --- a/ms_agent/plugins/runtime.py +++ b/ms_agent/plugins/runtime.py @@ -563,6 +563,8 @@ def _is_managed_plugin_path( (global_root / 'plugins').expanduser().resolve(), ] if project_root is not None: + allowed_roots.append( + (project_root / '.ms_agent' / 'plugins').expanduser().resolve()) allowed_roots.append( (project_root / '.ms-agent' / 'plugins').expanduser().resolve()) for root in allowed_roots: diff --git a/ms_agent/project/manager.py b/ms_agent/project/manager.py index 3dcbec892..5c892cf40 100644 --- a/ms_agent/project/manager.py +++ b/ms_agent/project/manager.py @@ -11,9 +11,22 @@ class ProjectManager: """Project CRUD. Pure SDK interface, no IO assumptions.""" PROJECTS_DIR = 'projects' - META_DIR = '.ms-agent' + META_DIR = '.ms_agent' + LEGACY_META_DIR = '.ms-agent' META_FILE = 'project.json' + def _meta_file(self, project_id: str) -> Path: + """Project meta path — new ``.ms_agent`` first, legacy ``.ms-agent`` if + only that exists (read-compat).""" + base = self._projects_root / project_id + new = base / self.META_DIR / self.META_FILE + if new.exists(): + return new + legacy = base / self.LEGACY_META_DIR / self.META_FILE + if legacy.exists(): + return legacy + return new + def __init__(self, base_dir: str = '~/.ms_agent') -> None: self._base = Path(os.path.expanduser(base_dir)) self._projects_root = self._base / self.PROJECTS_DIR @@ -59,7 +72,7 @@ def list(self) -> list[Project]: for entry in sorted(self._projects_root.iterdir()): if not entry.is_dir(): continue - meta_file = entry / self.META_DIR / self.META_FILE + meta_file = self._meta_file(entry.name) if meta_file.exists(): store = JSONFileStore(meta_file) try: @@ -98,10 +111,10 @@ def get_default_project(self) -> Project: # -- internal -- def _ensure_default_project(self) -> None: - default_path = self._projects_root / DEFAULT_PROJECT_ID - meta_file = default_path / self.META_DIR / self.META_FILE + meta_file = self._meta_file(DEFAULT_PROJECT_ID) if meta_file.exists(): return + default_path = self._projects_root / DEFAULT_PROJECT_ID project = Project( id=DEFAULT_PROJECT_ID, name='Default', @@ -113,15 +126,15 @@ def _ensure_default_project(self) -> None: def _init_project_dirs(self, project: Project) -> None: project_dir = self._projects_root / project.id (project_dir / self.META_DIR).mkdir(parents=True, exist_ok=True) - (project_dir / self.META_DIR / 'sessions').mkdir(exist_ok=True) + # Sessions live at /sessions (flat), matching SessionManager + # (paths.py) — no meta-nested sessions dir. + (project_dir / 'sessions').mkdir(exist_ok=True) project_path = Path(project.path) project_path.mkdir(parents=True, exist_ok=True) (project_path / 'workspace').mkdir(exist_ok=True) def _meta_store(self, project_id: str) -> JSONFileStore: - return JSONFileStore( - self._projects_root / project_id / self.META_DIR / self.META_FILE - ) + return JSONFileStore(self._meta_file(project_id)) def _save_meta(self, project: Project) -> None: project_dir = self._projects_root / project.id diff --git a/ms_agent/project/paths.py b/ms_agent/project/paths.py new file mode 100644 index 000000000..ed0431d8f --- /dev/null +++ b/ms_agent/project/paths.py @@ -0,0 +1,159 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Canonical on-disk layout (Claude Code / Codex aligned). + +Single source of truth for where things live, so the runtime agent, the M1 +project layer, and a future WebUI/Server all agree: + +- **Working / project directory** == ``output_dir`` (one concept). Work products + may be written anywhere under it, like any coding agent. +- **Framework internals** live under ``/.ms_agent/`` (memory, snapshots, + project-level settings), collapsed into one namespace. +- **Session logs live globally**, decoupled from the work dir: + ``~/.ms_agent/projects//.jsonl`` where ```` is the escaped + absolute path of the work dir (CC-style, zero-config per directory). + +Legacy locations (``/sessions``, ``/.ms-agent``, +``/.ms_agent_snapshots``) are still *read* for back-compat. +""" +from __future__ import annotations + +import os +import re +from pathlib import Path + +INTERNAL_DIR_NAME = '.ms_agent' # project-local framework dir +LEGACY_INTERNAL_DIR_NAME = '.ms-agent' # older hyphenated name (read-compat) + + +def global_home() -> Path: + """The global ms-agent home. Read dynamically so ``MS_AGENT_HOME`` can be + redirected (e.g. per-test) without re-importing.""" + return Path(os.environ.get('MS_AGENT_HOME', '~/.ms_agent')).expanduser() + + +# Back-compat module attribute; prefer global_home(). +GLOBAL_HOME = global_home() + + +def _abs(work_dir: str | Path) -> Path: + return Path(work_dir).expanduser().resolve() + + +def project_key(work_dir: str | Path) -> str: + """CC-style stable key from the work dir's absolute path. + + ``/Users/x/proj`` -> ``-Users-x-proj``. Readable and directory-unique. + """ + abs_path = str(_abs(work_dir)) + key = re.sub(r'[^A-Za-z0-9._-]+', '-', abs_path) + return key.strip('-') or 'root' + + +def global_projects_root() -> Path: + return global_home() / 'projects' + + +def global_project_dir(work_dir: str | Path) -> Path: + """``~/.ms_agent/projects//`` — where sessions (and later project + metadata) for this work dir live.""" + return global_projects_root() / project_key(work_dir) + + +def global_sessions_dir(work_dir: str | Path) -> Path: + """Session logs live flat directly under the global project dir.""" + return global_project_dir(work_dir) + + +def local_internal_dir(work_dir: str | Path) -> Path: + """``/.ms_agent/`` — framework internals for this project.""" + return _abs(work_dir) / INTERNAL_DIR_NAME + + +def legacy_local_internal_dir(work_dir: str | Path) -> Path: + return _abs(work_dir) / LEGACY_INTERNAL_DIR_NAME + + +def snapshots_dir(work_dir: str | Path) -> Path: + """``/.ms_agent/snapshots`` (was ``/.ms_agent_snapshots``).""" + return local_internal_dir(work_dir) / 'snapshots' + + +def memory_dir(work_dir: str | Path) -> Path: + """``/.ms_agent/memory`` (was ``/.memory`` / ``.default_memory``).""" + return local_internal_dir(work_dir) / 'memory' + + +# ── generic + specific work-local internal subdirs ────────────────────────── +# Everything the framework writes for its own bookkeeping goes under one of +# these, so a work dir only ever contains the user's artifacts plus a single +# ``.ms_agent/`` namespace (no scattered ms_agent.log / .memory / .index / …). + +def internal_subdir(work_dir: str | Path, *parts: str) -> Path: + """``/.ms_agent/``.""" + return local_internal_dir(work_dir).joinpath(*parts) + + +def artifacts_dir(work_dir: str | Path) -> Path: + """Large tool-output spill (was ``/.ms_agent_artifacts``).""" + return internal_subdir(work_dir, 'artifacts') + + +def index_dir(work_dir: str | Path) -> Path: + """Search / filesystem / code indexes (was ``/.index``).""" + return internal_subdir(work_dir, 'index') + + +def locks_dir(work_dir: str | Path) -> Path: + """Lock files (was ``/.locks``).""" + return internal_subdir(work_dir, 'locks') + + +def tmp_dir(work_dir: str | Path) -> Path: + """Ephemeral working files (was ``/.temp``).""" + return internal_subdir(work_dir, 'tmp') + + +def subagents_dir(work_dir: str | Path) -> Path: + """Sub-agent stream logs (was ``/subagents``).""" + return internal_subdir(work_dir, 'subagents') + + +def web_search_dir(work_dir: str | Path) -> Path: + """Web-search payload spill (was ``/web_search_artifacts``).""" + return internal_subdir(work_dir, 'web_search') + + +def search_index_dir(work_dir: str | Path, name: str = 'search') -> Path: + """RAG / sirchmunk indexes (was ``./.sirchmunk`` / ``./llama_index``).""" + return internal_subdir(work_dir, 'search_index', name) + + +def stats_file(work_dir: str | Path, name: str = 'workflow_stats.json') -> Path: + """Token/usage stats (was ``/workflow_stats.json``).""" + return internal_subdir(work_dir, name) + + +def global_logs_dir() -> Path: + """``~/.ms_agent/logs`` — opt-in file logging target (never CWD).""" + return global_home() / 'logs' + + +# ── project-local config dir (writes prefer new, reads fall back to legacy) ── + +def project_internal_dir(project_path: str | Path) -> Path: + """New project-local framework dir for **writes** (``/.ms_agent``).""" + return _abs(project_path) / INTERNAL_DIR_NAME + + +def project_internal_file(project_path: str | Path, filename: str) -> Path: + """Resolve ``/.ms_agent/`` for reads, falling back to the legacy + ``/.ms-agent/`` when only the legacy exists. Returns the new-dir + path when neither exists so callers treat "missing" uniformly. Writers + should use ``project_internal_dir()`` (always the new dir).""" + new = _abs(project_path) / INTERNAL_DIR_NAME / filename + if new.exists(): + return new + legacy = _abs(project_path) / LEGACY_INTERNAL_DIR_NAME / filename + if legacy.exists(): + return legacy + return new diff --git a/ms_agent/project/session.py b/ms_agent/project/session.py index f7a35e52d..4189b303b 100644 --- a/ms_agent/project/session.py +++ b/ms_agent/project/session.py @@ -26,9 +26,21 @@ class SessionManager: def __init__(self, project: Project) -> None: self._project = project + # Sessions live globally under ~/.ms_agent/projects//sessions, + # co-located with the runtime SessionLog root (paths.py), not inside the + # project's work dir. Legacy /.ms-agent/sessions is migrated + # forward once if present. + from ms_agent.project.paths import global_projects_root self._sessions_dir = ( - Path(project.path) / '.ms-agent' / 'sessions' + global_projects_root() / project.id / 'sessions' ) + legacy = Path(project.path) / '.ms-agent' / 'sessions' + if legacy.is_dir() and not self._sessions_dir.exists(): + try: + import shutil + shutil.copytree(legacy, self._sessions_dir) + except Exception: + pass self._sessions_dir.mkdir(parents=True, exist_ok=True) @property diff --git a/ms_agent/project/workspace.py b/ms_agent/project/workspace.py index 1e289d8fd..d6c192d94 100644 --- a/ms_agent/project/workspace.py +++ b/ms_agent/project/workspace.py @@ -85,6 +85,57 @@ def import_path(self, source: str) -> None: else: shutil.copy2(src, dest) + # -- binary / HTTP transfer (WebUI upload & packaged download) -- + + def read_bytes(self, rel_path: str) -> bytes: + target = self._resolve_safe(rel_path) + if not target.is_file(): + raise FileNotFoundError(f'Not a file: {rel_path}') + return target.read_bytes() + + def write_bytes(self, rel_path: str, data: bytes) -> None: + target = self._resolve_safe(rel_path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + + def save_upload(self, filename: str, stream, subdir: str = '.') -> str: + """HTTP upload: write a binary stream (file-like or bytes) into the + workspace. ``filename`` is basename-only (dir components stripped); + returns the stored path relative to the workspace root.""" + rel = str(Path(subdir) / Path(filename).name) + target = self._resolve_safe(rel) + target.parent.mkdir(parents=True, exist_ok=True) + if hasattr(stream, 'read'): + with open(target, 'wb') as f: + shutil.copyfileobj(stream, f) + else: + target.write_bytes(bytes(stream)) + return str(target.relative_to(self._root)) + + def zip_download(self, rel_path: str = '.') -> bytes: + """Package a workspace file/dir into a zip and return the bytes.""" + import io + import zipfile + target = self._resolve_safe(rel_path) + if not target.exists(): + raise FileNotFoundError(f'Path not found: {rel_path}') + buf = io.BytesIO() + with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf: + if target.is_file(): + zf.write(target, target.name) + else: + for p in sorted(target.rglob('*')): + if p.is_file(): + # Skip entries that resolve outside the workspace: a + # planted symlink (e.g. -> /etc/passwd) would otherwise + # be followed by is_file() and packaged (info leak). + try: + p.resolve().relative_to(self._root) + except ValueError: + continue + zf.write(p, p.relative_to(target)) + return buf.getvalue() + def _resolve_safe(self, rel_path: str) -> Path: target = (self._root / rel_path).resolve() # Use relative_to() rather than str.startswith(): the latter would diff --git a/ms_agent/rag/llama_index_rag.py b/ms_agent/rag/llama_index_rag.py index 129fda95f..990edbfd4 100644 --- a/ms_agent/rag/llama_index_rag.py +++ b/ms_agent/rag/llama_index_rag.py @@ -33,7 +33,12 @@ def __init__(self, config: DictConfig): self.chunk_size = getattr(config.rag, 'chunk_size', 512) self.chunk_overlap = getattr(config.rag, 'chunk_overlap', 50) self.retrieve_only = getattr(config.rag, 'retrieve_only', False) - self.storage_dir = getattr(config.rag, 'storage_dir', './llama_index') + _rag_store = getattr(config.rag, 'storage_dir', None) + if not _rag_store: + from ms_agent.project.paths import search_index_dir + _rag_store = str( + search_index_dir(getattr(config, 'output_dir', None) or '.', 'rag')) + self.storage_dir = _rag_store self._validate_requirements() self._setup_embedding_model(config) diff --git a/ms_agent/skill/catalog.py b/ms_agent/skill/catalog.py index 90137fc69..7ad9a4ec3 100644 --- a/ms_agent/skill/catalog.py +++ b/ms_agent/skill/catalog.py @@ -79,7 +79,8 @@ def _download_skill_zip(skill_id: str, local_dir: str) -> str: if _candidate.exists(): BUILTIN_SKILLS_DIR = _candidate -USER_SKILLS_DIR = Path.home() / ".ms_agent" / "skills" +from ms_agent.project.paths import global_home as _global_home +USER_SKILLS_DIR = _global_home() / "skills" class SkillCatalog: diff --git a/ms_agent/tools/agent_tool.py b/ms_agent/tools/agent_tool.py index 51fbfd121..5150f9571 100644 --- a/ms_agent/tools/agent_tool.py +++ b/ms_agent/tools/agent_tool.py @@ -1283,8 +1283,8 @@ def _save_transcript(self, messages: Any, if not isinstance(messages, list) or not agent_tag: return try: - output_dir = getattr(self.config, 'output_dir', './output') - subagents_dir = os.path.join(output_dir, 'subagents') + output_dir = getattr(self.config, 'output_dir', None) or '.' + subagents_dir = os.path.join(output_dir, '.ms_agent', 'subagents') os.makedirs(subagents_dir, exist_ok=True) path = os.path.join(subagents_dir, f'{agent_tag}.jsonl') with open(path, 'w', encoding='utf-8') as f: diff --git a/ms_agent/tools/audio_generator/audio_gen.py b/ms_agent/tools/audio_generator/audio_gen.py index 2b533c08b..14067e1dd 100644 --- a/ms_agent/tools/audio_generator/audio_gen.py +++ b/ms_agent/tools/audio_generator/audio_gen.py @@ -9,7 +9,7 @@ class AudioGenerator(ToolBase): def __init__(self, config): super().__init__(config) - self.temp_dir = os.path.join(self.output_dir, '.temp', + self.temp_dir = os.path.join(self.output_dir, '.ms_agent', 'tmp', 'audio_generator') os.makedirs(self.temp_dir, exist_ok=True) audio_generator = self.config.audio_generator diff --git a/ms_agent/tools/code/local_code_executor.py b/ms_agent/tools/code/local_code_executor.py index 54ed4fc21..15cd47735 100644 --- a/ms_agent/tools/code/local_code_executor.py +++ b/ms_agent/tools/code/local_code_executor.py @@ -60,7 +60,7 @@ async def start(self) -> None: # Ensure dependencies exist before importing. install_package('ipykernel') - install_package('jupyter-client') + install_package('jupyter-client', 'jupyter_client') from jupyter_client import AsyncKernelManager diff --git a/ms_agent/tools/code_server/lsp_code_server.py b/ms_agent/tools/code_server/lsp_code_server.py index ac7f3fbaf..d993c6a31 100644 --- a/ms_agent/tools/code_server/lsp_code_server.py +++ b/ms_agent/tools/code_server/lsp_code_server.py @@ -492,7 +492,7 @@ async def start(self) -> bool: return False # Create workspace data directory for jdtls - workspace_data_dir = Path(self.workspace_dir) / '.jdtls_workspace' + workspace_data_dir = Path(self.workspace_dir) / '.ms_agent' / 'lsp' / 'jdtls' workspace_data_dir.mkdir(exist_ok=True) # Start jdtls @@ -559,7 +559,7 @@ async def connect(self) -> None: def cleanup_lsp_index_dirs(self): cleanup_dirs = [ - os.path.join(self.output_dir, '.jdtls_workspace'), # Java LSP + os.path.join(self.output_dir, '.ms_agent', 'lsp', 'jdtls'), # Java LSP os.path.join(self.output_dir, '.pyright'), # Python LSP (if exists) os.path.join(self.output_dir, 'node_modules', diff --git a/ms_agent/tools/image_generator/image_gen.py b/ms_agent/tools/image_generator/image_gen.py index 3ab8a71df..c74291386 100644 --- a/ms_agent/tools/image_generator/image_gen.py +++ b/ms_agent/tools/image_generator/image_gen.py @@ -9,7 +9,7 @@ class ImageGenerator(ToolBase): def __init__(self, config): super().__init__(config) - self.temp_dir = os.path.join(self.output_dir, '.temp', + self.temp_dir = os.path.join(self.output_dir, '.ms_agent', 'tmp', 'image_generator') os.makedirs(self.temp_dir, exist_ok=True) image_generator = self.config.image_generator diff --git a/ms_agent/tools/search/sirchmunk_search.py b/ms_agent/tools/search/sirchmunk_search.py index cd8aaacf5..90b6a8d56 100644 --- a/ms_agent/tools/search/sirchmunk_search.py +++ b/ms_agent/tools/search/sirchmunk_search.py @@ -84,7 +84,12 @@ def __init__(self, config: DictConfig): str(Path(p).expanduser().resolve()) for p in paths ] - _work_path = rag_config.get('work_path', './.sirchmunk') + _work_path = rag_config.get('work_path', None) + if not _work_path: + from ms_agent.project.paths import search_index_dir + _work_path = str( + search_index_dir(getattr(config, 'output_dir', None) or '.', + 'sirchmunk')) self.work_path: Path = Path(_work_path).expanduser().resolve() self.reuse_knowledge = rag_config.get('reuse_knowledge', True) diff --git a/ms_agent/tools/search/websearch_tool.py b/ms_agent/tools/search/websearch_tool.py index 167bbe3bb..aab578184 100644 --- a/ms_agent/tools/search/websearch_tool.py +++ b/ms_agent/tools/search/websearch_tool.py @@ -625,8 +625,8 @@ def __init__(self, config, **kwargs): getattr(tool_cfg, 'spill_max_inline_chars', 120000) or 120000) if tool_cfg else 120000 self._spill_subdir = str( - getattr(tool_cfg, 'spill_subdir', 'web_search_artifacts') - or 'web_search_artifacts') if tool_cfg else 'web_search_artifacts' + getattr(tool_cfg, 'spill_subdir', '.ms_agent/web_search') + or '.ms_agent/web_search') if tool_cfg else '.ms_agent/web_search' self._spill_preview_chars = int( getattr(tool_cfg, 'spill_preview_chars', 600) or 600) if tool_cfg else 600 diff --git a/ms_agent/tools/tool_manager.py b/ms_agent/tools/tool_manager.py index 10884be22..5a0430db0 100644 --- a/ms_agent/tools/tool_manager.py +++ b/ms_agent/tools/tool_manager.py @@ -458,6 +458,7 @@ async def single_call_tool(self, tool_info: ToolCall): # --- Permission checks --- args_dict = dict(tool_args) if isinstance(tool_args, dict) else {} + safety_force_decision = None if self._safety_guard is not None: from ms_agent.permission.ask_resolver import resolve_ask safety_decision = self._safety_guard.check(tool_name, args_dict) @@ -470,7 +471,12 @@ async def single_call_tool(self, tool_info: ToolCall): if resolved.action == 'ask': if self._permission_enforcer is None: return f'Blocked by safety policy (requires confirmation): {resolved.reason}' - # interactive mode: fall through to enforcer/handler + # interactive mode: force the enforcer to confirm with + # the user; whitelist/memory must not bypass a safety + # ask (REVIEW P1-2). + from ms_agent.permission.enforcer import PermissionDecision + safety_force_decision = PermissionDecision( + action='ask', reason=resolved.reason) # --- PreToolUse hooks --- hook_result = None @@ -497,6 +503,7 @@ async def single_call_tool(self, tool_info: ToolCall): permission_enforcer=self._permission_enforcer, permission_config=self._permission_config, hook_runtime=self._hook_runtime, + force_decision=safety_force_decision, ) if isinstance(perm_out, str): return perm_out diff --git a/ms_agent/tools/video_generator/video_gen.py b/ms_agent/tools/video_generator/video_gen.py index 7578ebf59..dddb7247b 100644 --- a/ms_agent/tools/video_generator/video_gen.py +++ b/ms_agent/tools/video_generator/video_gen.py @@ -9,7 +9,7 @@ class VideoGenerator(ToolBase): def __init__(self, config): super().__init__(config) - self.temp_dir = os.path.join(self.output_dir, '.temp', + self.temp_dir = os.path.join(self.output_dir, '.ms_agent', 'tmp', 'video_generator') os.makedirs(self.temp_dir, exist_ok=True) video_generator = self.config.video_generator diff --git a/ms_agent/tui/__init__.py b/ms_agent/tui/__init__.py new file mode 100644 index 000000000..3cc364901 --- /dev/null +++ b/ms_agent/tui/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Lightweight terminal UI (TUI) for ms-agent. + +A route-B REPL (per TUI-PLAN.md): reuses the LLMAgent lifecycle and the +CommandRouter, renders with `rich`, and reads input with `prompt_toolkit` +(graceful fallback to `input()`), so the same SDK that backs the future WebUI +can be exercised end-to-end from a terminal. +""" +from ms_agent.tui.io import RichConsoleIO + +__all__ = ['RichConsoleIO'] diff --git a/ms_agent/tui/app.py b/ms_agent/tui/app.py new file mode 100644 index 000000000..608f7d19b --- /dev/null +++ b/ms_agent/tui/app.py @@ -0,0 +1,416 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""TUI application: a route-B REPL over the LLMAgent lifecycle with sessions. + +Positioning: single user · multi-project · many sessions per project. Every +launch starts a fresh, independent conversation; ``/resume`` restores a past +one; ``/new`` starts another; ``/sessions`` lists them. Sessions are owned by +the M1 ``SessionManager`` (global ``~/.ms_agent/projects//sessions``) — the +same layer the WebUI will use — so the agent's own per-run session log is +disabled here and the TUI persists the one canonical conversation itself. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, List, Optional + +from omegaconf import OmegaConf +from rich.console import Console +from rich.markdown import Markdown +from rich.panel import Panel +from rich.syntax import Syntax +from rich.table import Table + +from ms_agent.config import Config +from ms_agent.config.env import Env +from ms_agent.llm.utils import Message +from ms_agent.tui.io import RichConsoleIO +from ms_agent.utils.logger import get_logger + +logger = get_logger() + +_MSG_KEYS = ('role', 'content', 'tool_calls', 'tool_call_id', 'name') + + +def _args_complete(args: Any) -> bool: + if isinstance(args, dict): + return bool(args) + if isinstance(args, str) and args.strip(): + try: + json.loads(args) + return True + except Exception: + return False + return False + + +def _looks_texty(s: str) -> bool: + t = s.lstrip() + return not (t.startswith('{') or t.startswith('[')) + + +def _dict_to_msg(d: dict) -> Message: + return Message(**{k: d[k] for k in _MSG_KEYS if k in d}) + + +class TuiApp: + + def __init__( + self, + config_path: str, + env_file: Optional[str] = None, + permission_mode: Optional[str] = None, + trust_remote_code: bool = False, + work_dir: Optional[str] = None, + ) -> None: + Env.load_dotenv_into_environ(env_file) + self.console = Console() + self.trust_remote_code = trust_remote_code + self.work_dir = str(Path(work_dir).expanduser().resolve() + if work_dir else Path.cwd().resolve()) + + config = Config.from_task(config_path) + config = self._prepare_config(config, permission_mode, self.work_dir) + self.config = config + + from ms_agent.command import CommandRouter, register_builtin_commands + from ms_agent.command.types import (CommandDef, CommandResult, + CommandResultType) + self.router = CommandRouter() + register_builtin_commands(self.router) + + # Session commands are intercepted in _dispatch (they need the + # SessionManager + REPL control); register defs so /help lists them and + # is_command() recognizes them. + async def _noop(ctx): + return CommandResult(type=CommandResultType.MESSAGE, content='') + self.router.register( + CommandDef(name='sessions', description='List sessions in this project', + category='session'), _noop) + self.router.register( + CommandDef(name='resume', + description='Resume a session: /resume <#|id>', + category='session'), _noop) + + self.io = RichConsoleIO(console=self.console, router=self.router) + + from ms_agent.agent.llm_agent import LLMAgent + self.agent = LLMAgent( + config, trust_remote_code=trust_remote_code, console_io=self.io) + + mode = str(getattr(getattr(config, 'permission', None), 'mode', 'auto')) + if mode in ('restricted', 'interactive'): + from ms_agent.tui.permission import TUIPermissionHandler + self.agent.set_permission_handler( + TUIPermissionHandler(console=self.console, io=self.io)) + self.permission_mode = mode + + # Session lifecycle (M1). The work dir *is* the project (CC-aligned): + # sessions live at ~/.ms_agent/projects//sessions, so + # different work dirs are different projects, each with its own sessions. + from ms_agent.project import SessionManager + from ms_agent.project.paths import project_key + from ms_agent.project.types import Project + proj = Project(id=project_key(self.work_dir), + name=Path(self.work_dir).name or 'project', + path=self.work_dir) + self._sm = SessionManager(proj) + self._model = str(getattr(getattr(config, 'llm', None), 'model', '') or '') + self.session = None + self._log = None + self._logged = 0 # non-system messages already persisted + self.messages: List[Message] = [Message(role='system', content='')] + + # -- config shaping -- + + @staticmethod + def _prepare_config(config, permission_mode, work_dir): + OmegaConf.update(config, 'generation_config.stream', True, merge=True) + OmegaConf.update( + config, 'generation_config.stream_output', True, merge=True) + OmegaConf.update( + config, 'generation_config.show_reasoning', False, merge=True) + OmegaConf.update(config, 'output_dir', work_dir, merge=True) + # The TUI owns session persistence via SessionManager; disable the + # agent's own per-run session log so turns don't fragment into files. + OmegaConf.update(config, 'session_log.enabled', False, merge=True) + if permission_mode: + OmegaConf.update( + config, 'permission.mode', permission_mode, merge=True) + # Merge the work-dir project patch (e.g. a persisted /model override) + # so it round-trips from /.ms_agent/config.yaml — consistent + # with where internals live. from_task only reads the config-file dir, + # which differs when --config points outside the work dir. + try: + from ms_agent.config.resolver import ConfigResolver + patch = ConfigResolver()._load_project_patch(work_dir) + if patch is not None: + config = OmegaConf.merge(config, patch) + except Exception: + logger.debug('work-dir config patch merge skipped', exc_info=True) + return config + + # -- session lifecycle -- + + def _new_session(self) -> None: + self.session = self._sm.create(model=self._model or None) + self._log = self._sm.get_session_log(self.session) + self._logged = 0 + self.messages = [Message(role='system', content='')] + + def _resume_session(self, session) -> bool: + self.session = session + self._log = self._sm.get_session_log(session) + try: + dicts = self._log.get_all_messages() + except Exception: + dicts = [] + convo = [_dict_to_msg(d) for d in dicts + if d.get('role') and d.get('role') != 'system'] + self.messages = [Message(role='system', content='')] + convo + self._logged = len(convo) + return True + + def _persist_turn(self) -> None: + convo = self.messages[1:] + for m in convo[self._logged:]: + try: + self._log.append({k: v for k, v in { + 'role': m.role, + 'content': m.content or '', + 'tool_calls': getattr(m, 'tool_calls', None), + 'tool_call_id': getattr(m, 'tool_call_id', None), + 'name': getattr(m, 'name', None), + }.items() if v not in (None, '')}) + except Exception: + pass + self._logged = len(convo) + # Name the session after its first user turn. + name = self.session.name + if name.startswith('Session '): + first_user = next((m.content for m in convo + if m.role == 'user' and m.content), '') + if first_user: + name = first_user.strip().splitlines()[0][:40] + try: + self._sm.update(self.session.id, name=name, + updated_at=datetime.now(timezone.utc).isoformat()) + self.session = self._sm.get(self.session.id) or self.session + except Exception: + pass + + # -- rendering -- + + def _banner(self) -> None: + llm = getattr(self.config, 'llm', None) + tools = list(self.config.tools.keys()) if getattr( + self.config, 'tools', None) else [] + t = Table.grid(padding=(0, 2)) + t.add_column(style='bold cyan', justify='right') + t.add_column() + t.add_row('model', f'{getattr(llm, "service", "?")} / ' + f'{getattr(llm, "model", "?")}') + t.add_row('tools', ', '.join(tools) or '[dim](none)[/]') + t.add_row('perm', self.permission_mode) + t.add_row('work dir', self.work_dir) + t.add_row('files', '[dim]internals → .ms_agent/ · ' + 'sessions → ~/.ms_agent/projects/[/]') + self.console.print( + Panel(t, title='[bold]ms-agent tui[/]', border_style='cyan', + subtitle='[dim]/help /sessions /resume /new /quit[/]')) + + def _render_tool_call(self, tc: dict) -> None: + name = tc.get('tool_name', '?') + args = tc.get('arguments', '') + if isinstance(args, str): + try: + args = json.loads(args) + except Exception: + pass + arg_str = json.dumps(args, ensure_ascii=False, indent=2) + if len(arg_str) > 800: + arg_str = arg_str[:800] + '\n…' + self.io.print(Panel( + Syntax(arg_str, 'json', theme='ansi_dark', word_wrap=True), + title=f'🔧 {name}', border_style='yellow', expand=False)) + + def _render_tool_result(self, msg: Message) -> None: + content = msg.content if isinstance(msg.content, str) else str( + msg.content) + if len(content) > 1200: + content = content[:1200] + '\n… (truncated)' + renderable = (Markdown(content) + if content.strip() and _looks_texty(content) + else content) + self.io.print(Panel(renderable, title=f'← {msg.name or "result"}', + border_style='green', expand=False)) + + # -- turn execution -- + + async def _agent_turn(self, text: str) -> None: + self.messages.append(Message(role='user', content=text)) + start = len(self.messages) + rendered_calls: set = set() + rendered_results: set = set() + final = None + try: + gen = await self.agent.run(self.messages, stream=True) + async for msgs in gen: + final = msgs + for m in msgs[start:]: + if m.role == 'assistant' and m.tool_calls: + for tc in m.tool_calls: + tid = tc.get('id') or json.dumps( + tc, sort_keys=True, default=str) + if tid not in rendered_calls and _args_complete( + tc.get('arguments')): + rendered_calls.add(tid) + self._render_tool_call(tc) + elif m.role == 'tool': + tid = getattr(m, 'tool_call_id', None) or id(m) + if tid not in rendered_results: + rendered_results.add(tid) + self._render_tool_result(m) + except Exception as e: + self.io.end_stream() + self.console.print(Panel( + f'[bold]{type(e).__name__}[/]: {e}', + title='[red]turn failed[/]', border_style='red', + subtitle='[dim]LOG_LEVEL=INFO for details[/]', expand=False)) + logger.warning('TUI turn failed', exc_info=True) + return + self.io.end_stream() + if final is not None: + self.messages = final + self._persist_turn() + + # -- session commands (TUI-owned; need SessionManager + REPL control) -- + + def _cmd_sessions(self) -> None: + sessions = self._sm.list() + if not sessions: + self.io.print('[dim]no sessions yet[/]') + return + t = Table(show_header=True, header_style='bold', box=None, padding=(0, 2)) + for c in ('#', 'name', 'id', 'updated', 'model'): + t.add_column(c) + for i, s in enumerate(sessions): + marker = '➤' if self.session and s.id == self.session.id else str(i) + t.add_row(marker, s.name, s.id, (s.updated_at or '')[:19], + s.model or '') + self.io.print(Panel(t, title='sessions', border_style='blue', + subtitle='[dim]/resume <#|id>[/]', expand=False)) + + def _cmd_resume(self, arg: str) -> None: + arg = arg.strip() + sessions = self._sm.list() + target = None + if arg.isdigit() and int(arg) < len(sessions): + target = sessions[int(arg)] + else: + target = next((s for s in sessions if s.id == arg), None) + if target is None: + self.io.print('[red]not found[/] — use /sessions to list, ' + '/resume <#|id>') + return + self._resume_session(target) + n = len(self.messages) - 1 + self.console.rule(f'[green]resumed[/] {target.name} ' + f'[dim]({target.id}, {n} msgs)[/]') + # replay a short tail so the user has context + for m in self.messages[1:][-4:]: + who = {'user': '[cyan]you[/]', 'assistant': '[green]assistant[/]', + 'tool': '[yellow]tool[/]'}.get(m.role, m.role) + snippet = (m.content or '')[:200].replace('\n', ' ') + if snippet: + self.io.print(f'{who} {snippet}') + + # -- dispatch -- + + async def _dispatch(self, text: str) -> bool: + """Returns True to quit the app.""" + cmd, args = self.router.parse_input(text) + if cmd == 'sessions': + self._cmd_sessions() + return False + if cmd == 'resume': + self._cmd_resume(args) + return False + if cmd in ('new', 'reset'): + self._new_session() + self.console.rule('[green]new session[/] ' + f'[dim]({self.session.id})[/]') + return False + from ms_agent.command.types import CommandContext, CommandResultType + ctx = CommandContext( + raw_input=text, command_name=cmd, args=args, source='tui', + runtime=getattr(self.agent, 'runtime', None), + extra={'router': self.router, 'messages': self.messages, + 'config': self.config}) + result = await self.router.dispatch(ctx) + if result is None: + await self._agent_turn(text) + return False + if result.type == CommandResultType.QUIT: + if result.content: + self.console.print(f'[dim]{result.content}[/]') + return True + if result.type == CommandResultType.SUBMIT_PROMPT: + await self._agent_turn(result.content) + return False + if result.content: + self.console.print( + Panel(result.content, border_style='blue', expand=False)) + return False + + @staticmethod + def _quiet_logs() -> None: + level = os.environ.get('LOG_LEVEL', 'ERROR').upper() + os.environ.setdefault('LOG_LEVEL', 'ERROR') + if level in ('INFO', 'DEBUG', 'WARNING'): + return + lg = logging.getLogger('ms_agent') + lg.setLevel(logging.ERROR) + for h in lg.handlers: + h.setLevel(logging.ERROR) + + # -- main loop -- + + def run(self) -> None: + self._quiet_logs() + self._banner() + self._new_session() # fresh, independent conversation each launch + while True: + try: + text = self.io.read_prompt().strip() + except (EOFError, KeyboardInterrupt): + self.console.print('\n[dim]bye[/]') + break + if not text: + continue + try: + if self.router.is_command(text): + should_quit = asyncio.run(self._dispatch(text)) + else: + asyncio.run(self._agent_turn(text)) + should_quit = False + if should_quit: + break + except KeyboardInterrupt: + self.console.print('\n[dim](interrupted)[/]') + continue + + +def main( + config_path: str, + env_file: Optional[str] = None, + permission_mode: Optional[str] = None, + trust_remote_code: bool = False, + work_dir: Optional[str] = None, +) -> None: + TuiApp(config_path, env_file=env_file, permission_mode=permission_mode, + trust_remote_code=trust_remote_code, work_dir=work_dir).run() diff --git a/ms_agent/tui/io.py b/ms_agent/tui/io.py new file mode 100644 index 000000000..2b7f76ba4 --- /dev/null +++ b/ms_agent/tui/io.py @@ -0,0 +1,104 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Console I/O sink for the TUI. + +``RichConsoleIO`` implements the small contract the agent's console-io seam +expects (``write`` / ``end_stream`` / ``print`` / ``read_prompt``). Streaming +assistant tokens are written live to stdout; the prompt uses ``prompt_toolkit`` +(slash-command completion + history) and falls back to ``input()`` when a TTY / +prompt_toolkit is unavailable (pipes, CI), so the TUI stays scriptable. +""" +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, List, Optional + +from rich.console import Console + +if TYPE_CHECKING: + from ms_agent.command.router import CommandRouter + + +class RichConsoleIO: + """Rich-backed console I/O used by the TUI and injected into LLMAgent.""" + + def __init__( + self, + console: Optional[Console] = None, + router: Optional['CommandRouter'] = None, + ) -> None: + self.console = console or Console() + self._router = router + self._streaming = False + # Shown as the prompt_toolkit bottom toolbar (model · perm · work-dir). + self.status = '' + self._pt_session = self._build_prompt_session() + + # -- prompt_toolkit session (best-effort) -- + + def _build_prompt_session(self): + try: + from prompt_toolkit import PromptSession + from prompt_toolkit.completion import WordCompleter + from prompt_toolkit.history import InMemoryHistory + + words: List[str] = [] + if self._router is not None: + for cmds in self._router.list_commands('tui').values(): + for c in cmds: + words.append(f'/{c.name}') + words.extend(f'/{a}' for a in c.aliases) + completer = WordCompleter( + sorted(set(words)), sentence=True, ignore_case=True) + return PromptSession( + history=InMemoryHistory(), completer=completer) + except Exception: + return None + + # -- streaming output (called by LLMAgent step) -- + + def write(self, text: str) -> None: + """Write a streaming assistant content delta (no newline).""" + if not self._streaming: + self.console.print('[bold green]assistant[/] ', end='') + self._streaming = True + sys.stdout.write(text) + sys.stdout.flush() + + def end_stream(self) -> None: + """End the current streaming assistant message.""" + if self._streaming: + sys.stdout.write('\n') + sys.stdout.flush() + self._streaming = False + + # -- structured output -- + + def print(self, text: str = '') -> None: + self._flush_stream() + self.console.print(text) + + def rule(self, title: str = '') -> None: + self._flush_stream() + self.console.rule(title) + + def _flush_stream(self) -> None: + if self._streaming: + sys.stdout.write('\n') + sys.stdout.flush() + self._streaming = False + + # -- input -- + + def read_prompt(self, prompt: str = '❯ ') -> str: + """Read a line of input. prompt_toolkit if possible, else input().""" + self._flush_stream() + if self._pt_session is not None and sys.stdin.isatty(): + try: + # No bottom_toolbar: it caused a persistent flicker in some + # terminals. Status (model/perm/work-dir) lives in the banner. + return self._pt_session.prompt(prompt) + except (EOFError, KeyboardInterrupt): + raise + except Exception: + pass + return input(prompt) diff --git a/ms_agent/tui/permission.py b/ms_agent/tui/permission.py new file mode 100644 index 000000000..c4206b07d --- /dev/null +++ b/ms_agent/tui/permission.py @@ -0,0 +1,71 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Rich-styled permission handler for the TUI. + +Same 5-choice contract as ``CLIPermissionHandler`` ([y]/[s]/[a]/[e]/[n]) but +rendered through the shared ``rich`` console so the confirmation matches the +rest of the TUI instead of a raw stderr box. +""" +from __future__ import annotations + +import asyncio +import json +from typing import Any, Optional + +from rich.console import Console +from rich.panel import Panel + + +class TUIPermissionHandler: + def __init__(self, console: Optional[Console] = None, io: Any = None) -> None: + self._console = console or Console() + self._io = io + + async def ask(self, tool_name, tool_args, context, suggestions=None): + from ms_agent.permission.handler import (PermissionAction, + PermissionResponse) + args_str = json.dumps(tool_args, ensure_ascii=False, indent=2) + if len(args_str) > 600: + args_str = args_str[:600] + '\n…' + suggestion = suggestions[0] if suggestions else tool_name + body = ( + f'[bold]tool[/] {tool_name}\n' + f'[bold]args[/] {args_str}\n' + + (f'[dim]{context}[/]\n' if context else '') + + '\n[green]y[/] allow once [green]s[/] allow session ' + + '[green]a[/] always [yellow]e[/] edit args [red]n[/] deny') + self._console.print( + Panel(body, title='🔒 Permission required', + border_style='magenta', expand=False)) + + # Read via plain input() in the executor. The permission ask runs from + # a worker thread (run_in_executor) while the main loop awaits; using + # prompt_toolkit here would wrestle the main thread for the terminal. + # Enforcer serializes asks, so a simple blocking read is safe. + def _read(prompt): + return input(prompt) + + loop = asyncio.get_running_loop() + choice = (await loop.run_in_executor( + None, lambda: _read('choice [y/s/a/e/n]: '))).strip().lower() + + if choice == 's': + return PermissionResponse(action=PermissionAction.ALLOW_SESSION, + pattern=suggestion) + if choice == 'a': + edited = (await loop.run_in_executor( + None, lambda: _read(f'pattern [{suggestion}]: '))).strip() + return PermissionResponse(action=PermissionAction.ALLOW_ALWAYS, + pattern=edited or suggestion) + if choice == 'e': + raw = (await loop.run_in_executor( + None, lambda: _read('new args (JSON): '))).strip() + try: + new_args = json.loads(raw) + except json.JSONDecodeError: + self._console.print('[red]invalid JSON — denying[/]') + return PermissionResponse(action=PermissionAction.DENY) + return PermissionResponse(action=PermissionAction.MODIFY, + updated_args=new_args) + if choice == 'n': + return PermissionResponse(action=PermissionAction.DENY) + return PermissionResponse(action=PermissionAction.ALLOW_ONCE) diff --git a/ms_agent/utils/artifact_manager.py b/ms_agent/utils/artifact_manager.py index 2d73457f0..35ddc74b0 100644 --- a/ms_agent/utils/artifact_manager.py +++ b/ms_agent/utils/artifact_manager.py @@ -19,7 +19,7 @@ def __init__( max_combined_bytes: int = 256 * 1024, preview_head_chars: int = 4000, preview_tail_chars: int = 2000, - artifact_subdir: str = '.ms_agent_artifacts', + artifact_subdir: str = '.ms_agent/artifacts', ) -> None: self._root = Path(output_dir).expanduser().resolve() self.max_combined_bytes = max_combined_bytes diff --git a/ms_agent/utils/constants.py b/ms_agent/utils/constants.py index cbd37e271..d502bd173 100644 --- a/ms_agent/utils/constants.py +++ b/ms_agent/utils/constants.py @@ -5,9 +5,11 @@ # When ``output_dir`` is omitted, ``resolve_workspace_root()`` uses cwd instead. DEFAULT_OUTPUT_DIR = './output' -DEFAULT_INDEX_DIR = '.index' +# Framework-internal subdirs, collapsed under /.ms_agent/ (see +# ms_agent.project.paths) so nothing scatters at the work-dir root. +DEFAULT_INDEX_DIR = '.ms_agent/index' -DEFAULT_LOCK_DIR = '.locks' +DEFAULT_LOCK_DIR = '.ms_agent/locks' # The key of user defined tools in the agent.yaml TOOL_PLUGIN_NAME = 'plugins' @@ -18,8 +20,10 @@ # Default agent code file DEFAULT_AGENT_FILE = 'agent.py' -# Default memory folder -DEFAULT_MEMORY_DIR = '.memory' +# History cache folder, collapsed under the framework-internal .ms_agent/ dir +# (was '.memory'). LEGACY_MEMORY_DIR is still read for back-compat. +DEFAULT_MEMORY_DIR = '.ms_agent/memory' +LEGACY_MEMORY_DIR = '.memory' # DEFAULT_WORKFLOW_YAML WORKFLOW_CONFIG_FILE = 'workflow.yaml' diff --git a/ms_agent/utils/logger.py b/ms_agent/utils/logger.py index 0ac9d48a8..aefe47750 100644 --- a/ms_agent/utils/logger.py +++ b/ms_agent/utils/logger.py @@ -30,14 +30,39 @@ def warning_once(self, msg, *args, **kwargs): self.warning(msg) +def _resolve_log_file(log_file: Optional[str]) -> Optional[str]: + """File logging is opt-in. Returns a path only when explicitly requested + (``log_file`` arg or ``MS_AGENT_LOG_FILE`` / ``LOG_FILE`` env); otherwise + ``None`` (console-only) so we never scatter ``ms_agent.log`` in the CWD. + + A truthy-flag env value (``1``/``true``) routes to the global + ``~/.ms_agent/logs/ms_agent.log``; an explicit path is used as-is. + """ + if log_file: + return log_file + env = os.environ.get('MS_AGENT_LOG_FILE') or os.environ.get('LOG_FILE') + if not env: + return None + if env.strip().lower() in ('1', 'true', 'yes', 'on'): + from ms_agent.project.paths import global_logs_dir + d = global_logs_dir() + try: + d.mkdir(parents=True, exist_ok=True) + return str(d / 'ms_agent.log') + except OSError: + # Never let logging setup crash the app; fall back to console-only. + return None + return env + + def get_logger(log_file: Optional[str] = None, log_level: Optional[int] = None, file_mode: str = 'w'): """ Get logging logger Args: - log_file: Log filename, if specified, file handler will be added to - logger. If None, defaults to 'ms_agent.log' in current working directory. + log_file: Log filename. File logging is opt-in: when omitted (and no + ``MS_AGENT_LOG_FILE``/``LOG_FILE`` env), logging is console-only. log_level: Logging level. file_mode: Specifies the mode to open the file, if filename is specified (if filemode is unspecified, it defaults to 'w'). @@ -46,9 +71,7 @@ def get_logger(log_file: Optional[str] = None, log_level = os.getenv('LOG_LEVEL', 'INFO').upper() log_level = getattr(logging, log_level, logging.INFO) - # Default log file path: current working directory - if log_file is None: - log_file = os.path.join(os.getcwd(), 'ms_agent.log') + log_file = _resolve_log_file(log_file) logger_name = __name__.split('.')[0] logger = logging.getLogger(logger_name) logger.propagate = False @@ -74,9 +97,9 @@ def get_logger(log_file: Optional[str] = None, stream_handler = logging.StreamHandler() handlers = [stream_handler] - # Always add file handler since log_file is set to default if None - file_handler = logging.FileHandler(log_file, file_mode) - handlers.append(file_handler) + # File handler only when opt-in (see _resolve_log_file); console otherwise. + if log_file: + handlers.append(logging.FileHandler(log_file, file_mode)) for handler in handlers: handler.setFormatter(logger_format) @@ -121,9 +144,9 @@ def add_file_handler_if_needed(logger, log_file, file_mode, log_level): is_worker0 = True if is_worker0: - # Default log file path if not specified + # File logging is opt-in; no CWD default. if log_file is None: - log_file = os.path.join(os.getcwd(), 'ms_agent.log') + return file_handler = logging.FileHandler(log_file, file_mode) file_handler.setFormatter(logger_format) file_handler.setLevel(log_level) diff --git a/ms_agent/utils/snapshot.py b/ms_agent/utils/snapshot.py index d7362405a..d731c20ec 100644 --- a/ms_agent/utils/snapshot.py +++ b/ms_agent/utils/snapshot.py @@ -17,7 +17,6 @@ logger = get_logger() -_SNAPSHOT_DIR_NAME = '.ms_agent_snapshots' _META_FILE = 'snapshot_meta.json' @@ -41,7 +40,9 @@ def _git(args: list[str], def _snapshot_git_dir(output_dir: str) -> str: - return os.path.join(output_dir, _SNAPSHOT_DIR_NAME) + # Collapsed under /.ms_agent/ (was /.ms_agent_snapshots). + from ms_agent.project.paths import snapshots_dir + return str(snapshots_dir(output_dir)) def _configure_snapshot_repo_for_automation(work_tree: str, @@ -81,7 +82,11 @@ def _ensure_repo(output_dir: str) -> str: os.makedirs(info_dir, exist_ok=True) exclude_file = os.path.join(info_dir, 'exclude') with open(exclude_file, 'a', encoding='utf-8') as f: - f.write(f'\n{_SNAPSHOT_DIR_NAME}/\n') + # Exclude only the snapshot git repo itself (must not self-track). + # The rest of .ms_agent/ (history cache, etc.) is still captured, + # preserving the pre-move behavior where /.memory was + # part of the snapshot. + f.write('\n.ms_agent/snapshots/\n') # Always (re)apply: repos created before this fix may still inherit hooks. _configure_snapshot_repo_for_automation(output_dir, git_dir) return git_dir diff --git a/ms_agent/utils/stats.py b/ms_agent/utils/stats.py index beeccfc1f..be49661a4 100644 --- a/ms_agent/utils/stats.py +++ b/ms_agent/utils/stats.py @@ -25,12 +25,13 @@ def _get_lock(path: str) -> asyncio.Lock: def get_stats_path(config: Any, default_filename: str = 'workflow_stats.json') -> str: stats_file = getattr(config, 'stats_file', None) - output_dir = getattr(config, 'output_dir', './output') + output_dir = getattr(config, 'output_dir', None) or '.' if stats_file: if os.path.isabs(stats_file): return stats_file return os.path.join(output_dir, stats_file) - return os.path.join(output_dir, default_filename) + from ms_agent.project.paths import stats_file as _stats_path + return str(_stats_path(output_dir, default_filename)) def summarize_usage(messages: Optional[Iterable[Message]]) -> Dict[str, int]: diff --git a/ms_agent/utils/stream_writer.py b/ms_agent/utils/stream_writer.py index cdeddec77..18f74ef8f 100644 --- a/ms_agent/utils/stream_writer.py +++ b/ms_agent/utils/stream_writer.py @@ -52,7 +52,7 @@ def __init__(self, output_dir: str, call_id: str, tool_name: str) -> None: self._agent_tag: Optional[str] = None self._file = None # opened lazily in on_start - subagents_dir = os.path.join(output_dir, 'subagents') + subagents_dir = os.path.join(output_dir, '.ms_agent', 'subagents') os.makedirs(subagents_dir, exist_ok=True) safe_id = self._call_id.replace('/', '_').replace('\\', '_') self._path: str = os.path.join(subagents_dir, diff --git a/ms_agent/utils/utils.py b/ms_agent/utils/utils.py index 91d9c6d42..2410aea3f 100644 --- a/ms_agent/utils/utils.py +++ b/ms_agent/utils/utils.py @@ -241,10 +241,21 @@ def read_history(output_dir: str, task: str): """ from ms_agent.config import Config from ms_agent.llm import Message + from .constants import LEGACY_MEMORY_DIR cache_dir = os.path.join(output_dir, DEFAULT_MEMORY_DIR) - os.makedirs(cache_dir, exist_ok=True) config_file = os.path.join(cache_dir, f'{task}.yaml') message_file = os.path.join(cache_dir, f'{task}.json') + # Back-compat: fall back to the legacy /.memory cache when the + # new .ms_agent/memory location has no history yet. + if not os.path.exists(message_file) and not os.path.exists(config_file): + legacy_dir = os.path.join(output_dir, LEGACY_MEMORY_DIR) + legacy_msg = os.path.join(legacy_dir, f'{task}.json') + if os.path.exists(legacy_msg) or os.path.exists( + os.path.join(legacy_dir, f'{task}.yaml')): + cache_dir = legacy_dir + config_file = os.path.join(cache_dir, f'{task}.yaml') + message_file = os.path.join(cache_dir, f'{task}.json') + os.makedirs(cache_dir, exist_ok=True) config = None messages = None if os.path.exists(config_file): @@ -651,8 +662,11 @@ def install_package(package_name: str, package_name = f'{package_name}[{extend_module}]' if not is_package_installed(import_name): + # Suppress pip's stdout ("Requirement already satisfied ...") so it does + # not pollute interactive UIs; stderr is kept for real errors. subprocess.check_call( - [sys.executable, '-m', 'pip', 'install', package_name]) + [sys.executable, '-m', 'pip', 'install', package_name], + stdout=subprocess.DEVNULL) logger.info(f'Package {package_name} installed successfully.') else: logger.info(f'Package {import_name} is already installed.') diff --git a/requirements/framework.txt b/requirements/framework.txt index 96a410faf..354db5e74 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -14,8 +14,10 @@ omegaconf openai pandas pillow +prompt_toolkit pyyaml pytz requests +rich sentence-transformers typing_extensions diff --git a/tests/command/test_interactive_input.py b/tests/command/test_interactive_input.py index 913932325..030578cca 100644 --- a/tests/command/test_interactive_input.py +++ b/tests/command/test_interactive_input.py @@ -173,6 +173,7 @@ def _make_callback_agent(config): # main's _get_command_router() -> _register_plugin_commands() reads this; # None makes it a no-op (real agents set it in __init__). agent._plugin_runtime = None + agent._console_io = None return agent diff --git a/tests/command/test_new_cmds.py b/tests/command/test_new_cmds.py index 15458130f..d7a9bee8b 100644 --- a/tests/command/test_new_cmds.py +++ b/tests/command/test_new_cmds.py @@ -172,7 +172,8 @@ async def test_switch_model_persists_to_project_patch(self, tmp_path): assert cfg_file.read_text(encoding='utf-8') == yaml_text # The override landed in the project patch, which from_task merges back. - patch_file = tmp_path / '.ms-agent' / 'config.yaml' + # Writer now targets the new .ms_agent/ dir (resolver reads both). + patch_file = tmp_path / '.ms_agent' / 'config.yaml' assert patch_file.exists() patch_cfg = OmegaConf.load(str(patch_file)) assert patch_cfg.llm.model == 'qwen3.7-max' diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 70bd35216..2ef4f4a94 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -31,23 +31,42 @@ def test_safe_get_config(self): config, 'tools.file_system.system_for_abbreviations') is None) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_from_task_merges_project_patch(self): - # A project patch (written by /model) wins over the committed YAML, - # while the source file stays untouched. + def test_from_task_does_not_leak_config_dir_patch(self): + # A patch in the config file's own directory must NOT be merged by + # from_task: overrides are anchored to the work dir, not the config + # file's folder, so running a shared/template config never picks up + # (or scatters) overrides in that folder. with tempfile.TemporaryDirectory() as d: with open(os.path.join(d, 'agent.yaml'), 'w') as f: f.write('llm:\n service: openai\n model: base-model\n') - patch_dir = os.path.join(d, '.ms-agent') + patch_dir = os.path.join(d, '.ms_agent') os.makedirs(patch_dir) with open(os.path.join(patch_dir, 'config.yaml'), 'w') as f: f.write('llm:\n model: override-model\n') - # Isolate from pytest's argv, which Config.parse_args() scans. with patch.object(sys, 'argv', ['ms-agent']): config = Config.from_task(d) - self.assertEqual('override-model', config.llm.model) - # Untouched base key is preserved through the merge. - self.assertEqual('openai', config.llm.service) + # from_task leaves the committed model intact (no config-dir merge). + self.assertEqual('base-model', config.llm.model) + + def test_work_dir_patch_merges(self): + # The project patch is anchored to the work dir and applied by the + # agent (BaseAgent.__init__ via ConfigResolver._load_project_patch). + from omegaconf import OmegaConf + + from ms_agent.config.resolver import ConfigResolver + with tempfile.TemporaryDirectory() as work: + patch_dir = os.path.join(work, '.ms_agent') + os.makedirs(patch_dir) + with open(os.path.join(patch_dir, 'config.yaml'), 'w') as f: + f.write('llm:\n model: override-model\n') + base = OmegaConf.create( + {'llm': {'service': 'openai', 'model': 'base-model'}}) + patch_cfg = ConfigResolver()._load_project_patch(work) + self.assertIsNotNone(patch_cfg) + merged = OmegaConf.merge(base, patch_cfg) + self.assertEqual('override-model', merged.llm.model) + self.assertEqual('openai', merged.llm.service) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_from_task_without_patch(self): @@ -58,6 +77,14 @@ def test_from_task_without_patch(self): config = Config.from_task(d) self.assertEqual('base-model', config.llm.model) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_parse_args_ignores_bare_separator(self): + # A bare '--' separator must not be parsed as an empty-key override + # (without the len(key) > 2 guard this yields {'': 'v'}). + with patch.object(sys, 'argv', ['ms-agent', '--', '--', 'v']): + parsed = Config.parse_args() + self.assertNotIn('', parsed) + if __name__ == '__main__': unittest.main() diff --git a/tests/config/test_model_settings.py b/tests/config/test_model_settings.py new file mode 100644 index 000000000..953fd6b0d --- /dev/null +++ b/tests/config/test_model_settings.py @@ -0,0 +1,51 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import json + +from ms_agent.config.model_settings import ModelSettingsManager + + +def test_add_list_remove_provider(tmp_path): + m = ModelSettingsManager(global_dir=str(tmp_path)) + m.add_provider('acme', name='Acme', protocol='openai', + api_key='k', base_url='https://acme/v1', models=['a-1']) + customs = m.list_custom_providers() + assert customs['acme']['base_url'] == 'https://acme/v1' + assert 'a-1' in customs['acme']['models'] + # builtin + custom listed together + ids = {p['id'] for p in m.list_providers()} + assert 'acme' in ids and 'openai' in ids and 'deepseek' in ids + m.remove_provider('acme') + assert 'acme' not in m.list_custom_providers() + + +def test_models_and_default(tmp_path): + m = ModelSettingsManager(global_dir=str(tmp_path)) + m.add_provider('acme', protocol='openai') + m.add_model('acme', 'a-2') + assert 'a-2' in m.list_custom_providers()['acme']['models'] + m.set_default_model('a-2', provider='acme') + assert m.get_default_model() == 'acme/a-2' + m.remove_model('acme', 'a-2') + assert 'a-2' not in m.list_custom_providers()['acme']['models'] + + +def test_preserves_other_sections(tmp_path): + p = tmp_path / 'settings.json' + p.write_text(json.dumps({'theme': 'dark', 'llm': {'provider': 'x'}})) + m = ModelSettingsManager(global_dir=str(tmp_path)) + m.add_provider('acme', protocol='openai') + data = json.loads(p.read_text()) + assert data['theme'] == 'dark' and data['llm']['provider'] == 'x' + assert 'acme' in data['providers'] + + +def test_resolver_consumes_default_model(): + from ms_agent.config.resolver import ConfigResolver + cfg = ConfigResolver._settings_to_agent_config( + {'default_model': 'deepseek/deepseek-chat'}) + assert cfg.llm.service == 'deepseek' + assert cfg.llm.model == 'deepseek-chat' + # explicit llm.model wins over default_model + cfg2 = ConfigResolver._settings_to_agent_config( + {'llm': {'model': 'pinned'}, 'default_model': 'deepseek/x'}) + assert cfg2.llm.model == 'pinned' diff --git a/tests/llm/test_continuation_guard.py b/tests/llm/test_continuation_guard.py new file mode 100644 index 000000000..c65005691 --- /dev/null +++ b/tests/llm/test_continuation_guard.py @@ -0,0 +1,48 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""PR#920 review #2/#3: continuation (partial/prefix) must only be attempted +when the provider supports it (``continue_gen_mode`` set). Otherwise a truncated +response (``finish_reason='length'``) on a strict OpenAI-compatible API would +send ``partial``/``prefix`` fields + consecutive assistant messages -> 400.""" +from unittest.mock import MagicMock, patch + +from ms_agent.llm.transport.openai_compat import OpenAICompatTransport +from ms_agent.llm.utils import Message + + +def _transport(continue_gen_mode): + # Bypass __init__ so no openai client / network is created. + t = OpenAICompatTransport.__new__(OpenAICompatTransport) + t.continue_gen_mode = continue_gen_mode + t._continue_flag = 'prefix' if continue_gen_mode == 'prefix' else 'partial' + return t + + +def _length_completion(): + c = MagicMock() + c.choices[0].finish_reason = 'length' # truncated + return c + + +def test_no_continuation_when_mode_none(): + """continue_gen_mode=None (standard OpenAI/Gemini) -> never continue.""" + t = _transport(None) + msgs = [Message(role='user', content='hi')] + with patch.object(t, '_format_output_message', + return_value=Message(role='assistant', content='x')), \ + patch.object(t, '_call_llm_for_continue_gen') as cont: + t._continue_generate(msgs, _length_completion(), tools=None) + cont.assert_not_called() + + +def test_continuation_when_mode_set(): + """A provider that supports continuation still continues on 'length'.""" + t = _transport('partial') + msgs = [Message(role='user', content='hi')] + stop = MagicMock() + stop.choices[0].finish_reason = 'stop' # ends the recursion + with patch.object(t, '_format_output_message', + return_value=Message(role='assistant', content='x')), \ + patch.object( + t, '_call_llm_for_continue_gen', return_value=stop) as cont: + t._continue_generate(msgs, _length_completion(), tools=None) + cont.assert_called_once() diff --git a/tests/llm/test_provider_router_live.py b/tests/llm/test_provider_router_live.py index c82f02761..ba92e51ab 100644 --- a/tests/llm/test_provider_router_live.py +++ b/tests/llm/test_provider_router_live.py @@ -49,8 +49,8 @@ PROVIDERS = { 'modelscope': ('Qwen/Qwen3-235B-A22B-Instruct-2507', 'MODELSCOPE_API_KEY'), 'dashscope': ('qwen3.7-plus', 'DASHSCOPE_API_KEY'), - 'deepseek': ('deepseek-v4-flash', 'DEEPSEEK_API_KEY'), - 'zhipu': ('glm-4.6', 'GLM_API_KEY'), + 'deepseek': ('deepseek-chat', 'DEEPSEEK_API_KEY'), + 'zhipu': ('glm-4.5', 'GLM_API_KEY'), 'kimi': ('moonshot-v1-8k', 'KIMI_API_KEY'), 'minimax': ('MiniMax-M2', 'MINIMAX_API_KEY'), 'openrouter': ('qwen/qwen3.7-plus', 'OpenRouter_API_KEY'), @@ -100,11 +100,24 @@ def _run(self, service): pass self.assertTrue(chunk and chunk.content, f'{service}: empty stream') - # tool call + # tool call (non-stream) llm = LLM.from_config(_config(service, model)) res = llm.generate(messages=_msgs('请创建一个名为 demo 的目录。'), tools=TOOLS) self.assertTrue(res.tool_calls, f'{service}: no tool call') + # tool call (stream) — the streamed tool_call merge must produce a + # complete name + arguments, not a partial fragment. + llm = LLM.from_config(_config(service, model, stream=True)) + chunk = None + for chunk in llm.generate( + messages=_msgs('请创建一个名为 demo 的目录。'), tools=TOOLS): + pass + self.assertTrue(chunk and chunk.tool_calls, + f'{service}: no streamed tool call') + tc = chunk.tool_calls[0] + self.assertTrue(tc.get('tool_name') and tc.get('arguments'), + f'{service}: incomplete streamed tool call') + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_modelscope(self): self._run('modelscope') @@ -133,6 +146,19 @@ def test_minimax(self): def test_openrouter(self): self._run('openrouter') + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_reasoning_content(self): + """A reasoning model surfaces reasoning_content separate from content.""" + if not os.getenv('DEEPSEEK_API_KEY'): + self.skipTest('needs DEEPSEEK_API_KEY') + model = os.getenv('DEEPSEEK_REASONER_TEST_MODEL', 'deepseek-reasoner') + llm = LLM.from_config(_config('deepseek', model)) + res = llm.generate(messages=_msgs('2+2 等于几？简要思考。'), tools=None) + self.assertTrue(res.content, 'reasoner: empty content') + self.assertTrue(getattr(res, 'reasoning_content', ''), + 'reasoner: no reasoning_content') + self.assertNotIn('', res.content) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_continue_gen_accumulates(self): if not os.getenv('DASHSCOPE_API_KEY'): diff --git a/tests/permission/test_parallel_permission.py b/tests/permission/test_parallel_permission.py new file mode 100644 index 000000000..421442a26 --- /dev/null +++ b/tests/permission/test_parallel_permission.py @@ -0,0 +1,45 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Parallel tool calls (ToolManager.parallel_call_tool → asyncio.gather) must +not invoke the interactive permission handler concurrently: N prompts fighting +over one terminal deadlocks. The enforcer serializes asks.""" +import asyncio + +import pytest + +from ms_agent.permission.config import PermissionConfig +from ms_agent.permission.enforcer import PermissionEnforcer +from ms_agent.permission.handler import PermissionAction, PermissionResponse +from ms_agent.permission.memory import PermissionMemory + + +class _ConcurrencyProbe: + def __init__(self): + self.in_flight = 0 + self.max_in_flight = 0 + self.calls = 0 + + async def ask(self, tool_name, tool_args, context, suggestions=None): + self.in_flight += 1 + self.calls += 1 + self.max_in_flight = max(self.max_in_flight, self.in_flight) + await asyncio.sleep(0.02) # simulate a human deciding + self.in_flight -= 1 + return PermissionResponse(action=PermissionAction.ALLOW_ONCE) + + +@pytest.mark.asyncio +async def test_parallel_asks_are_serialized(tmp_path): + cfg = PermissionConfig.from_dict({'mode': 'restricted'}) # empty wl -> ask + handler = _ConcurrencyProbe() + enf = PermissionEnforcer( + config=cfg, handler=handler, + memory=PermissionMemory(project_path=str(tmp_path))) + + # 5 concurrent checks, like 5 parallel tool calls in one round. + results = await asyncio.gather( + *[enf.check('some_tool', {'i': i}) for i in range(5)]) + + assert all(r.action == 'allow' for r in results) + assert handler.calls == 5 + # The decisive assertion: asks never overlapped (was 5 before the fix). + assert handler.max_in_flight == 1 diff --git a/tests/permission/test_safety_ask_force.py b/tests/permission/test_safety_ask_force.py new file mode 100644 index 000000000..188c58144 --- /dev/null +++ b/tests/permission/test_safety_ask_force.py @@ -0,0 +1,52 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""REVIEW P1-2: a SafetyGuard `ask` (passed as force_decision) must reach the +handler even when a whitelist/memory entry would otherwise allow the tool.""" +import pytest + +from ms_agent.permission.config import PermissionConfig +from ms_agent.permission.enforcer import PermissionDecision, PermissionEnforcer +from ms_agent.permission.memory import PermissionMemory + + +class _RecordingHandler: + def __init__(self): + self.asked = False + + async def ask(self, tool_name, tool_args, context, suggestions=None): + self.asked = True + from ms_agent.permission.handler import (PermissionAction, + PermissionResponse) + return PermissionResponse(action=PermissionAction.DENY) + + +@pytest.mark.asyncio +async def test_force_ask_not_bypassed_by_whitelist(tmp_path): + cfg = PermissionConfig.from_dict({ + 'mode': 'interactive', + 'whitelist': ['code_executor---shell_executor:*'], + }) + handler = _RecordingHandler() + enf = PermissionEnforcer( + config=cfg, handler=handler, + memory=PermissionMemory(project_path=str(tmp_path))) + force = PermissionDecision(action='ask', reason='safety') + out = await enf.check( + 'code_executor---shell_executor', {'command': 'ls'}, + force_decision=force) + # whitelist would allow, but the safety force_decision routed to the handler + assert handler.asked is True + assert out.action == 'deny' + + +@pytest.mark.asyncio +async def test_whitelist_allows_without_force(tmp_path): + cfg = PermissionConfig.from_dict({ + 'mode': 'interactive', + 'whitelist': ['code_executor---shell_executor:*'], + }) + handler = _RecordingHandler() + enf = PermissionEnforcer( + config=cfg, handler=handler, + memory=PermissionMemory(project_path=str(tmp_path))) + out = await enf.check('code_executor---shell_executor', {'command': 'ls'}) + assert handler.asked is False and out.action == 'allow' diff --git a/tests/project/conftest.py b/tests/project/conftest.py new file mode 100644 index 000000000..2997ab7ba --- /dev/null +++ b/tests/project/conftest.py @@ -0,0 +1,14 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Isolate the global ms-agent home for project tests. + +SessionManager (and the runtime SessionLog) place sessions under the global +``~/.ms_agent/projects/<...>`` root (paths.py). Redirect ``MS_AGENT_HOME`` to a +per-test temp dir so tests never read or pollute the real user home. +""" +import pytest + + +@pytest.fixture(autouse=True) +def _isolate_ms_agent_home(tmp_path, monkeypatch): + monkeypatch.setenv('MS_AGENT_HOME', str(tmp_path / '_ms_home')) + yield diff --git a/tests/project/test_manager.py b/tests/project/test_manager.py index 6a59f490e..b3c3ee2da 100644 --- a/tests/project/test_manager.py +++ b/tests/project/test_manager.py @@ -81,8 +81,7 @@ def test_workspace_dir_created(self, pm): def test_sessions_dir_created(self, pm): project = pm.create(name='WithSessions') from pathlib import Path - meta_dir = pm._projects_root / project.id / '.ms-agent' - assert (meta_dir / 'sessions').is_dir() + assert (pm._projects_root / project.id / 'sessions').is_dir() def test_project_is_frozen(self, pm): project = pm.create(name='Frozen') diff --git a/tests/project/test_paths.py b/tests/project/test_paths.py new file mode 100644 index 000000000..af0f06423 --- /dev/null +++ b/tests/project/test_paths.py @@ -0,0 +1,47 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Every framework-internal path resolves under /.ms_agent/ (or the +global ~/.ms_agent), so a work dir never accumulates scattered files.""" +from pathlib import Path + +from ms_agent.project import paths + + +def test_work_local_helpers_under_ms_agent(tmp_path): + w = str(tmp_path) + internal = str(paths.local_internal_dir(w)) + for fn in (paths.memory_dir, paths.snapshots_dir, paths.artifacts_dir, + paths.index_dir, paths.locks_dir, paths.tmp_dir, + paths.subagents_dir, paths.web_search_dir): + assert str(fn(w)).startswith(internal), fn.__name__ + assert str(paths.search_index_dir(w, 'rag')).startswith(internal) + assert str(paths.stats_file(w)).startswith(internal) + + +def test_global_helpers_under_ms_agent_home(tmp_path, monkeypatch): + monkeypatch.setenv('MS_AGENT_HOME', str(tmp_path / 'home')) + home = str((tmp_path / 'home').resolve()) + assert str(paths.global_projects_root()).startswith(home) + assert str(paths.global_logs_dir()).startswith(home) + assert str(paths.global_project_dir('/some/work')).startswith(home) + + +def test_project_internal_file_prefers_new_then_legacy(tmp_path): + proj = tmp_path / 'proj' + # nothing exists -> returns the new .ms_agent path + p = paths.project_internal_file(proj, 'mcp.json') + assert p.parent.name == '.ms_agent' + # only legacy exists -> returns legacy + legacy = proj / '.ms-agent' + legacy.mkdir(parents=True) + (legacy / 'mcp.json').write_text('{}') + assert paths.project_internal_file(proj, 'mcp.json').parent.name == '.ms-agent' + # new exists -> prefers new + new = proj / '.ms_agent' + new.mkdir(parents=True) + (new / 'mcp.json').write_text('{}') + assert paths.project_internal_file(proj, 'mcp.json').parent.name == '.ms_agent' + + +def test_project_key_stable_and_readable(): + assert paths.project_key('/Users/x/proj') == paths.project_key('/Users/x/proj') + assert 'proj' in paths.project_key('/Users/x/proj') diff --git a/tests/project/test_workspace_upload.py b/tests/project/test_workspace_upload.py new file mode 100644 index 000000000..0d5d4e29d --- /dev/null +++ b/tests/project/test_workspace_upload.py @@ -0,0 +1,58 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import io +import zipfile + +import pytest + +from ms_agent.project.workspace import Workspace + + +def test_write_read_bytes(tmp_path): + ws = Workspace(str(tmp_path / 'ws')) + ws.write_bytes('sub/data.bin', b'\x00\x01\x02') + assert ws.read_bytes('sub/data.bin') == b'\x00\x01\x02' + + +def test_save_upload_stream_and_bytes(tmp_path): + ws = Workspace(str(tmp_path / 'ws')) + rel = ws.save_upload('report.pdf', io.BytesIO(b'PDFDATA'), subdir='uploads') + assert ws.read_bytes(rel) == b'PDFDATA' + # basename-only: a filename with dir components is stripped + rel2 = ws.save_upload('../../evil.txt', b'x') + assert 'evil.txt' in rel2 and '..' not in rel2 + + +def test_save_upload_traversal_blocked(tmp_path): + ws = Workspace(str(tmp_path / 'ws')) + with pytest.raises(PermissionError): + ws.save_upload('ok.txt', b'x', subdir='../../etc') + + +def test_zip_download(tmp_path): + ws = Workspace(str(tmp_path / 'ws')) + ws.write_file('a.txt', 'A') + ws.write_file('d/b.txt', 'B') + data = ws.zip_download('.') + with zipfile.ZipFile(io.BytesIO(data)) as zf: + names = set(zf.namelist()) + assert any(n.endswith('a.txt') for n in names) + assert any(n.endswith('b.txt') for n in names) + + +def test_zip_download_skips_external_symlink(tmp_path): + # A symlink planted in the workspace pointing outside must NOT be followed + # and packaged (arbitrary-file-read / info disclosure via zip_download). + secret = tmp_path / 'secret.txt' + secret.write_text('TOP-SECRET') + ws_dir = tmp_path / 'ws' + ws = Workspace(str(ws_dir)) + ws.write_file('safe.txt', 'ok') + (ws_dir / 'leak.txt').symlink_to(secret) # escapes the workspace + + data = ws.zip_download('.') + with zipfile.ZipFile(io.BytesIO(data)) as zf: + names = zf.namelist() + blob = b''.join(zf.read(n) for n in names) + assert any(n.endswith('safe.txt') for n in names) + assert not any('leak' in n for n in names) # symlink entry skipped + assert b'TOP-SECRET' not in blob # external content not leaked diff --git a/tests/utils/test_logger.py b/tests/utils/test_logger.py new file mode 100644 index 000000000..b2eaa96da --- /dev/null +++ b/tests/utils/test_logger.py @@ -0,0 +1,26 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""_resolve_log_file must never crash logging setup (and thus the app) on a +filesystem error when opt-in file logging is enabled.""" +from unittest.mock import MagicMock, patch + +from ms_agent.utils.logger import _resolve_log_file + + +def test_default_console_only(monkeypatch): + monkeypatch.delenv('MS_AGENT_LOG_FILE', raising=False) + monkeypatch.delenv('LOG_FILE', raising=False) + assert _resolve_log_file(None) is None + + +def test_explicit_arg_wins(): + assert _resolve_log_file('/tmp/custom.log') == '/tmp/custom.log' + + +def test_mkdir_failure_falls_back_to_console(monkeypatch): + monkeypatch.setenv('MS_AGENT_LOG_FILE', 'true') + fake_dir = MagicMock() + fake_dir.mkdir.side_effect = OSError('read-only filesystem') + with patch( + 'ms_agent.project.paths.global_logs_dir', return_value=fake_dir): + # Must not raise; degrades to console-only (None). + assert _resolve_log_file(None) is None