From d9017e87fd3d724e58f1d088d61123e984a2300d Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sun, 21 Jun 2026 15:06:43 -0700 Subject: [PATCH 1/2] Add per-technique converters to scenario strategies and --list-converters Allow attaching registered converter instances to attack techniques via the --strategies :converter. syntax (repeatable, also works on aggregate strategies). Converters are appended request-side on top of any baked-in converters. Adds a --list-converters discovery command to pyrit_scan and pyrit_shell. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../instructions/scenarios.instructions.md | 2 + doc/scanner/1_pyrit_scan.ipynb | 34 ++++- doc/scanner/1_pyrit_scan.py | 25 ++++ doc/scanner/2_pyrit_shell.md | 16 +++ .../backend/services/scenario_run_service.py | 121 ++++++++++++++-- pyrit/cli/_banner.py | 1 + pyrit/cli/_cli_args.py | 5 +- pyrit/cli/_output.py | 39 ++++++ pyrit/cli/api_client.py | 13 ++ pyrit/cli/pyrit_scan.py | 13 +- pyrit/cli/pyrit_shell.py | 16 +++ .../scenario/core/attack_technique_factory.py | 20 ++- pyrit/scenario/core/scenario.py | 19 +++ .../unit/backend/test_scenario_run_service.py | 129 ++++++++++++++++++ tests/unit/cli/test_api_client.py | 6 + tests/unit/cli/test_output.py | 34 +++++ tests/unit/cli/test_pyrit_scan.py | 17 +++ tests/unit/cli/test_pyrit_shell.py | 18 +++ .../unit/scenario/airt/test_rapid_response.py | 37 +++++ .../core/test_attack_technique_factory.py | 63 +++++++++ 20 files changed, 612 insertions(+), 16 deletions(-) diff --git a/.github/instructions/scenarios.instructions.md b/.github/instructions/scenarios.instructions.md index 40fb4150b8..1000fbc846 100644 --- a/.github/instructions/scenarios.instructions.md +++ b/.github/instructions/scenarios.instructions.md @@ -161,6 +161,8 @@ The default implementation: `AttackTechniqueRegistry` singleton) 2. Iterates over every (technique × dataset) pair from `self._dataset_config` 3. Calls `factory.create()` with `objective_target` and conditional scorer override + (also forwards any per-technique converters from `self._strategy_converters`, populated + from the CLI `--strategies :converter.` modifier, as `extra_request_converters`) 4. Uses `self._build_display_group()` for user-facing grouping 5. Builds `AtomicAttack` with unique `atomic_attack_name` = `"{technique}_{dataset}"` diff --git a/doc/scanner/1_pyrit_scan.ipynb b/doc/scanner/1_pyrit_scan.ipynb index 1ff21c6d5b..1d5b957798 100644 --- a/doc/scanner/1_pyrit_scan.ipynb +++ b/doc/scanner/1_pyrit_scan.ipynb @@ -163,6 +163,36 @@ "cell_type": "markdown", "id": "9", "metadata": {}, + "source": [ + "#### Attaching Converters to a Technique\n", + "\n", + "Strategies (techniques) can have a registered converter instance appended to them with the\n", + "`:converter.` syntax. The converter is added to the request side of every attack\n", + "the technique produces, on top of any converters the technique already bakes in. This also works on\n", + "aggregate strategies (the converter is applied to every technique the aggregate expands to).\n", + "\n", + "First discover the registered converter instances with `--list-converters` (converters are\n", + "registered by initializers, so pass the same `--initializers`/`--initialization-scripts` you use to run):\n", + "\n", + "```shell\n", + "pyrit_scan --list-converters --initializers my_converters\n", + "```\n", + "\n", + "Then reference a converter by name in `--strategies`:\n", + "\n", + "```shell\n", + "# Add the registered \"translation_spanish\" converter to role_play only\n", + "pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --strategies role_play:converter.translation_spanish\n", + "\n", + "# Chain multiple converters (applied in order) and combine with plain strategies\n", + "pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --strategies role_play:converter.translation_spanish:converter.base64 many_shot\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, "source": [ "#### Using Custom Scenarios\n", "\n", @@ -172,7 +202,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "11", "metadata": { "lines_to_next_cell": 2 }, @@ -225,7 +255,7 @@ }, { "cell_type": "markdown", - "id": "11", + "id": "12", "metadata": {}, "source": [ "Then discover and run it:\n", diff --git a/doc/scanner/1_pyrit_scan.py b/doc/scanner/1_pyrit_scan.py index c1b452a071..f659fcc3f2 100644 --- a/doc/scanner/1_pyrit_scan.py +++ b/doc/scanner/1_pyrit_scan.py @@ -114,6 +114,31 @@ # pyrit_scan garak.encoding --initialization-scripts ./my_custom_config.py # ``` +# %% [markdown] +# #### Attaching Converters to a Technique +# +# Strategies (techniques) can have a registered converter instance appended to them with the +# `:converter.` syntax. The converter is added to the request side of every attack +# the technique produces, on top of any converters the technique already bakes in. This also works on +# aggregate strategies (the converter is applied to every technique the aggregate expands to). +# +# First discover the registered converter instances with `--list-converters` (converters are +# registered by initializers, so pass the same `--initializers`/`--initialization-scripts` you use to run): +# +# ```shell +# pyrit_scan --list-converters --initializers my_converters +# ``` +# +# Then reference a converter by name in `--strategies`: +# +# ```shell +# # Add the registered "translation_spanish" converter to role_play only +# pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --strategies role_play:converter.translation_spanish +# +# # Chain multiple converters (applied in order) and combine with plain strategies +# pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --strategies role_play:converter.translation_spanish:converter.base64 many_shot +# ``` + # %% [markdown] # #### Using Custom Scenarios # diff --git a/doc/scanner/2_pyrit_shell.md b/doc/scanner/2_pyrit_shell.md index 9f9731b601..d5d68cb4c6 100644 --- a/doc/scanner/2_pyrit_shell.md +++ b/doc/scanner/2_pyrit_shell.md @@ -40,6 +40,7 @@ Once starting the shell, you will see the list of commands you have access to. S | `list-scenarios` | List all available scenarios | | `list-initializers` | List all available initializers | | `list-targets` | List all available targets from the registry | +| `list-converters` | List all registered converter instances | | `run [options]` | Run a scenario with optional parameters | | `scenario-history` | List all previous scenario runs in this session | | `print-scenario [N]` | Print detailed results for scenario run(s) | @@ -65,6 +66,21 @@ pyrit> run garak.encoding --target my_target --initializers target load_default_ pyrit> run foundry.red_team_agent --target my_target --initializers target load_default_datasets -s jailbreak crescendo ``` +### Attaching Converters to a Technique + +Append a registered converter instance to a single technique (or an aggregate strategy) with the +`:converter.` syntax. The converter is added to the request side of every attack +the technique produces, on top of any converters the technique already bakes in. Use +`list-converters` to discover the registered converter names: + +```bash +# Add the registered "translation_spanish" converter to role_play only +pyrit> run airt.rapid_response --target my_target --initializers target load_default_datasets -s role_play:converter.translation_spanish + +# Chain multiple converters (applied in order) and combine with plain strategies +pyrit> run airt.rapid_response --target my_target --initializers target load_default_datasets -s role_play:converter.translation_spanish:converter.base64 many_shot +``` + ### With Runtime Parameters ```bash diff --git a/pyrit/backend/services/scenario_run_service.py b/pyrit/backend/services/scenario_run_service.py index b4997828a0..7cd44eb1ae 100644 --- a/pyrit/backend/services/scenario_run_service.py +++ b/pyrit/backend/services/scenario_run_service.py @@ -22,17 +22,25 @@ ) from pyrit.memory import CentralMemory from pyrit.models import AttackOutcome, ScenarioResult -from pyrit.registry import InitializerRegistry, ScenarioRegistry, TargetRegistry +from pyrit.registry import ( + ConverterRegistry, + InitializerRegistry, + ScenarioRegistry, + TargetRegistry, +) from pyrit.scenario import Scenario from pyrit.scenario.core import DatasetConfiguration if TYPE_CHECKING: + from pyrit.prompt_converter import PromptConverter from pyrit.prompt_target import PromptTarget logger = logging.getLogger(__name__) _DEFAULT_MAX_CONCURRENT_RUNS = 3 +_CONVERTER_MODIFIER_PREFIX = "converter." + @dataclass class _ActiveTask: @@ -318,17 +326,14 @@ def _build_init_kwargs( if request.strategies: strategy_class = introspection_instance._strategy_class - strategy_enums = [] - for name in request.strategies: - try: - strategy_enums.append(strategy_class(name)) - except ValueError: - available_strategies = [s.value for s in strategy_class] - raise ValueError( - f"Strategy '{name}' not found for scenario '{request.scenario_name}'. " - f"Available: {', '.join(available_strategies)}" - ) from None + strategy_enums, strategy_converters = self._resolve_strategies_and_converters( + tokens=request.strategies, + strategy_class=strategy_class, + scenario_name=request.scenario_name, + ) init_kwargs["scenario_strategies"] = strategy_enums + if strategy_converters: + init_kwargs["strategy_converters"] = strategy_converters if request.dataset_names or request.max_dataset_size is not None: default_config = introspection_instance._default_dataset_config @@ -369,6 +374,100 @@ def _build_init_kwargs( return init_kwargs + def _resolve_strategies_and_converters( + self, + *, + tokens: list[str], + strategy_class: type[Any], + scenario_name: str, + ) -> tuple[list[Any], dict[str, list["PromptConverter"]]]: + """ + Resolve ``--strategies`` tokens into strategy enums and per-technique converters. + + Each token has the form ``[:converter.[:converter....]]``. + The base ```` is resolved to a ``ScenarioStrategy`` enum member (which may + be an aggregate). Each ``converter.`` modifier is resolved to a registered + converter instance and appended (in token order) to every concrete technique that the + base strategy expands to. + + Args: + tokens: The raw strategy tokens from the request. + strategy_class: The scenario's ``ScenarioStrategy`` subclass. + scenario_name: The scenario name, used for error messages. + + Returns: + A tuple of (strategy enums to pass as ``scenario_strategies``, mapping from concrete + technique name to the list of converters to append for that technique). + + Raises: + ValueError: If a base strategy name is unknown, a modifier is malformed, or a + converter name is not registered. + """ + strategy_enums: list[Any] = [] + strategy_converters: dict[str, list[PromptConverter]] = {} + + for token in tokens: + base_name, _, remainder = token.partition(":") + modifiers = [m for m in remainder.split(":") if m] if remainder else [] + + try: + strategy_enum = strategy_class(base_name) + except ValueError: + available_strategies = [s.value for s in strategy_class] + raise ValueError( + f"Strategy '{base_name}' not found for scenario '{scenario_name}'. " + f"Available: {', '.join(available_strategies)}" + ) from None + strategy_enums.append(strategy_enum) + + converters = self._resolve_converter_modifiers(modifiers=modifiers, token=token) + if not converters: + continue + + for concrete in strategy_class.expand({strategy_enum}): + strategy_converters.setdefault(concrete.value, []).extend(converters) + + return strategy_enums, strategy_converters + + def _resolve_converter_modifiers(self, *, modifiers: list[str], token: str) -> list["PromptConverter"]: + """ + Resolve the converter modifiers of a single strategy token to converter instances. + + Args: + modifiers: The modifier segments of the token (everything after the base strategy). + token: The full original token, used for error messages. + + Returns: + The resolved converter instances in token order. + + Raises: + ValueError: If a modifier does not use the ``converter.`` prefix or names a + converter that is not registered. + """ + if not modifiers: + return [] + + instances = ConverterRegistry.get_registry_singleton().instances + converters: list[PromptConverter] = [] + for modifier in modifiers: + if not modifier.startswith(_CONVERTER_MODIFIER_PREFIX): + raise ValueError( + f"Unknown strategy modifier '{modifier}' in '{token}'. " + f"Supported modifiers must use the '{_CONVERTER_MODIFIER_PREFIX}' prefix " + f"(e.g. '{_CONVERTER_MODIFIER_PREFIX}translation_spanish')." + ) + converter_name = modifier[len(_CONVERTER_MODIFIER_PREFIX) :] + converter = instances.get(converter_name) + if converter is None: + available = instances.get_names() + available_text = ", ".join(available) if available else "(none registered)" + raise ValueError( + f"Converter '{converter_name}' in '{token}' is not a registered converter " + f"instance. Available converters: {available_text}" + ) + converters.append(converter) + return converters + async def _initialize_scenario_async(self, *, request: RunScenarioRequest, init_kwargs: dict[str, Any]) -> Scenario: """ Instantiate the scenario and call initialize_async. diff --git a/pyrit/cli/_banner.py b/pyrit/cli/_banner.py index 21e1f1c5bd..74ac94a02e 100644 --- a/pyrit/cli/_banner.py +++ b/pyrit/cli/_banner.py @@ -255,6 +255,7 @@ def add(line: str, role: ColorRole, segments: list[tuple[int, int, ColorRole]] | " • list-scenarios - See all available scenarios", " • list-initializers - See all available initializers", " • list-targets [opts] - See all available targets in the registry", + " • list-converters - See all registered converter instances", " • run [opts] - Execute a security scenario", " • scenario-history - View your session history", " • print-scenario [N] - Display detailed results", diff --git a/pyrit/cli/_cli_args.py b/pyrit/cli/_cli_args.py index eddad0f0f3..13b4eb6e7a 100644 --- a/pyrit/cli/_cli_args.py +++ b/pyrit/cli/_cli_args.py @@ -262,7 +262,10 @@ def parse_memory_labels(json_string: str) -> dict[str, str]: "initialization_scripts": "Paths to custom Python initialization scripts to run before the scenario", "env_files": "Paths to environment files to load in order (e.g., .env.production .env.local). Later files " "override earlier ones.", - "scenario_strategies": "List of strategy names to run (e.g., base64 rot13)", + "scenario_strategies": "List of strategy names to run (e.g., base64 rot13). Append one or more " + "registered converters to a technique with ':converter.' (repeatable), e.g. " + "role_play:converter.translation_spanish:converter.leetspeak. The converter is appended on top of " + "the technique's built-in converters. Use --list-converters to see registered converter names", "max_concurrency": "Maximum number of concurrent attack executions (must be >= 1)", "max_retries": "Maximum number of automatic retries on exception (must be >= 0)", "memory_labels": 'Additional labels as JSON string (e.g., \'{"experiment": "test1"}\')', diff --git a/pyrit/cli/_output.py b/pyrit/cli/_output.py index 2daa939887..ded3288b48 100644 --- a/pyrit/cli/_output.py +++ b/pyrit/cli/_output.py @@ -195,6 +195,45 @@ def print_target_list(*, items: list[dict[str, Any]]) -> None: print(f"\nTotal targets: {len(items)}") +# --------------------------------------------------------------------------- +# Converter listing +# --------------------------------------------------------------------------- + + +def print_converter_list(*, items: list[dict[str, Any]]) -> None: + """ + Print a formatted list of registered converter instances. + + Args: + items: List of converter dicts from ``GET /api/converters``. + """ + if not items: + print("\nNo converters found in registry.") + print( + "\nConverters are registered by initializers. Include an initializer that " + "registers converters to attach them to scenario techniques, for example:\n" + " --strategies role_play:converter.translation_spanish\n" + ) + return + + print("\nRegistered Converters:") + print("=" * 80) + for conv in items: + name = conv.get("converter_id", "unknown") + _header(name) + print(f" Class: {conv.get('converter_type', '')}") + display_name = conv.get("display_name") or "" + if display_name: + print(f" Name: {display_name}") + sub_ids = conv.get("sub_converter_ids") or [] + if sub_ids: + print(f" Sub-converters: {', '.join(sub_ids)}") + print("\n" + "=" * 80) + print(f"\nTotal converters: {len(items)}") + print("\nAttach a converter to a scenario technique with, for example:") + print(" --strategies role_play:converter.\n") + + # --------------------------------------------------------------------------- # Scenario run progress & summary # --------------------------------------------------------------------------- diff --git a/pyrit/cli/api_client.py b/pyrit/cli/api_client.py index 937dfad9a4..bdd8c58e3c 100644 --- a/pyrit/cli/api_client.py +++ b/pyrit/cli/api_client.py @@ -171,6 +171,19 @@ async def list_targets_async(self, *, limit: int = 200) -> dict[str, Any]: """ return await self._get_json_async(path="/api/targets", params={"limit": limit}) + # ------------------------------------------------------------------ + # Converters + # ------------------------------------------------------------------ + + async def list_converters_async(self) -> dict[str, Any]: + """ + List all registered converter instances. + + Returns: + dict: ``ConverterInstanceListResponse`` payload. + """ + return await self._get_json_async(path="/api/converters") + # ------------------------------------------------------------------ # Scenario runs # ------------------------------------------------------------------ diff --git a/pyrit/cli/pyrit_scan.py b/pyrit/cli/pyrit_scan.py index 1e4467f929..629506b4dc 100644 --- a/pyrit/cli/pyrit_scan.py +++ b/pyrit/cli/pyrit_scan.py @@ -77,10 +77,11 @@ def _print_cli_exception(*, exc: BaseException) -> None: # Start the backend server pyrit_scan --start-server - # List scenarios, initializers, or targets + # List scenarios, initializers, targets, or converters pyrit_scan --list-scenarios pyrit_scan --list-initializers pyrit_scan --list-targets + pyrit_scan --list-converters # Run single-turn cyber attacks against a target pyrit_scan airt.cyber --target openai_chat --strategies single_turn @@ -179,6 +180,11 @@ def _build_base_parser(*, add_help: bool = True) -> ArgumentParser: action="store_true", help="List all available targets and exit", ) + discovery_group.add_argument( + "--list-converters", + action="store_true", + help="List all registered converter instances and exit", + ) discovery_group.add_argument( "--add-initializer", type=str, @@ -418,6 +424,7 @@ def _is_command_specified(*, parsed_args: Namespace) -> bool: parsed_args.list_scenarios or parsed_args.list_initializers or parsed_args.list_targets + or parsed_args.list_converters or parsed_args.add_initializer or parsed_args.scenario_name ) @@ -481,6 +488,10 @@ async def _handle_list_commands_async(*, client: Any, parsed_args: Namespace) -> resp = await client.list_targets_async() _output.print_target_list(items=resp.get("items", [])) return 0 + if parsed_args.list_converters: + resp = await client.list_converters_async() + _output.print_converter_list(items=resp.get("items", [])) + return 0 return None diff --git a/pyrit/cli/pyrit_shell.py b/pyrit/cli/pyrit_shell.py index 1a0760eb7c..d36860c880 100644 --- a/pyrit/cli/pyrit_shell.py +++ b/pyrit/cli/pyrit_shell.py @@ -36,6 +36,7 @@ class PyRITShell(cmd.Cmd): list-scenarios - List all available scenarios list-initializers - List all available initializers list-targets - List all available targets + list-converters - List all registered converter instances run [opts] - Run a scenario with optional parameters scenario-history [N] - List the last N (default 10) scenario runs print-scenario [id] - Print detailed results for a scenario run @@ -234,6 +235,21 @@ def do_list_targets(self, arg: str) -> None: except Exception as e: print(f"Error listing targets: {e}") + def do_list_converters(self, arg: str) -> None: + """List all registered converter instances.""" + if arg.strip(): + print(f"Error: list-converters does not accept arguments, got: {arg.strip()}") + return + if not self._ensure_client(): + return + from pyrit.cli import _output + + try: + resp = self._run_async(self._api_client.list_converters_async()) + _output.print_converter_list(items=resp.get("items", [])) + except Exception as e: + print(f"Error listing converters: {e}") + def do_add_initializer(self, arg: str) -> None: """ Register an initializer from a Python script file. diff --git a/pyrit/scenario/core/attack_technique_factory.py b/pyrit/scenario/core/attack_technique_factory.py index cd9da5a3c8..5b64eb8a20 100644 --- a/pyrit/scenario/core/attack_technique_factory.py +++ b/pyrit/scenario/core/attack_technique_factory.py @@ -31,6 +31,7 @@ from pyrit.executor.attack import PromptSendingAttack from pyrit.executor.attack.core.attack_config import ( AttackAdversarialConfig, + AttackConverterConfig, AttackScoringConfig, ) from pyrit.models import ( @@ -47,7 +48,7 @@ if TYPE_CHECKING: from pyrit.executor.attack import AttackStrategy - from pyrit.executor.attack.core.attack_config import AttackConverterConfig + from pyrit.prompt_normalizer import PromptConverterConfiguration from pyrit.prompt_target import PromptTarget logger = logging.getLogger(__name__) @@ -406,6 +407,7 @@ def create( adversarial_seed_prompt: SeedPrompt | str | None = None, attack_adversarial_config_override: AttackAdversarialConfig | None = None, attack_converter_config_override: AttackConverterConfig | None = None, + extra_request_converters: list[PromptConverterConfiguration] | None = None, ) -> AttackTechnique: """ Create a fresh AttackTechnique bound to the given target. @@ -451,6 +453,13 @@ def create( attack_converter_config_override: When non-None, replaces any converter config baked into the factory. Only forwarded if the attack class constructor accepts ``attack_converter_config``. + extra_request_converters: Optional request converters to append on + top of the technique's existing request converters (whether baked + into the factory or supplied via + ``attack_converter_config_override``). Unlike + ``attack_converter_config_override`` these are additive and never + replace the existing converters. Only forwarded if the attack + class constructor accepts ``attack_converter_config``. Returns: A fresh AttackTechnique with a newly-constructed attack strategy. @@ -515,6 +524,15 @@ def create( if attack_converter_config_override is not None and "attack_converter_config" in accepted_params: kwargs["attack_converter_config"] = attack_converter_config_override + if extra_request_converters and "attack_converter_config" in accepted_params: + existing = kwargs.get("attack_converter_config") + base_request = list(existing.request_converters) if existing else [] + base_response = list(existing.response_converters) if existing else [] + kwargs["attack_converter_config"] = AttackConverterConfig( + request_converters=base_request + list(extra_request_converters), + response_converters=base_response, + ) + attack = self._attack_class(**kwargs) return AttackTechnique(attack=attack, seed_technique=self._seed_technique) diff --git a/pyrit/scenario/core/scenario.py b/pyrit/scenario/core/scenario.py index dce77c3bbd..4d46068953 100644 --- a/pyrit/scenario/core/scenario.py +++ b/pyrit/scenario/core/scenario.py @@ -65,6 +65,7 @@ if TYPE_CHECKING: from pyrit.models import ComponentIdentifier + from pyrit.prompt_converter import PromptConverter from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory logger = logging.getLogger(__name__) @@ -263,6 +264,9 @@ def __init__( # Store prepared strategies for use in _get_atomic_attacks_async self._scenario_strategies: list[ScenarioStrategy] = [] + # Maps concrete technique name → extra request converters to append for that technique. + self._strategy_converters: dict[str, list[PromptConverter]] = {} + # Maps atomic_attack_name → display_group for user-facing aggregation self._display_group_map: dict[str, str] = {} @@ -582,6 +586,7 @@ async def initialize_async( *, objective_target: PromptTarget = REQUIRED_VALUE, # type: ignore[ty:invalid-parameter-default] scenario_strategies: Sequence[ScenarioStrategy] | None = None, + strategy_converters: dict[str, list["PromptConverter"]] | None = None, dataset_config: DatasetConfiguration | None = None, max_concurrency: int = 4, max_retries: int = 0, @@ -604,6 +609,11 @@ async def initialize_async( scenario_strategies (Sequence[ScenarioStrategy] | None): The strategies to execute. Can be a list of ScenarioStrategy enum members. If None, uses the default aggregate from the scenario's configuration. + strategy_converters (dict[str, list[PromptConverter]] | None): Optional mapping from + concrete technique name (``ScenarioStrategy.value``) to a list of request converters + to append on top of that technique's built-in converters. Techniques not present in + the mapping are left unchanged. Aggregate strategy names must already be expanded to + concrete technique names by the caller. dataset_config (DatasetConfiguration | None): Configuration for the dataset source. Use this to specify dataset names or maximum dataset size from the CLI. If not provided, scenarios use their constructor-supplied default_dataset_config. @@ -674,6 +684,7 @@ async def initialize_async( # Prepare scenario strategies using the stored configuration self._scenario_strategies = self._prepare_strategies(scenario_strategies) + self._strategy_converters = strategy_converters or {} # Materialize declared defaults for programmatic callers that skip the # explicit set_params_from_args step. Frontend-driven flows already @@ -1039,6 +1050,7 @@ async def _get_atomic_attacks_async(self) -> list[AtomicAttack]: ) from pyrit.executor.attack import AttackScoringConfig + from pyrit.prompt_normalizer import PromptConverterConfiguration selected_techniques = {s.value for s in self._scenario_strategies} @@ -1075,9 +1087,16 @@ async def _get_atomic_attacks_async(self) -> list[AtomicAttack]: else: compatible_groups = list(seed_groups) + extra_converters = self._strategy_converters.get(technique_name) + extra_request_converters = ( + PromptConverterConfiguration.from_converters(converters=extra_converters) + if extra_converters + else None + ) attack_technique = factory.create( objective_target=self._objective_target, attack_scoring_config=scoring_config, + extra_request_converters=extra_request_converters, ) display_group = self._build_display_group( technique_name=technique_name, diff --git a/tests/unit/backend/test_scenario_run_service.py b/tests/unit/backend/test_scenario_run_service.py index 15116a0ac9..de69a78d35 100644 --- a/tests/unit/backend/test_scenario_run_service.py +++ b/tests/unit/backend/test_scenario_run_service.py @@ -21,7 +21,31 @@ ScenarioRunService, ) from pyrit.models import AttackOutcome +from pyrit.prompt_converter import PromptConverter from pyrit.scenario.core import DatasetConfiguration +from pyrit.scenario.core.scenario_strategy import ScenarioStrategy + + +class _StubStrategy(ScenarioStrategy): + """Minimal concrete ScenarioStrategy used to exercise converter-token parsing.""" + + ALL = ("all", {"all"}) + EASY = ("easy", {"easy"}) + ROLE_PLAY = ("role_play", {"easy"}) + SINGLE_TURN = ("single_turn", {"easy"}) + + @classmethod + def get_aggregate_tags(cls) -> set[str]: + return {"all", "easy"} + + +def _patch_converter_registry(instances: dict[str, Any]): + """Patch the converter registry singleton so ``.instances`` reflects ``instances``.""" + reg = MagicMock() + reg.instances.get.side_effect = lambda name: instances.get(name) + reg.instances.get_names.return_value = list(instances.keys()) + return patch.object(_svc_mod.ConverterRegistry, "get_registry_singleton", return_value=reg) + _REGISTRY_PATCH_BASE = "pyrit.registry" _MEMORY_PATCH = "pyrit.memory.CentralMemory.get_memory_instance" @@ -790,3 +814,108 @@ def test_completed_run_still_shows_full_counts(self, mock_memory) -> None: assert fetched.completed_attacks == 1 assert fetched.strategies_used == ["attack_a"] assert fetched.objective_achieved_rate == 100 + + +class TestResolveStrategiesAndConverters: + """Tests for per-technique converter resolution from ``--strategies`` tokens.""" + + def test_plain_strategy_no_converters(self, mock_memory) -> None: + service = ScenarioRunService() + with _patch_converter_registry({}): + enums, converters = service._resolve_strategies_and_converters( + tokens=["role_play"], strategy_class=_StubStrategy, scenario_name="x" + ) + assert enums == [_StubStrategy.ROLE_PLAY] + assert converters == {} + + def test_single_converter_appended(self, mock_memory) -> None: + conv = MagicMock(spec=PromptConverter) + service = ScenarioRunService() + with _patch_converter_registry({"translation_spanish": conv}): + enums, converters = service._resolve_strategies_and_converters( + tokens=["role_play:converter.translation_spanish"], + strategy_class=_StubStrategy, + scenario_name="x", + ) + assert enums == [_StubStrategy.ROLE_PLAY] + assert converters == {"role_play": [conv]} + + def test_aggregate_token_applies_converter_to_all_concrete(self, mock_memory) -> None: + conv = MagicMock(spec=PromptConverter) + service = ScenarioRunService() + with _patch_converter_registry({"c1": conv}): + enums, converters = service._resolve_strategies_and_converters( + tokens=["easy:converter.c1"], strategy_class=_StubStrategy, scenario_name="x" + ) + assert enums == [_StubStrategy.EASY] + assert converters == {"role_play": [conv], "single_turn": [conv]} + + def test_multiple_converters_preserve_order(self, mock_memory) -> None: + c1 = MagicMock(spec=PromptConverter) + c2 = MagicMock(spec=PromptConverter) + service = ScenarioRunService() + with _patch_converter_registry({"c1": c1, "c2": c2}): + _, converters = service._resolve_strategies_and_converters( + tokens=["role_play:converter.c1:converter.c2"], + strategy_class=_StubStrategy, + scenario_name="x", + ) + assert converters == {"role_play": [c1, c2]} + + def test_overlapping_tokens_append_in_order(self, mock_memory) -> None: + c1 = MagicMock(spec=PromptConverter) + c2 = MagicMock(spec=PromptConverter) + service = ScenarioRunService() + with _patch_converter_registry({"c1": c1, "c2": c2}): + _, converters = service._resolve_strategies_and_converters( + tokens=["easy:converter.c1", "role_play:converter.c2"], + strategy_class=_StubStrategy, + scenario_name="x", + ) + # role_play is targeted by both the aggregate token and the concrete token. + assert converters["role_play"] == [c1, c2] + assert converters["single_turn"] == [c1] + + def test_unknown_converter_raises(self, mock_memory) -> None: + service = ScenarioRunService() + with _patch_converter_registry({"known": MagicMock(spec=PromptConverter)}): + with pytest.raises(ValueError, match="not a registered converter"): + service._resolve_strategies_and_converters( + tokens=["role_play:converter.missing"], + strategy_class=_StubStrategy, + scenario_name="x", + ) + + def test_unknown_modifier_prefix_raises(self, mock_memory) -> None: + service = ScenarioRunService() + with _patch_converter_registry({}): + with pytest.raises(ValueError, match="Unknown strategy modifier"): + service._resolve_strategies_and_converters( + tokens=["role_play:scorer.something"], + strategy_class=_StubStrategy, + scenario_name="x", + ) + + def test_unknown_base_strategy_raises(self, mock_memory) -> None: + service = ScenarioRunService() + with _patch_converter_registry({}): + with pytest.raises(ValueError, match="not found for scenario"): + service._resolve_strategies_and_converters( + tokens=["nope:converter.c1"], + strategy_class=_StubStrategy, + scenario_name="x", + ) + + async def test_start_run_forwards_strategy_converters(self, mock_all_registries) -> None: + """A converter token is resolved and forwarded to ``initialize_async`` as ``strategy_converters``.""" + conv = MagicMock(spec=PromptConverter) + scenario_instance = mock_all_registries["scenario_instance"] + scenario_instance._strategy_class = _StubStrategy + + service = ScenarioRunService() + with _patch_converter_registry({"translation_spanish": conv}): + await service.start_run_async(request=_make_request(strategies=["role_play:converter.translation_spanish"])) + + init_call = scenario_instance.initialize_async.await_args + assert init_call.kwargs["scenario_strategies"] == [_StubStrategy.ROLE_PLAY] + assert init_call.kwargs["strategy_converters"] == {"role_play": [conv]} diff --git a/tests/unit/cli/test_api_client.py b/tests/unit/cli/test_api_client.py index 95c232273e..6813a60f8b 100644 --- a/tests/unit/cli/test_api_client.py +++ b/tests/unit/cli/test_api_client.py @@ -204,6 +204,12 @@ async def test_list_targets_async(client, mock_httpx_client): mock_httpx_client.get.assert_awaited_once_with("/api/targets", params={"limit": 7}) +async def test_list_converters_async(client, mock_httpx_client): + mock_httpx_client.get.return_value = _make_response(json_data={"items": []}) + await client.list_converters_async() + mock_httpx_client.get.assert_awaited_once_with("/api/converters", params=None) + + # --------------------------------------------------------------------------- # Scenario runs # --------------------------------------------------------------------------- diff --git a/tests/unit/cli/test_output.py b/tests/unit/cli/test_output.py index 7d9c62d2cf..888397888a 100644 --- a/tests/unit/cli/test_output.py +++ b/tests/unit/cli/test_output.py @@ -219,6 +219,40 @@ def test_print_target_list_full(capsys): assert "Total targets: 3" in captured.out +# --------------------------------------------------------------------------- +# print_converter_list +# --------------------------------------------------------------------------- + + +def test_print_converter_list_empty(capsys): + _output.print_converter_list(items=[]) + captured = capsys.readouterr() + assert "No converters found in registry" in captured.out + assert "converter.translation_spanish" in captured.out + + +def test_print_converter_list_full(capsys): + items = [ + { + "converter_id": "translation_spanish", + "converter_type": "TranslationConverter", + "display_name": "Spanish translation", + }, + { + "converter_id": "pipeline_1", + "converter_type": "PromptConverterPipeline", + "sub_converter_ids": ["base64", "rot13"], + }, + ] + _output.print_converter_list(items=items) + captured = capsys.readouterr() + assert "translation_spanish" in captured.out + assert "Class: TranslationConverter" in captured.out + assert "Name: Spanish translation" in captured.out + assert "Sub-converters: base64, rot13" in captured.out + assert "Total converters: 2" in captured.out + + # --------------------------------------------------------------------------- # print_scenario_run_progress # --------------------------------------------------------------------------- diff --git a/tests/unit/cli/test_pyrit_scan.py b/tests/unit/cli/test_pyrit_scan.py index d204e4433c..956c0354cb 100644 --- a/tests/unit/cli/test_pyrit_scan.py +++ b/tests/unit/cli/test_pyrit_scan.py @@ -118,6 +118,10 @@ def test_parse_args_with_list_targets(self): args = pyrit_scan.parse_args(["--list-targets"]) assert args.list_targets is True + def test_parse_args_with_list_converters(self): + args = pyrit_scan.parse_args(["--list-converters"]) + assert args.list_converters is True + def test_parse_args_with_server_url(self): args = pyrit_scan.parse_args(["--list-scenarios", "--server-url", "http://remote:9000"]) assert args.server_url == "http://remote:9000" @@ -156,6 +160,7 @@ def _mock_api_client(): client.list_scenarios_async.return_value = {"items": [], "pagination": {"total": 0}} client.list_initializers_async.return_value = {"items": [], "pagination": {"total": 0}} client.list_targets_async.return_value = {"items": [], "pagination": {"total": 0}} + client.list_converters_async.return_value = {"items": []} client.get_scenario_async.return_value = { "scenario_name": "test_scenario", "supported_parameters": [], @@ -227,6 +232,18 @@ def test_main_list_targets(self, mock_client_class, mock_probe): assert result == 0 mock_client.list_targets_async.assert_awaited_once() + @patch("pyrit.cli._server_launcher.ServerLauncher.probe_health_async", new_callable=AsyncMock, return_value=True) + @patch("pyrit.cli.api_client.PyRITApiClient") + def test_main_list_converters(self, mock_client_class, mock_probe): + """Test main with --list-converters flag.""" + mock_client = _mock_api_client() + mock_client_class.return_value = mock_client + + result = pyrit_scan.main(["--list-converters"]) + + assert result == 0 + mock_client.list_converters_async.assert_awaited_once() + def test_main_no_args_shows_help(self): """Test main with no arguments shows help.""" result = pyrit_scan.main([]) diff --git a/tests/unit/cli/test_pyrit_shell.py b/tests/unit/cli/test_pyrit_shell.py index 788ed71ad5..70775196b2 100644 --- a/tests/unit/cli/test_pyrit_shell.py +++ b/tests/unit/cli/test_pyrit_shell.py @@ -21,6 +21,7 @@ def mock_api_client(): client.list_scenarios_async.return_value = {"items": [], "pagination": {"total": 0}} client.list_initializers_async.return_value = {"items": [], "pagination": {"total": 0}} client.list_targets_async.return_value = {"items": [], "pagination": {"total": 0}} + client.list_converters_async.return_value = {"items": []} client.list_scenario_runs_async.return_value = {"items": []} # Default: scenario fetch returns no declared params (back-compat for older tests) client.get_scenario_async.return_value = {"scenario_name": "foo", "supported_parameters": []} @@ -93,6 +94,17 @@ def test_do_list_targets(self, shell): s.do_list_targets("") client.list_targets_async.assert_awaited_once() + def test_do_list_converters(self, shell): + s, client = shell + s.do_list_converters("") + client.list_converters_async.assert_awaited_once() + + def test_do_list_converters_rejects_args(self, shell, capsys): + s, _ = shell + s.do_list_converters("extra") + captured = capsys.readouterr() + assert "does not accept arguments" in captured.out + def test_do_run_empty_args(self, shell, capsys): s, _ = shell s.do_run("") @@ -522,6 +534,12 @@ def test_list_targets_error(self, shell, capsys): s.do_list_targets("") assert "Error listing targets" in capsys.readouterr().out + def test_list_converters_error(self, shell, capsys): + s, client = shell + client.list_converters_async = AsyncMock(side_effect=RuntimeError("x")) + s.do_list_converters("") + assert "Error listing converters" in capsys.readouterr().out + def test_scenario_history_error(self, shell, capsys): s, client = shell client.list_scenario_runs_async = AsyncMock(side_effect=RuntimeError("x")) diff --git a/tests/unit/scenario/airt/test_rapid_response.py b/tests/unit/scenario/airt/test_rapid_response.py index 42a1059138..4fbe00da2d 100644 --- a/tests/unit/scenario/airt/test_rapid_response.py +++ b/tests/unit/scenario/airt/test_rapid_response.py @@ -341,6 +341,43 @@ async def test_single_technique_selection(self, mock_objective_target, mock_obje for a in attacks: assert isinstance(a.attack_technique.attack, RolePlayAttack) + async def test_strategy_converters_are_threaded_to_factory_create( + self, mock_objective_target, mock_objective_scorer + ): + """``strategy_converters`` passed to ``initialize_async`` reach ``factory.create`` for the keyed technique.""" + from pyrit.prompt_converter import Base64Converter + + strat = _strategy_class() + role_play = strat("role_play") + converter = Base64Converter() + captured: list[object] = [] + original_create = AttackTechniqueFactory.create + + def _spy_create(self, **kwargs): + captured.append(kwargs.get("extra_request_converters")) + return original_create(self, **kwargs) + + groups = {"hate": _make_seed_groups("hate")} + with ( + patch.object(DatasetConfiguration, "get_seed_attack_groups", return_value=groups), + patch.object(AttackTechniqueFactory, "create", _spy_create), + ): + scenario = RapidResponse(objective_scorer=mock_objective_scorer) + await scenario.initialize_async( + objective_target=mock_objective_target, + include_baseline=False, + scenario_strategies=[role_play], + strategy_converters={role_play.value: [converter]}, + ) + await scenario._get_atomic_attacks_async() + + # ROLE_PLAY was selected with a converter modifier, so every resulting factory.create + # call must receive the extra request converter. + assert captured + for extra in captured: + assert extra is not None + assert len(extra) == 1 + async def test_attack_count_is_techniques_times_datasets(self, mock_objective_target, mock_objective_scorer): """With 2 datasets and DEFAULT (2 techniques), expect 4 atomic attacks.""" two_datasets = { diff --git a/tests/unit/scenario/core/test_attack_technique_factory.py b/tests/unit/scenario/core/test_attack_technique_factory.py index c76a7724c8..9cdb6cdfef 100644 --- a/tests/unit/scenario/core/test_attack_technique_factory.py +++ b/tests/unit/scenario/core/test_attack_technique_factory.py @@ -14,6 +14,8 @@ ) from pyrit.executor.attack.single_turn.prompt_sending import PromptSendingAttack from pyrit.models import ComponentIdentifier, Identifiable, SeedAttackTechniqueGroup, SeedPrompt +from pyrit.prompt_converter import Base64Converter, ROT13Converter +from pyrit.prompt_normalizer import PromptConverterConfiguration from pyrit.prompt_target import PromptTarget from pyrit.scenario.core.attack_technique import AttackTechnique from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory, ScorerOverridePolicy @@ -286,6 +288,67 @@ def get_identifier(self): assert not technique.attack.adversarial_was_passed assert not technique.attack.converter_was_passed + def test_create_appends_extra_request_converters_without_baked(self): + """``extra_request_converters`` become the request converters when none are baked.""" + factory = AttackTechniqueFactory(name="test", attack_class=_StubAttack) + target = MagicMock(spec=PromptTarget) + extra = PromptConverterConfiguration.from_converters(converters=[Base64Converter()]) + + technique = factory.create( + objective_target=target, + attack_scoring_config=self._scoring(), + extra_request_converters=extra, + ) + + cfg = technique.attack.attack_converter_config + assert cfg.request_converters == extra + assert cfg.response_converters == [] + + def test_create_appends_extra_request_converters_on_top_of_baked(self): + """``extra_request_converters`` are appended after baked request converters; responses are preserved.""" + baked_request = PromptConverterConfiguration.from_converters(converters=[Base64Converter()]) + baked_response = PromptConverterConfiguration.from_converters(converters=[ROT13Converter()]) + baked = AttackConverterConfig(request_converters=baked_request, response_converters=baked_response) + factory = AttackTechniqueFactory( + name="test", + attack_class=_StubAttack, + attack_kwargs={"attack_converter_config": baked}, + ) + target = MagicMock(spec=PromptTarget) + extra = PromptConverterConfiguration.from_converters(converters=[Base64Converter()]) + + technique = factory.create( + objective_target=target, + attack_scoring_config=self._scoring(), + extra_request_converters=extra, + ) + + cfg = technique.attack.attack_converter_config + assert cfg.request_converters == baked_request + extra + assert cfg.response_converters == baked_response + + def test_create_extra_request_converters_skipped_when_unsupported(self): + """Attacks that don't accept ``attack_converter_config`` silently ignore extras.""" + + class _NoConverterAttack: + def __init__(self, *, objective_target, attack_scoring_config=None): + self.objective_target = objective_target + + def get_identifier(self): + return ComponentIdentifier(class_name="_NoConverterAttack", class_module="test") + + factory = AttackTechniqueFactory(name="test", attack_class=_NoConverterAttack, uses_adversarial=False) + target = MagicMock(spec=PromptTarget) + extra = PromptConverterConfiguration.from_converters(converters=[Base64Converter()]) + + technique = factory.create( + objective_target=target, + attack_scoring_config=self._scoring(), + extra_request_converters=extra, + ) + + assert isinstance(technique, AttackTechnique) + class TestFactoryIdentifier: """Tests for AttackTechniqueFactory._build_identifier().""" From 4585c55b74830f633b290a320a158e25c1e9e92f Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sun, 21 Jun 2026 21:08:09 -0700 Subject: [PATCH 2/2] Document per-technique converter syntax in scan epilog and shell run help Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyrit/cli/pyrit_scan.py | 4 ++++ pyrit/cli/pyrit_shell.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pyrit/cli/pyrit_scan.py b/pyrit/cli/pyrit_scan.py index 629506b4dc..baf7afbb31 100644 --- a/pyrit/cli/pyrit_scan.py +++ b/pyrit/cli/pyrit_scan.py @@ -91,6 +91,10 @@ def _print_cli_exception(*, exc: BaseException) -> None: --strategies role_play --dataset-names airt_hate --max-dataset-size 5 --max-concurrency 4 + # Attach registered converters to a technique (repeatable, applied in order) + pyrit_scan airt.rapid_response --target openai_chat + --strategies role_play:converter.translation_spanish:converter.leetspeak + # Run multi-turn red team agent with labels for tracking pyrit_scan airt.red_team_agent --target openai_chat --strategies crescendo diff --git a/pyrit/cli/pyrit_shell.py b/pyrit/cli/pyrit_shell.py index d36860c880..fdd84d5d3c 100644 --- a/pyrit/cli/pyrit_shell.py +++ b/pyrit/cli/pyrit_shell.py @@ -297,7 +297,11 @@ def do_run(self, line: str) -> None: Options: --target Target name (required) --initializers ... Initializer names (supports name:key=val syntax) - --strategies, -s ... Strategy names + --strategies, -s ... Strategy names. Append registered + converters to a technique with + ':converter.' (repeatable), e.g. + role_play:converter.translation_spanish. + Use list-converters to see names. --max-concurrency Maximum concurrent operations --max-retries Maximum retry attempts --memory-labels JSON string of labels