diff --git a/doc/scanner/airt.ipynb b/doc/scanner/airt.ipynb index e1881ef631..703c2db422 100644 --- a/doc/scanner/airt.ipynb +++ b/doc/scanner/airt.ipynb @@ -373,7 +373,7 @@ " --max-dataset-size 1\n", "```\n", "\n", - "**Available strategies:** ALL, MULTI_TURN, red_teaming" + "**Available strategies:** ALL, DEFAULT, MULTI_TURN, red_teaming" ] }, { diff --git a/doc/scanner/airt.py b/doc/scanner/airt.py index 05312e7b42..ca296caf67 100644 --- a/doc/scanner/airt.py +++ b/doc/scanner/airt.py @@ -129,7 +129,7 @@ # --max-dataset-size 1 # ``` # -# **Available strategies:** ALL, MULTI_TURN, red_teaming +# **Available strategies:** ALL, DEFAULT, MULTI_TURN, red_teaming # %% from pyrit.scenario.airt import Cyber, CyberStrategy diff --git a/pyrit/scenario/scenarios/airt/cyber.py b/pyrit/scenario/scenarios/airt/cyber.py index b4d4299af7..9a364e3aae 100644 --- a/pyrit/scenario/scenarios/airt/cyber.py +++ b/pyrit/scenario/scenarios/airt/cyber.py @@ -21,6 +21,9 @@ logger = logging.getLogger(__name__) +# Techniques Cyber selects from the shared catalog. Adding a technique here that also +# carries the ``core`` tag will pull it into the ``DEFAULT`` aggregate (see _build_cyber_strategy); +# revisit the aggregate wiring if a future technique should stay out of the default run. _CYBER_TECHNIQUE_NAMES = {"red_teaming"} @@ -34,6 +37,9 @@ def _build_cyber_strategy() -> type[ScenarioStrategy]: prepended automatically by ``Scenario._build_baseline_atomic_attack`` via ``BaselineAttackPolicy.Enabled``. + The ``DEFAULT`` aggregate is the curated default run; for Cyber it expands to the + same single ``red_teaming`` technique as ``ALL``. + Returns: type[ScenarioStrategy]: The dynamically generated strategy enum class. """ @@ -48,6 +54,11 @@ def _build_cyber_strategy() -> type[ScenarioStrategy]: class_name="CyberStrategy", factories=cyber_factories, aggregate_tags={ + # Cyber curates a single technique (red_teaming) at the scenario level. That + # technique carries the canonical ``core`` tag but not the catalog-wide + # ``default`` tag, so DEFAULT matches ``core`` here to select it (rather than + # tagging red_teaming ``default`` globally, which would alter other scenarios). + "default": TagQuery.any_of("core"), "multi_turn": TagQuery.any_of("multi_turn"), }, ) @@ -102,7 +113,7 @@ def __init__( version=self.VERSION, objective_scorer=self._objective_scorer, strategy_class=strategy_class, - default_strategy=strategy_class("all"), + default_strategy=strategy_class("default"), default_dataset_config=DatasetConfiguration(dataset_names=["airt_malware"], max_dataset_size=4), scenario_result_id=scenario_result_id, ) diff --git a/tests/unit/scenario/airt/test_cyber.py b/tests/unit/scenario/airt/test_cyber.py index de86aa365e..152ca4dc1c 100644 --- a/tests/unit/scenario/airt/test_cyber.py +++ b/tests/unit/scenario/airt/test_cyber.py @@ -133,9 +133,25 @@ def test_get_strategy_class(self): strat = _strategy_class() assert Cyber()._strategy_class is strat - def test_get_default_strategy_returns_all(self): + def test_get_default_strategy_returns_default(self): strat = _strategy_class() - assert Cyber()._default_strategy == strat.ALL + assert Cyber()._default_strategy == strat.DEFAULT + + def test_default_aggregate_expands_to_red_teaming(self): + """DEFAULT must be non-empty and select the single curated technique. + + Guards against wiring the ``default`` aggregate to a tag ``red_teaming`` lacks + (e.g. the catalog ``default`` tag), which would silently make DEFAULT empty and + collapse the default run to baseline-only. + """ + strat = _strategy_class() + assert "default" in strat.get_aggregate_tags() + default_members = strat.expand({strat.DEFAULT}) + assert default_members == [strat("red_teaming")] + + def test_default_is_subset_of_all(self): + strat = _strategy_class() + assert set(strat.expand({strat.DEFAULT})) <= set(strat.expand({strat.ALL})) def test_default_dataset_config_has_malware_dataset(self): config = Cyber()._default_dataset_config @@ -163,7 +179,7 @@ def test_scenario_name_is_cyber(self, mock_objective_scorer): @patch.object( DatasetConfiguration, "get_seed_attack_groups", return_value={"malware": _make_seed_groups("malware")} ) - async def test_initialization_defaults_to_all_strategy( + async def test_initialization_defaults_to_default_strategy( self, _mock_groups, mock_objective_target, @@ -171,7 +187,7 @@ async def test_initialization_defaults_to_all_strategy( ): scenario = Cyber(objective_scorer=mock_objective_scorer) await scenario.initialize_async(objective_target=mock_objective_target) - # ALL expands to red_teaming (the only registered Cyber technique); a + # DEFAULT expands to red_teaming (the only registered Cyber technique); a # PromptSendingAttack baseline is added separately via the baseline # policy, not as a strategy. assert len(scenario._scenario_strategies) == 1 @@ -256,7 +272,7 @@ async def test_multi_turn_strategy_produces_red_teaming(self, mock_objective_tar assert technique_classes == {RedTeamingAttack} async def test_default_strategy_produces_red_teaming(self, mock_objective_target, mock_objective_scorer): - """Default (ALL) should produce RedTeaming. PromptSendingAttack baseline is + """Default (DEFAULT) should produce RedTeaming. PromptSendingAttack baseline is prepended automatically by BaselineAttackPolicy.Enabled when include_baseline=True (the helper here uses include_baseline=False).""" attacks = await self._init_and_get_attacks(