Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/scanner/airt.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@
" --max-dataset-size 1\n",
"```\n",
"\n",
"**Available strategies:** ALL, MULTI_TURN, red_teaming"
"**Available strategies:** ALL, DEFAULT, MULTI_TURN, red_teaming"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion doc/scanner/airt.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@
# --max-dataset-size 1
# ```
#
# **Available strategies:** ALL, MULTI_TURN, red_teaming
# **Available strategies:** ALL, DEFAULT, MULTI_TURN, red_teaming

# %%
from pyrit.scenario.airt import Cyber, CyberStrategy
Expand Down
13 changes: 12 additions & 1 deletion pyrit/scenario/scenarios/airt/cyber.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@

logger = logging.getLogger(__name__)

# Techniques Cyber selects from the shared catalog. Adding a technique here that also
# carries the ``core`` tag will pull it into the ``DEFAULT`` aggregate (see _build_cyber_strategy);
# revisit the aggregate wiring if a future technique should stay out of the default run.
_CYBER_TECHNIQUE_NAMES = {"red_teaming"}


Expand All @@ -34,6 +37,9 @@ def _build_cyber_strategy() -> type[ScenarioStrategy]:
prepended automatically by ``Scenario._build_baseline_atomic_attack`` via
``BaselineAttackPolicy.Enabled``.

The ``DEFAULT`` aggregate is the curated default run; for Cyber it expands to the
same single ``red_teaming`` technique as ``ALL``.

Returns:
type[ScenarioStrategy]: The dynamically generated strategy enum class.
"""
Expand All @@ -48,6 +54,11 @@ def _build_cyber_strategy() -> type[ScenarioStrategy]:
class_name="CyberStrategy",
factories=cyber_factories,
aggregate_tags={
# Cyber curates a single technique (red_teaming) at the scenario level. That
# technique carries the canonical ``core`` tag but not the catalog-wide
# ``default`` tag, so DEFAULT matches ``core`` here to select it (rather than
# tagging red_teaming ``default`` globally, which would alter other scenarios).
"default": TagQuery.any_of("core"),
"multi_turn": TagQuery.any_of("multi_turn"),
},
)
Expand Down Expand Up @@ -102,7 +113,7 @@ def __init__(
version=self.VERSION,
objective_scorer=self._objective_scorer,
strategy_class=strategy_class,
default_strategy=strategy_class("all"),
default_strategy=strategy_class("default"),
default_dataset_config=DatasetConfiguration(dataset_names=["airt_malware"], max_dataset_size=4),
scenario_result_id=scenario_result_id,
)
Expand Down
26 changes: 21 additions & 5 deletions tests/unit/scenario/airt/test_cyber.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,25 @@ def test_get_strategy_class(self):
strat = _strategy_class()
assert Cyber()._strategy_class is strat

def test_get_default_strategy_returns_all(self):
def test_get_default_strategy_returns_default(self):
strat = _strategy_class()
assert Cyber()._default_strategy == strat.ALL
assert Cyber()._default_strategy == strat.DEFAULT

def test_default_aggregate_expands_to_red_teaming(self):
"""DEFAULT must be non-empty and select the single curated technique.

Guards against wiring the ``default`` aggregate to a tag ``red_teaming`` lacks
(e.g. the catalog ``default`` tag), which would silently make DEFAULT empty and
collapse the default run to baseline-only.
"""
strat = _strategy_class()
assert "default" in strat.get_aggregate_tags()
default_members = strat.expand({strat.DEFAULT})
assert default_members == [strat("red_teaming")]

def test_default_is_subset_of_all(self):
strat = _strategy_class()
assert set(strat.expand({strat.DEFAULT})) <= set(strat.expand({strat.ALL}))

def test_default_dataset_config_has_malware_dataset(self):
config = Cyber()._default_dataset_config
Expand Down Expand Up @@ -163,15 +179,15 @@ def test_scenario_name_is_cyber(self, mock_objective_scorer):
@patch.object(
DatasetConfiguration, "get_seed_attack_groups", return_value={"malware": _make_seed_groups("malware")}
)
async def test_initialization_defaults_to_all_strategy(
async def test_initialization_defaults_to_default_strategy(
self,
_mock_groups,
mock_objective_target,
mock_objective_scorer,
):
scenario = Cyber(objective_scorer=mock_objective_scorer)
await scenario.initialize_async(objective_target=mock_objective_target)
# ALL expands to red_teaming (the only registered Cyber technique); a
# DEFAULT expands to red_teaming (the only registered Cyber technique); a
# PromptSendingAttack baseline is added separately via the baseline
# policy, not as a strategy.
assert len(scenario._scenario_strategies) == 1
Expand Down Expand Up @@ -256,7 +272,7 @@ async def test_multi_turn_strategy_produces_red_teaming(self, mock_objective_tar
assert technique_classes == {RedTeamingAttack}

async def test_default_strategy_produces_red_teaming(self, mock_objective_target, mock_objective_scorer):
"""Default (ALL) should produce RedTeaming. PromptSendingAttack baseline is
"""Default (DEFAULT) should produce RedTeaming. PromptSendingAttack baseline is
prepended automatically by BaselineAttackPolicy.Enabled when
include_baseline=True (the helper here uses include_baseline=False)."""
attacks = await self._init_and_get_attacks(
Expand Down
Loading